Skip to content

Commit ad0e183

Browse files
Enable Unpack/TypeVarTuple support (#16354)
Fixes #12280 Fixes #14697 In this PR: * Enable `TypeVarTuple` and `Unpack` features. * Delete the old blanket `--enable-incomplete-features` flag that was deprecated a year ago. * Switch couple corner cases to `PreciseTupleTypes` feature. * Add the draft docs about the new feature. * Handle a previously unhandled case where variadic tuple appears in string formatting (discovered on mypy self-check, where `PreciseTupleTypes` is already enabled). --------- Co-authored-by: Jelle Zijlstra <[email protected]>
1 parent b064a5c commit ad0e183

15 files changed

+116
-55
lines changed

docs/source/command_line.rst

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -991,6 +991,58 @@ format into the specified directory.
991991
library or specify mypy installation with the setuptools extra
992992
``mypy[reports]``.
993993

994+
995+
Enabling incomplete/experimental features
996+
*****************************************
997+
998+
.. option:: --enable-incomplete-feature FEATURE
999+
1000+
Some features may require several mypy releases to implement, for example
1001+
due to their complexity, potential for backwards incompatibility, or
1002+
ambiguous semantics that would benefit from feedback from the community.
1003+
You can enable such features for early preview using this flag. Note that
1004+
it is not guaranteed that all features will be ultimately enabled by
1005+
default. In *rare cases* we may decide to not go ahead with certain
1006+
features.
1007+
1008+
List of currently incomplete/experimental features:
1009+
1010+
* ``PreciseTupleTypes``: this feature will infer more precise tuple types in
1011+
various scenarios. Before variadic types were added to the Python type system
1012+
by :pep:`646`, it was impossible to express a type like "a tuple with
1013+
at least two integers". The best type available was ``tuple[int, ...]``.
1014+
Therefore, mypy applied very lenient checking for variable-length tuples.
1015+
Now this type can be expressed as ``tuple[int, int, *tuple[int, ...]]``.
1016+
For such more precise types (when explicitly *defined* by a user) mypy,
1017+
for example, warns about unsafe index access, and generally handles them
1018+
in a type-safe manner. However, to avoid problems in existing code, mypy
1019+
does not *infer* these precise types when it technically can. Here are
1020+
notable examples where ``PreciseTupleTypes`` infers more precise types:
1021+
1022+
.. code-block:: python
1023+
1024+
numbers: tuple[int, ...]
1025+
1026+
more_numbers = (1, *numbers, 1)
1027+
reveal_type(more_numbers)
1028+
# Without PreciseTupleTypes: tuple[int, ...]
1029+
# With PreciseTupleTypes: tuple[int, *tuple[int, ...], int]
1030+
1031+
other_numbers = (1, 1) + numbers
1032+
reveal_type(other_numbers)
1033+
# Without PreciseTupleTypes: tuple[int, ...]
1034+
# With PreciseTupleTypes: tuple[int, int, *tuple[int, ...]]
1035+
1036+
if len(numbers) > 2:
1037+
reveal_type(numbers)
1038+
# Without PreciseTupleTypes: tuple[int, ...]
1039+
# With PreciseTupleTypes: tuple[int, int, int, *tuple[int, ...]]
1040+
else:
1041+
reveal_type(numbers)
1042+
# Without PreciseTupleTypes: tuple[int, ...]
1043+
# With PreciseTupleTypes: tuple[()] | tuple[int] | tuple[int, int]
1044+
1045+
9941046
Miscellaneous
9951047
*************
9961048

mypy/checkexpr.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@
9797
YieldExpr,
9898
YieldFromExpr,
9999
)
100-
from mypy.options import TYPE_VAR_TUPLE
100+
from mypy.options import PRECISE_TUPLE_TYPES
101101
from mypy.plugin import (
102102
FunctionContext,
103103
FunctionSigContext,
@@ -3377,7 +3377,7 @@ def visit_op_expr(self, e: OpExpr) -> Type:
33773377
):
33783378
return self.concat_tuples(proper_left_type, proper_right_type)
33793379
elif (
3380-
TYPE_VAR_TUPLE in self.chk.options.enable_incomplete_feature
3380+
PRECISE_TUPLE_TYPES in self.chk.options.enable_incomplete_feature
33813381
and isinstance(proper_right_type, Instance)
33823382
and self.chk.type_is_iterable(proper_right_type)
33833383
):
@@ -3411,7 +3411,7 @@ def visit_op_expr(self, e: OpExpr) -> Type:
34113411
if is_named_instance(proper_right_type, "builtins.dict"):
34123412
use_reverse = USE_REVERSE_NEVER
34133413

3414-
if TYPE_VAR_TUPLE in self.chk.options.enable_incomplete_feature:
3414+
if PRECISE_TUPLE_TYPES in self.chk.options.enable_incomplete_feature:
34153415
# Handle tuple[X, ...] + tuple[Y, Z] = tuple[*tuple[X, ...], Y, Z].
34163416
if (
34173417
e.op == "+"
@@ -4988,7 +4988,7 @@ def visit_tuple_expr(self, e: TupleExpr) -> Type:
49884988
j += len(tt.items)
49894989
else:
49904990
if (
4991-
TYPE_VAR_TUPLE in self.chk.options.enable_incomplete_feature
4991+
PRECISE_TUPLE_TYPES in self.chk.options.enable_incomplete_feature
49924992
and not seen_unpack_in_items
49934993
):
49944994
# Handle (x, *y, z), where y is e.g. tuple[Y, ...].

mypy/checkstrformat.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,11 @@
4747
TupleType,
4848
Type,
4949
TypeOfAny,
50+
TypeVarTupleType,
5051
TypeVarType,
5152
UnionType,
53+
UnpackType,
54+
find_unpack_in_list,
5255
get_proper_type,
5356
get_proper_types,
5457
)
@@ -728,6 +731,22 @@ def check_simple_str_interpolation(
728731
rep_types: list[Type] = []
729732
if isinstance(rhs_type, TupleType):
730733
rep_types = rhs_type.items
734+
unpack_index = find_unpack_in_list(rep_types)
735+
if unpack_index is not None:
736+
# TODO: we should probably warn about potentially short tuple.
737+
# However, without special-casing for tuple(f(i) for in other_tuple)
738+
# this causes false positive on mypy self-check in report.py.
739+
extras = max(0, len(checkers) - len(rep_types) + 1)
740+
unpacked = rep_types[unpack_index]
741+
assert isinstance(unpacked, UnpackType)
742+
unpacked = get_proper_type(unpacked.type)
743+
if isinstance(unpacked, TypeVarTupleType):
744+
unpacked = get_proper_type(unpacked.upper_bound)
745+
assert (
746+
isinstance(unpacked, Instance) and unpacked.type.fullname == "builtins.tuple"
747+
)
748+
unpack_items = [unpacked.args[0]] * extras
749+
rep_types = rep_types[:unpack_index] + unpack_items + rep_types[unpack_index + 1 :]
731750
elif isinstance(rhs_type, AnyType):
732751
return
733752
elif isinstance(rhs_type, Instance) and rhs_type.type.fullname == "builtins.tuple":

mypy/main.py

Lines changed: 5 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
from mypy.find_sources import InvalidSourceList, create_source_list
2323
from mypy.fscache import FileSystemCache
2424
from mypy.modulefinder import BuildSource, FindModuleCache, SearchPaths, get_search_dirs, mypy_path
25-
from mypy.options import INCOMPLETE_FEATURES, BuildType, Options
25+
from mypy.options import COMPLETE_FEATURES, INCOMPLETE_FEATURES, BuildType, Options
2626
from mypy.split_namespace import SplitNamespace
2727
from mypy.version import __version__
2828

@@ -1151,10 +1151,7 @@ def add_invertible_flag(
11511151
# --debug-serialize will run tree.serialize() even if cache generation is disabled.
11521152
# Useful for mypy_primer to detect serialize errors earlier.
11531153
parser.add_argument("--debug-serialize", action="store_true", help=argparse.SUPPRESS)
1154-
# This one is deprecated, but we will keep it for few releases.
1155-
parser.add_argument(
1156-
"--enable-incomplete-features", action="store_true", help=argparse.SUPPRESS
1157-
)
1154+
11581155
parser.add_argument(
11591156
"--disable-bytearray-promotion", action="store_true", help=argparse.SUPPRESS
11601157
)
@@ -1334,14 +1331,10 @@ def set_strict_flags() -> None:
13341331

13351332
# Validate incomplete features.
13361333
for feature in options.enable_incomplete_feature:
1337-
if feature not in INCOMPLETE_FEATURES:
1334+
if feature not in INCOMPLETE_FEATURES | COMPLETE_FEATURES:
13381335
parser.error(f"Unknown incomplete feature: {feature}")
1339-
if options.enable_incomplete_features:
1340-
print(
1341-
"Warning: --enable-incomplete-features is deprecated, use"
1342-
" --enable-incomplete-feature=FEATURE instead"
1343-
)
1344-
options.enable_incomplete_feature = list(INCOMPLETE_FEATURES)
1336+
if feature in COMPLETE_FEATURES:
1337+
print(f"Warning: {feature} is already enabled by default")
13451338

13461339
# Compute absolute path for custom typeshed (if present).
13471340
if options.custom_typeshed_dir is not None:

mypy/options.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -69,11 +69,12 @@ class BuildType:
6969
}
7070
) - {"debug_cache"}
7171

72-
# Features that are currently incomplete/experimental
72+
# Features that are currently (or were recently) incomplete/experimental
7373
TYPE_VAR_TUPLE: Final = "TypeVarTuple"
7474
UNPACK: Final = "Unpack"
7575
PRECISE_TUPLE_TYPES: Final = "PreciseTupleTypes"
76-
INCOMPLETE_FEATURES: Final = frozenset((TYPE_VAR_TUPLE, UNPACK, PRECISE_TUPLE_TYPES))
76+
INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES,))
77+
COMPLETE_FEATURES: Final = frozenset((TYPE_VAR_TUPLE, UNPACK))
7778

7879

7980
class Options:
@@ -307,7 +308,6 @@ def __init__(self) -> None:
307308
self.dump_type_stats = False
308309
self.dump_inference_stats = False
309310
self.dump_build_stats = False
310-
self.enable_incomplete_features = False # deprecated
311311
self.enable_incomplete_feature: list[str] = []
312312
self.timing_stats: str | None = None
313313
self.line_checking_stats: str | None = None

mypy/semanal.py

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,7 @@
179179
type_aliases_source_versions,
180180
typing_extensions_aliases,
181181
)
182-
from mypy.options import TYPE_VAR_TUPLE, Options
182+
from mypy.options import Options
183183
from mypy.patterns import (
184184
AsPattern,
185185
ClassPattern,
@@ -4417,9 +4417,6 @@ def process_typevartuple_declaration(self, s: AssignmentStmt) -> bool:
44174417
else:
44184418
self.fail(f'Unexpected keyword argument "{param_name}" for "TypeVarTuple"', s)
44194419

4420-
if not self.incomplete_feature_enabled(TYPE_VAR_TUPLE, s):
4421-
return False
4422-
44234420
name = self.extract_typevarlike_name(s, call)
44244421
if name is None:
44254422
return False

mypy/test/testcheck.py

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
from mypy.build import Graph
1111
from mypy.errors import CompileError
1212
from mypy.modulefinder import BuildSource, FindModuleCache, SearchPaths
13-
from mypy.options import TYPE_VAR_TUPLE, UNPACK
1413
from mypy.test.config import test_data_prefix, test_temp_dir
1514
from mypy.test.data import DataDrivenTestCase, DataSuite, FileOperation, module_from_path
1615
from mypy.test.helpers import (
@@ -125,8 +124,6 @@ def run_case_once(
125124
# Parse options after moving files (in case mypy.ini is being moved).
126125
options = parse_options(original_program_text, testcase, incremental_step)
127126
options.use_builtins_fixtures = True
128-
if not testcase.name.endswith("_no_incomplete"):
129-
options.enable_incomplete_feature += [TYPE_VAR_TUPLE, UNPACK]
130127
options.show_traceback = True
131128

132129
# Enable some options automatically based on test file name.

mypy/test/testfinegrained.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
from mypy.errors import CompileError
2929
from mypy.find_sources import create_source_list
3030
from mypy.modulefinder import BuildSource
31-
from mypy.options import TYPE_VAR_TUPLE, UNPACK, Options
31+
from mypy.options import Options
3232
from mypy.server.mergecheck import check_consistency
3333
from mypy.server.update import sort_messages_preserving_file_order
3434
from mypy.test.config import test_temp_dir
@@ -149,7 +149,6 @@ def get_options(self, source: str, testcase: DataDrivenTestCase, build_cache: bo
149149
options.use_fine_grained_cache = self.use_cache and not build_cache
150150
options.cache_fine_grained = self.use_cache
151151
options.local_partial_types = True
152-
options.enable_incomplete_feature = [TYPE_VAR_TUPLE, UNPACK]
153152
# Treat empty bodies safely for these test cases.
154153
options.allow_empty_bodies = not testcase.name.endswith("_no_empty")
155154
if re.search("flags:.*--follow-imports", source) is None:

mypy/test/testsemanal.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
from mypy.errors import CompileError
1111
from mypy.modulefinder import BuildSource
1212
from mypy.nodes import TypeInfo
13-
from mypy.options import TYPE_VAR_TUPLE, UNPACK, Options
13+
from mypy.options import Options
1414
from mypy.test.config import test_temp_dir
1515
from mypy.test.data import DataDrivenTestCase, DataSuite
1616
from mypy.test.helpers import (
@@ -45,7 +45,6 @@ def get_semanal_options(program_text: str, testcase: DataDrivenTestCase) -> Opti
4545
options.semantic_analysis_only = True
4646
options.show_traceback = True
4747
options.python_version = PYTHON3_VERSION
48-
options.enable_incomplete_feature = [TYPE_VAR_TUPLE, UNPACK]
4948
options.force_uppercase_builtins = True
5049
return options
5150

mypy/test/testtransform.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
from mypy import build
66
from mypy.errors import CompileError
77
from mypy.modulefinder import BuildSource
8-
from mypy.options import TYPE_VAR_TUPLE, UNPACK
98
from mypy.test.config import test_temp_dir
109
from mypy.test.data import DataDrivenTestCase, DataSuite
1110
from mypy.test.helpers import assert_string_arrays_equal, normalize_error_messages, parse_options
@@ -38,7 +37,6 @@ def test_transform(testcase: DataDrivenTestCase) -> None:
3837
options = parse_options(src, testcase, 1)
3938
options.use_builtins_fixtures = True
4039
options.semantic_analysis_only = True
41-
options.enable_incomplete_feature = [TYPE_VAR_TUPLE, UNPACK]
4240
options.show_traceback = True
4341
options.force_uppercase_builtins = True
4442
result = build.build(

0 commit comments

Comments
 (0)