From f0163c433c546d4e406539e8c9889c5a6917de2f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 9 Aug 2022 12:08:48 +0100 Subject: [PATCH 01/13] Try another approach --- mypy/fixup.py | 6 +++ mypy/messages.py | 5 ++ mypy/nodes.py | 5 ++ mypy/semanal.py | 49 ++++++++---------- mypy/semanal_namedtuple.py | 49 +++++++++++++++--- mypy/semanal_shared.py | 18 ++++++- mypy/server/astmerge.py | 2 + mypy/typeanal.py | 2 + test-data/unit/check-recursive-types.test | 61 ++++++++++++++++++++++- 9 files changed, 161 insertions(+), 36 deletions(-) diff --git a/mypy/fixup.py b/mypy/fixup.py index 08a17e541d44..56ca8c3f4263 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -337,6 +337,12 @@ def lookup_fully_qualified_alias( node = stnode.node if stnode else None if isinstance(node, TypeAlias): return node + elif isinstance(node, TypeInfo): + assert node.tuple_type + target = node.tuple_type.copy_modified(fallback=Instance(node, [])) + alias = TypeAlias(target, node.fullname, node.line, node.column) + node.tuple_alias = alias + return alias else: # Looks like a missing TypeAlias during an initial daemon load, put something there assert ( diff --git a/mypy/messages.py b/mypy/messages.py index d27dad0df5b7..8214455a12c6 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -2292,6 +2292,11 @@ def visit_instance(self, t: Instance) -> None: self.instances.append(t) super().visit_instance(t) + def visit_type_alias_type(self, t: TypeAliasType) -> None: + if t.alias and not t.is_recursive: + t.alias.target.accept(self) + super().visit_type_alias_type(t) + def find_type_overlaps(*types: Type) -> Set[str]: """Return a set of fullnames that share a short name and appear in either type. diff --git a/mypy/nodes.py b/mypy/nodes.py index 606a2073219f..b49412e45be9 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2651,6 +2651,7 @@ class is generic then it will be a type constructor of higher kind. "bases", "_promote", "tuple_type", + "tuple_alias", "is_named_tuple", "typeddict_type", "is_newtype", @@ -2789,6 +2790,9 @@ class is generic then it will be a type constructor of higher kind. # It is useful for plugins to add their data to save in the cache. metadata: Dict[str, JsonDict] + # Store type alias representing this type (for named tuples). + tuple_alias: Optional["TypeAlias"] + FLAGS: Final = [ "is_abstract", "is_enum", @@ -2835,6 +2839,7 @@ def __init__(self, names: "SymbolTable", defn: ClassDef, module_name: str) -> No self._promote = [] self.alt_promote = None self.tuple_type = None + self.tuple_alias = None self.is_named_tuple = False self.typeddict_type = None self.is_newtype = False diff --git a/mypy/semanal.py b/mypy/semanal.py index 3ac0af8ba11e..04c8e51304fd 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -221,11 +221,11 @@ PRIORITY_FALLBACKS, SemanticAnalyzerInterface, calculate_tuple_fallback, + has_placeholder, set_callable_name as set_callable_name, ) from mypy.semanal_typeddict import TypedDictAnalyzer from mypy.tvar_scope import TypeVarLikeScope -from mypy.type_visitor import TypeQuery from mypy.typeanal import ( TypeAnalyser, TypeVarLikeList, @@ -1424,7 +1424,12 @@ def analyze_class_body_common(self, defn: ClassDef) -> None: def analyze_namedtuple_classdef(self, defn: ClassDef) -> bool: """Check if this class can define a named tuple.""" - if defn.info and defn.info.is_named_tuple: + if ( + defn.info + and defn.info.is_named_tuple + and defn.info.tuple_type + and not has_placeholder(defn.info.tuple_type) + ): # Don't reprocess everything. We just need to process methods defined # in the named tuple class body. is_named_tuple, info = True, defn.info # type: bool, Optional[TypeInfo] @@ -1436,6 +1441,8 @@ def analyze_namedtuple_classdef(self, defn: ClassDef) -> bool: if info is None: self.mark_incomplete(defn.name, defn) else: + if info.tuple_type and has_placeholder(info.tuple_type): + self.defer(force_progress=True) self.prepare_class_def(defn, info) with self.scope.class_scope(defn.info): with self.named_tuple_analyzer.save_namedtuple_body(info): @@ -1784,7 +1791,7 @@ def configure_base_classes( info.tuple_type = None for base, base_expr in bases: if isinstance(base, TupleType): - actual_base = self.configure_tuple_base_class(defn, base, base_expr) + actual_base = self.configure_tuple_base_class(defn, base) base_types.append(actual_base) elif isinstance(base, Instance): if base.type.is_newtype: @@ -1827,9 +1834,7 @@ def configure_base_classes( return self.calculate_class_mro(defn, self.object_type) - def configure_tuple_base_class( - self, defn: ClassDef, base: TupleType, base_expr: Expression - ) -> Instance: + def configure_tuple_base_class(self, defn: ClassDef, base: TupleType) -> Instance: info = defn.info # There may be an existing valid tuple type from previous semanal iterations. @@ -1838,10 +1843,6 @@ def configure_tuple_base_class( self.fail("Class has two incompatible bases derived from tuple", defn) defn.has_incompatible_baseclass = True info.tuple_type = base - if isinstance(base_expr, CallExpr): - defn.analyzed = NamedTupleExpr(base.partial_fallback.type) - defn.analyzed.line = defn.line - defn.analyzed.column = defn.column if base.partial_fallback.type.fullname == "builtins.tuple": # Fallback can only be safely calculated after semantic analysis, since base @@ -2626,7 +2627,10 @@ def analyze_enum_assign(self, s: AssignmentStmt) -> bool: def analyze_namedtuple_assign(self, s: AssignmentStmt) -> bool: """Check if s defines a namedtuple.""" if isinstance(s.rvalue, CallExpr) and isinstance(s.rvalue.analyzed, NamedTupleExpr): - return True # This is a valid and analyzed named tuple definition, nothing to do here. + if s.rvalue.analyzed.info.tuple_type and not has_placeholder( + s.rvalue.analyzed.info.tuple_type + ): + return True # This is a valid and analyzed named tuple definition, nothing to do here. if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], (NameExpr, MemberExpr)): return False lvalue = s.lvalues[0] @@ -3025,6 +3029,9 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool: # unless using PEP 613 `cls: TypeAlias = A` return False + if isinstance(s.rvalue, CallExpr) and s.rvalue.analyzed: + return False + existing = self.current_symbol_table().get(lvalue.name) # Third rule: type aliases can't be re-defined. For example: # A: Type[float] = int @@ -3157,9 +3164,8 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool: self.cannot_resolve_name(lvalue.name, "name", s) return True else: - self.progress = True # We need to defer so that this change can get propagated to base classes. - self.defer(s) + self.defer(s, force_progress=True) else: self.add_symbol(lvalue.name, alias_node, s) if isinstance(rvalue, RefExpr) and isinstance(rvalue.node, TypeAlias): @@ -5464,7 +5470,7 @@ def tvar_scope_frame(self, frame: TypeVarLikeScope) -> Iterator[None]: yield self.tvar_scope = old_scope - def defer(self, debug_context: Optional[Context] = None) -> None: + def defer(self, debug_context: Optional[Context] = None, force_progress: bool = False) -> None: """Defer current analysis target to be analyzed again. This must be called if something in the current target is @@ -5478,6 +5484,8 @@ def defer(self, debug_context: Optional[Context] = None) -> None: They are usually preferable to a direct defer() call. """ assert not self.final_iteration, "Must not defer during final iteration" + if force_progress: + self.progress = True self.deferred = True # Store debug info for this deferral. line = ( @@ -5979,19 +5987,6 @@ def is_future_flag_set(self, flag: str) -> bool: return self.modules[self.cur_mod_id].is_future_flag_set(flag) -class HasPlaceholders(TypeQuery[bool]): - def __init__(self) -> None: - super().__init__(any) - - def visit_placeholder_type(self, t: PlaceholderType) -> bool: - return True - - -def has_placeholder(typ: Type) -> bool: - """Check if a type contains any placeholder types (recursively).""" - return typ.accept(HasPlaceholders()) - - def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: if isinstance(sig, CallableType): if len(sig.arg_types) == 0: diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index 55e38cdfa11d..f2b49ede330c 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -35,6 +35,7 @@ SymbolTableNode, TempNode, TupleExpr, + TypeAlias, TypeInfo, TypeVarExpr, Var, @@ -44,12 +45,14 @@ PRIORITY_FALLBACKS, SemanticAnalyzerInterface, calculate_tuple_fallback, + has_placeholder, set_callable_name, ) from mypy.types import ( TYPED_NAMEDTUPLE_NAMES, AnyType, CallableType, + Instance, LiteralType, TupleType, Type, @@ -109,8 +112,11 @@ def analyze_namedtuple_classdef( items, types, default_items = result if is_func_scope and "@" not in defn.name: defn.name += "@" + str(defn.line) + existing_info = None + if isinstance(defn.analyzed, NamedTupleExpr): + existing_info = defn.analyzed.info info = self.build_namedtuple_typeinfo( - defn.name, items, types, default_items, defn.line + defn.name, items, types, default_items, defn.line, existing_info ) defn.info = info defn.analyzed = NamedTupleExpr(info, is_typed=True) @@ -164,7 +170,9 @@ def check_namedtuple_classdef( if stmt.type is None: types.append(AnyType(TypeOfAny.unannotated)) else: - analyzed = self.api.anal_type(stmt.type) + analyzed = self.api.anal_type( + stmt.type, allow_placeholder=self.options.enable_recursive_aliases + ) if analyzed is None: # Something is incomplete. We need to defer this named tuple. return None @@ -226,7 +234,7 @@ def check_namedtuple( name += "@" + str(call.line) else: name = var_name = "namedtuple@" + str(call.line) - info = self.build_namedtuple_typeinfo(name, [], [], {}, node.line) + info = self.build_namedtuple_typeinfo(name, [], [], {}, node.line, None) self.store_namedtuple_info(info, var_name, call, is_typed) if name != var_name or is_func_scope: # NOTE: we skip local namespaces since they are not serialized. @@ -262,12 +270,28 @@ def check_namedtuple( } else: default_items = {} - info = self.build_namedtuple_typeinfo(name, items, types, default_items, node.line) + + existing_info = None + if isinstance(node.analyzed, NamedTupleExpr): + existing_info = node.analyzed.info + info = self.build_namedtuple_typeinfo( + name, items, types, default_items, node.line, existing_info + ) + + if isinstance(node.analyzed, NamedTupleExpr) and node.analyzed.info.tuple_type: + if has_placeholder(node.analyzed.info.tuple_type): + self.api.defer(force_progress=True) + # No need to store in symbol tables. + return typename, info + # If var_name is not None (i.e. this is not a base class expression), we always # store the generated TypeInfo under var_name in the current scope, so that # other definitions can use it. if var_name: self.store_namedtuple_info(info, var_name, call, is_typed) + else: + call.analyzed = NamedTupleExpr(info, is_typed=is_typed) + call.analyzed.set_line(call) # There are three cases where we need to store the generated TypeInfo # second time (for the purpose of serialization): # * If there is a name mismatch like One = NamedTuple('Other', [...]) @@ -408,7 +432,9 @@ def parse_namedtuple_fields_with_types( except TypeTranslationError: self.fail("Invalid field type", type_node) return None - analyzed = self.api.anal_type(type) + analyzed = self.api.anal_type( + type, allow_placeholder=self.options.enable_recursive_aliases + ) # Workaround #4987 and avoid introducing a bogus UnboundType if isinstance(analyzed, UnboundType): analyzed = AnyType(TypeOfAny.from_error) @@ -428,6 +454,7 @@ def build_namedtuple_typeinfo( types: List[Type], default_items: Mapping[str, Expression], line: int, + existing_info: Optional[TypeInfo], ) -> TypeInfo: strtype = self.api.named_type("builtins.str") implicit_any = AnyType(TypeOfAny.special_form) @@ -448,10 +475,15 @@ def build_namedtuple_typeinfo( literals: List[Type] = [LiteralType(item, strtype) for item in items] match_args_type = TupleType(literals, basetuple_type) - info = self.api.basic_new_typeinfo(name, fallback, line) + info = existing_info or self.api.basic_new_typeinfo(name, fallback, line) info.is_named_tuple = True tuple_base = TupleType(types, fallback) info.tuple_type = tuple_base + target = tuple_base.copy_modified(fallback=Instance(info, [])) + if not info.tuple_alias: + info.tuple_alias = TypeAlias(target, info.fullname, info.line, info.column) + else: + info.tuple_alias.target = target info.line = line # For use by mypyc. info.metadata["namedtuple"] = {"fields": items.copy()} @@ -459,7 +491,10 @@ def build_namedtuple_typeinfo( # We can't calculate the complete fallback type until after semantic # analysis, since otherwise base classes might be incomplete. Postpone a # callback function that patches the fallback. - self.api.schedule_patch(PRIORITY_FALLBACKS, lambda: calculate_tuple_fallback(tuple_base)) + if not has_placeholder(tuple_base): + self.api.schedule_patch( + PRIORITY_FALLBACKS, lambda: calculate_tuple_fallback(tuple_base) + ) def add_field( var: Var, is_initialized_in_class: bool = False, is_property: bool = False diff --git a/mypy/semanal_shared.py b/mypy/semanal_shared.py index fd7bc363b077..873ee2788d58 100644 --- a/mypy/semanal_shared.py +++ b/mypy/semanal_shared.py @@ -19,6 +19,7 @@ TypeInfo, ) from mypy.tvar_scope import TypeVarLikeScope +from mypy.type_visitor import TypeQuery from mypy.types import ( TPDICT_FB_NAMES, FunctionLike, @@ -26,6 +27,7 @@ Parameters, ParamSpecFlavor, ParamSpecType, + PlaceholderType, ProperType, TupleType, Type, @@ -82,7 +84,7 @@ def record_incomplete_ref(self) -> None: raise NotImplementedError @abstractmethod - def defer(self) -> None: + def defer(self, debug_context: Optional[Context] = ..., force_progress: bool = ...) -> None: raise NotImplementedError @abstractmethod @@ -147,6 +149,7 @@ def anal_type( allow_tuple_literal: bool = False, allow_unbound_tvars: bool = False, allow_required: bool = False, + allow_placeholder: bool = False, report_invalid_types: bool = True, ) -> Optional[Type]: raise NotImplementedError @@ -297,3 +300,16 @@ def paramspec_kwargs( column=column, prefix=prefix, ) + + +class HasPlaceholders(TypeQuery[bool]): + def __init__(self) -> None: + super().__init__(any) + + def visit_placeholder_type(self, t: PlaceholderType) -> bool: + return True + + +def has_placeholder(typ: Type) -> bool: + """Check if a type contains any placeholder types (recursively).""" + return typ.accept(HasPlaceholders()) diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index d90061b60cf7..0ced8bf8ee54 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -364,6 +364,8 @@ def process_type_info(self, info: Optional[TypeInfo]) -> None: self.fixup_type(target) self.fixup_type(info.tuple_type) self.fixup_type(info.typeddict_type) + if info.tuple_alias: + self.fixup_type(info.tuple_alias.target) info.defn.info = self.fixup(info) replace_nodes_in_symbol_table(info.names, self.replacements) for i, item in enumerate(info.mro): diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 552990a8482b..4e4904331a58 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -610,6 +610,8 @@ def analyze_type_with_type_info( if args: self.fail("Generic tuple types not supported", ctx) return AnyType(TypeOfAny.from_error) + if info.tuple_alias: + return TypeAliasType(info.tuple_alias, []) return tup.copy_modified(items=self.anal_array(tup.items), fallback=instance) td = info.typeddict_type if td is not None: diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index e8b223d08fd9..1ef14e476cc9 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -90,7 +90,7 @@ else: A = int | list[A] [builtins fixtures/isinstancelist.pyi] --- Tests duplicating some existing tests with recursive aliases enabled +-- Tests duplicating some type alias existing tests with recursive aliases enabled [case testRecursiveAliasesMutual] # flags: --enable-recursive-aliases @@ -426,3 +426,62 @@ U = Type[Union[int, U]] # E: Type[...] cannot contain another Type[...] x: U reveal_type(x) # N: Revealed type is "Type[Any]" [builtins fixtures/isinstancelist.pyi] + +[case testBasicNamedTuple] +# flags: --enable-recursive-aliases +from typing import NamedTuple, Optional + +NT = NamedTuple("NT", [("x", Optional[NT]), ("y", int)]) +nt: NT +reveal_type(nt) # N: Revealed type is "Tuple[Union[..., None], builtins.int, fallback=__main__.NT]" +reveal_type(nt.x) # N: Revealed type is "Union[Tuple[Union[..., None], builtins.int, fallback=__main__.NT], None]" +reveal_type(nt[0]) # N: Revealed type is "Union[Tuple[Union[..., None], builtins.int, fallback=__main__.NT], None]" +y: str +if nt.x is not None: + y = nt.x[0] # E: Incompatible types in assignment (expression has type "Optional[NT]", variable has type "str") +[builtins fixtures/tuple.pyi] + +[case testBasicNamedTupleSpecial] +# flags: --enable-recursive-aliases +from typing import NamedTuple + +NT = NamedTuple("NT", [("x", NT), ("y", int)]) +nt: NT +reveal_type(nt) # N: Revealed type is "Tuple[..., builtins.int, fallback=__main__.NT]" +reveal_type(nt.x) # N: Revealed type is "Tuple[Tuple[..., builtins.int, fallback=__main__.NT], builtins.int, fallback=__main__.NT]" +reveal_type(nt[0]) # N: Revealed type is "Tuple[Tuple[..., builtins.int, fallback=__main__.NT], builtins.int, fallback=__main__.NT]" +y: str +if nt.x is not None: + y = nt.x[0] # E: Incompatible types in assignment (expression has type "NT", variable has type "str") +# XXX check join no infinite recursion on fallbacks +[builtins fixtures/tuple.pyi] + +[case testBasicNamedTupleClass] +# flags: --enable-recursive-aliases +from typing import NamedTuple, Optional + +class NT(NamedTuple): + x: Optional[NT] + y: int + +nt: NT +reveal_type(nt) # N: Revealed type is "Tuple[Union[..., None], builtins.int, fallback=__main__.NT]" +reveal_type(nt.x) # N: Revealed type is "Union[Tuple[Union[..., None], builtins.int, fallback=__main__.NT], None]" +reveal_type(nt[0]) # N: Revealed type is "Union[Tuple[Union[..., None], builtins.int, fallback=__main__.NT], None]" +y: str +if nt.x is not None: + y = nt.x[0] # E: Incompatible types in assignment (expression has type "Optional[NT]", variable has type "str") +[builtins fixtures/tuple.pyi] + +[case testBasicRegularTupleClass] +# flags: --enable-recursive-aliases +[builtins fixtures/tuple.pyi] + +[case testBasicTupleClassesNewType] +# flags: --enable-recursive-aliases +# check both regular and named +[builtins fixtures/tuple.pyi] + +-- Add fine-grained tests + +-- Tests duplicating some named tuple existing tests with recursive aliases enabled From dde4b92038f410d244bb55f58da5ac5276f92a52 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 9 Aug 2022 13:45:46 +0100 Subject: [PATCH 02/13] Fix/simplify things --- mypy/semanal.py | 2 -- mypy/semanal_namedtuple.py | 20 ++++++++++++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 04c8e51304fd..2392ec1e3c05 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1441,8 +1441,6 @@ def analyze_namedtuple_classdef(self, defn: ClassDef) -> bool: if info is None: self.mark_incomplete(defn.name, defn) else: - if info.tuple_type and has_placeholder(info.tuple_type): - self.defer(force_progress=True) self.prepare_class_def(defn, info) with self.scope.class_scope(defn.info): with self.named_tuple_analyzer.save_namedtuple_body(info): diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index f2b49ede330c..557e342d1177 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -171,7 +171,9 @@ def check_namedtuple_classdef( types.append(AnyType(TypeOfAny.unannotated)) else: analyzed = self.api.anal_type( - stmt.type, allow_placeholder=self.options.enable_recursive_aliases + stmt.type, + allow_placeholder=self.options.enable_recursive_aliases + and not self.api.is_func_scope(), ) if analyzed is None: # Something is incomplete. We need to defer this named tuple. @@ -278,12 +280,6 @@ def check_namedtuple( name, items, types, default_items, node.line, existing_info ) - if isinstance(node.analyzed, NamedTupleExpr) and node.analyzed.info.tuple_type: - if has_placeholder(node.analyzed.info.tuple_type): - self.api.defer(force_progress=True) - # No need to store in symbol tables. - return typename, info - # If var_name is not None (i.e. this is not a base class expression), we always # store the generated TypeInfo under var_name in the current scope, so that # other definitions can use it. @@ -433,7 +429,9 @@ def parse_namedtuple_fields_with_types( self.fail("Invalid field type", type_node) return None analyzed = self.api.anal_type( - type, allow_placeholder=self.options.enable_recursive_aliases + type, + allow_placeholder=self.options.enable_recursive_aliases + and not self.api.is_func_scope(), ) # Workaround #4987 and avoid introducing a bogus UnboundType if isinstance(analyzed, UnboundType): @@ -478,6 +476,9 @@ def build_namedtuple_typeinfo( info = existing_info or self.api.basic_new_typeinfo(name, fallback, line) info.is_named_tuple = True tuple_base = TupleType(types, fallback) + old_tuple_type = None + if existing_info: + old_tuple_type = existing_info.tuple_type info.tuple_type = tuple_base target = tuple_base.copy_modified(fallback=Instance(info, [])) if not info.tuple_alias: @@ -488,6 +489,9 @@ def build_namedtuple_typeinfo( # For use by mypyc. info.metadata["namedtuple"] = {"fields": items.copy()} + if old_tuple_type and has_placeholder(old_tuple_type): + self.api.defer(force_progress=True) + # We can't calculate the complete fallback type until after semantic # analysis, since otherwise base classes might be incomplete. Postpone a # callback function that patches the fallback. From 50279bacacde4bda41733422d981b1ee9c7f7f40 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 9 Aug 2022 14:13:32 +0100 Subject: [PATCH 03/13] Add more tests; fix fine-grained --- mypy/server/deps.py | 21 ++++++++++++------ test-data/unit/check-recursive-types.test | 27 +++++++++++++++++++++-- test-data/unit/fine-grained.test | 24 ++++++++++++++++++++ 3 files changed, 63 insertions(+), 9 deletions(-) diff --git a/mypy/server/deps.py b/mypy/server/deps.py index 179e430afad5..092eb2968bce 100644 --- a/mypy/server/deps.py +++ b/mypy/server/deps.py @@ -941,18 +941,23 @@ def get_type_triggers(self, typ: Type) -> List[str]: return get_type_triggers(typ, self.use_logical_deps()) -def get_type_triggers(typ: Type, use_logical_deps: bool) -> List[str]: +def get_type_triggers( + typ: Type, use_logical_deps: bool, seen_aliases: Optional[Set[TypeAliasType]] = None +) -> List[str]: """Return all triggers that correspond to a type becoming stale.""" - return typ.accept(TypeTriggersVisitor(use_logical_deps)) + return typ.accept(TypeTriggersVisitor(use_logical_deps, seen_aliases)) class TypeTriggersVisitor(TypeVisitor[List[str]]): - def __init__(self, use_logical_deps: bool) -> None: + def __init__( + self, use_logical_deps: bool, seen_aliases: Optional[Set[TypeAliasType]] = None + ) -> None: self.deps: List[str] = [] + self.seen_aliases: Set[TypeAliasType] = seen_aliases or set() self.use_logical_deps = use_logical_deps def get_type_triggers(self, typ: Type) -> List[str]: - return get_type_triggers(typ, self.use_logical_deps) + return get_type_triggers(typ, self.use_logical_deps, self.seen_aliases) def visit_instance(self, typ: Instance) -> List[str]: trigger = make_trigger(typ.type.fullname) @@ -964,14 +969,16 @@ def visit_instance(self, typ: Instance) -> List[str]: return triggers def visit_type_alias_type(self, typ: TypeAliasType) -> List[str]: + if typ in self.seen_aliases: + return [] + self.seen_aliases.add(typ) assert typ.alias is not None trigger = make_trigger(typ.alias.fullname) triggers = [trigger] for arg in typ.args: triggers.extend(self.get_type_triggers(arg)) - # TODO: Add guard for infinite recursion here. Moreover, now that type aliases - # are its own kind of types we can simplify the logic to rely on intermediate - # dependencies (like for instance types). + # TODO: Now that type aliases are its own kind of types we can simplify + # the logic to rely on intermediate dependencies (like for instance types). triggers.extend(self.get_type_triggers(typ.alias.target)) return triggers diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index 1ef14e476cc9..020128a32462 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -475,13 +475,36 @@ if nt.x is not None: [case testBasicRegularTupleClass] # flags: --enable-recursive-aliases +from typing import Tuple + +x: B +class B(Tuple[B, int]): + x: int + +b, _ = x +reveal_type(b.x) [builtins fixtures/tuple.pyi] [case testBasicTupleClassesNewType] # flags: --enable-recursive-aliases -# check both regular and named +from typing import Tuple, NamedTuple, NewType + +x: C +class B(Tuple[B, int]): + x: int +C = NewType(B) +b, _ = x +reveal_type(b.x) + +y: BNT +class BNT(NamedTuple): + x: BNT + y: int +CNT = NewType(BNT) +bnt, _ = y +reveal_type(bnt.y) [builtins fixtures/tuple.pyi] --- Add fine-grained tests +-- TODO: Add more fine-grained tests for various kinds of recursive types -- Tests duplicating some named tuple existing tests with recursive aliases enabled diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 4a1dc5cc93c7..27bd857b70a7 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -3448,6 +3448,30 @@ f(a.x) [out] == +[case testNamedTupleUpdate5] +# flags: --enable-recursive-aliases +import b +[file a.py] +from typing import NamedTuple, Optional +class N(NamedTuple): + r: Optional[N] + x: int +x = N(None, 1) +[file a.py.2] +from typing import NamedTuple, Optional +class N(NamedTuple): + r: Optional[N] + x: str +x = N(None, 'hi') +[file b.py] +import a +def f(x: a.N) -> None: + pass +f(a.x) +[builtins fixtures/tuple.pyi] +[out] +== + [case testTypedDictRefresh] [builtins fixtures/dict.pyi] import a From 68263eeb68feb6e9e120cf3d66ba93e774325006 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 9 Aug 2022 15:08:43 +0100 Subject: [PATCH 04/13] Fix regular tuple bases and newtypes --- mypy/semanal.py | 7 +++++++ mypy/semanal_newtype.py | 1 + test-data/unit/check-recursive-types.test | 15 ++++++++------- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 2392ec1e3c05..65236daedf5e 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1841,6 +1841,13 @@ def configure_tuple_base_class(self, defn: ClassDef, base: TupleType) -> Instanc self.fail("Class has two incompatible bases derived from tuple", defn) defn.has_incompatible_baseclass = True info.tuple_type = base + target = base.copy_modified(fallback=Instance(info, [])) + if not info.tuple_alias: + info.tuple_alias = TypeAlias(target, info.fullname, info.line, info.column) + else: + if has_placeholder(info.tuple_alias.target): + self.defer(force_progress=True) + info.tuple_alias.target = target if base.partial_fallback.type.fullname == "builtins.tuple": # Fallback can only be safely calculated after semantic analysis, since base diff --git a/mypy/semanal_newtype.py b/mypy/semanal_newtype.py index 11d325b26bb1..e03cc1bcabe1 100644 --- a/mypy/semanal_newtype.py +++ b/mypy/semanal_newtype.py @@ -93,6 +93,7 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> bool: name, old_type, old_type.partial_fallback, s.line ) newtype_class_info.tuple_type = old_type + newtype_class_info.tuple_alias = old_type.partial_fallback.type.tuple_alias elif isinstance(old_type, Instance): if old_type.type.is_protocol: self.fail("NewType cannot be used with protocol classes", s) diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index 020128a32462..ba82fe62482b 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -482,7 +482,7 @@ class B(Tuple[B, int]): x: int b, _ = x -reveal_type(b.x) +reveal_type(b.x) # N:Revealed type is "builtins.int" [builtins fixtures/tuple.pyi] [case testBasicTupleClassesNewType] @@ -492,17 +492,18 @@ from typing import Tuple, NamedTuple, NewType x: C class B(Tuple[B, int]): x: int -C = NewType(B) +C = NewType("C", B) b, _ = x -reveal_type(b.x) +reveal_type(b) # N: Revealed type is "Tuple[..., builtins.int, fallback=__main__.B]" +reveal_type(b.x) # N: Revealed type is "builtins.int" -y: BNT +y: CNT class BNT(NamedTuple): - x: BNT + x: CNT y: int -CNT = NewType(BNT) +CNT = NewType("CNT", BNT) bnt, _ = y -reveal_type(bnt.y) +reveal_type(bnt.y) # N: Revealed type is "builtins.int" [builtins fixtures/tuple.pyi] -- TODO: Add more fine-grained tests for various kinds of recursive types From 786655b3cae74f8081382394b3403e702efc5152 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 9 Aug 2022 18:19:01 +0100 Subject: [PATCH 05/13] Fix newtype tests --- mypy/checkmember.py | 5 +-- mypy/nodes.py | 11 ++++++ mypy/semanal_newtype.py | 43 ++++++++++++++++++----- test-data/unit/check-recursive-types.test | 2 +- 4 files changed, 49 insertions(+), 12 deletions(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 4bd645f1150c..a2f9db117325 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -563,6 +563,7 @@ def analyze_descriptor_access(descriptor_type: Type, mx: MemberContext) -> Type: The return type of the appropriate ``__get__`` overload for the descriptor. """ instance_type = get_proper_type(mx.original_type) + orig_descriptor_type = descriptor_type descriptor_type = get_proper_type(descriptor_type) if isinstance(descriptor_type, UnionType): @@ -571,10 +572,10 @@ def analyze_descriptor_access(descriptor_type: Type, mx: MemberContext) -> Type: [analyze_descriptor_access(typ, mx) for typ in descriptor_type.items] ) elif not isinstance(descriptor_type, Instance): - return descriptor_type + return orig_descriptor_type if not descriptor_type.type.has_readable_member("__get__"): - return descriptor_type + return orig_descriptor_type dunder_get = descriptor_type.type.get_method("__get__") if dunder_get is None: diff --git a/mypy/nodes.py b/mypy/nodes.py index b49412e45be9..15b02e2c15c0 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -3258,6 +3258,17 @@ def __init__( self.eager = eager super().__init__(line, column) + @classmethod + def from_tuple_type(cls, info: TypeInfo) -> "TypeAlias": + return TypeAlias( + info.tuple_type.copy_modified( + fallback=mypy.types.Instance(info, []) + ), + info.fullname, + info.line, + info.column, + ) + @property def name(self) -> str: return self._fullname.split(".")[-1] diff --git a/mypy/semanal_newtype.py b/mypy/semanal_newtype.py index e03cc1bcabe1..529966a5edb2 100644 --- a/mypy/semanal_newtype.py +++ b/mypy/semanal_newtype.py @@ -24,11 +24,12 @@ RefExpr, StrExpr, SymbolTableNode, + TypeAlias, TypeInfo, Var, ) from mypy.options import Options -from mypy.semanal_shared import SemanticAnalyzerInterface +from mypy.semanal_shared import SemanticAnalyzerInterface, has_placeholder from mypy.typeanal import check_for_explicit_any, has_any_from_unimported_type from mypy.types import ( AnyType, @@ -90,14 +91,20 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> bool: # Create the corresponding class definition if the aliased type is subtypeable if isinstance(old_type, TupleType): newtype_class_info = self.build_newtype_typeinfo( - name, old_type, old_type.partial_fallback, s.line + name, old_type, old_type.partial_fallback, s.line, call.analyzed.info ) newtype_class_info.tuple_type = old_type - newtype_class_info.tuple_alias = old_type.partial_fallback.type.tuple_alias + alias = TypeAlias.from_tuple_type(newtype_class_info) + if not newtype_class_info.tuple_alias: + newtype_class_info.tuple_alias = alias + else: + newtype_class_info.tuple_alias.target = alias.target elif isinstance(old_type, Instance): if old_type.type.is_protocol: self.fail("NewType cannot be used with protocol classes", s) - newtype_class_info = self.build_newtype_typeinfo(name, old_type, old_type, s.line) + newtype_class_info = self.build_newtype_typeinfo( + name, old_type, old_type, s.line, call.analyzed.info + ) else: if old_type is not None: message = "Argument 2 to NewType(...) must be subclassable (got {})" @@ -105,7 +112,9 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> bool: # Otherwise the error was already reported. old_type = AnyType(TypeOfAny.from_error) object_type = self.api.named_type("builtins.object") - newtype_class_info = self.build_newtype_typeinfo(name, old_type, object_type, s.line) + newtype_class_info = self.build_newtype_typeinfo( + name, old_type, object_type, s.line, call.analyzed.info + ) newtype_class_info.fallback_to_any = True check_for_explicit_any( @@ -195,9 +204,18 @@ def check_newtype_args( # We want to use our custom error message (see above), so we suppress # the default error message for invalid types here. - old_type = get_proper_type(self.api.anal_type(unanalyzed_type, report_invalid_types=False)) + old_type = get_proper_type( + self.api.anal_type( + unanalyzed_type, + report_invalid_types=False, + allow_placeholder=self.options.enable_recursive_aliases + and not self.api.is_func_scope(), + ) + ) should_defer = False - if old_type is None or isinstance(old_type, PlaceholderType): + if isinstance(old_type, PlaceholderType): + old_type = None + if old_type is None: should_defer = True # The caller of this function assumes that if we return a Type, it's always @@ -209,9 +227,14 @@ def check_newtype_args( return None if has_failed else old_type, should_defer def build_newtype_typeinfo( - self, name: str, old_type: Type, base_type: Instance, line: int + self, + name: str, + old_type: Type, + base_type: Instance, + line: int, + existing_info: Optional[TypeInfo], ) -> TypeInfo: - info = self.api.basic_new_typeinfo(name, base_type, line) + info = existing_info or self.api.basic_new_typeinfo(name, base_type, line) info.is_newtype = True # Add __init__ method @@ -232,6 +255,8 @@ def build_newtype_typeinfo( init_func._fullname = info.fullname + ".__init__" info.names["__init__"] = SymbolTableNode(MDEF, init_func) + if info.tuple_type and has_placeholder(info.tuple_type): + self.api.defer(force_progress=True) return info # Helpers diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index ba82fe62482b..3d24c91c9e65 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -482,7 +482,7 @@ class B(Tuple[B, int]): x: int b, _ = x -reveal_type(b.x) # N:Revealed type is "builtins.int" +reveal_type(b.x) # N: Revealed type is "builtins.int" [builtins fixtures/tuple.pyi] [case testBasicTupleClassesNewType] From 952692bc7b89e58bbcb70e9cd59e1ab4dbe17667 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 9 Aug 2022 19:05:37 +0100 Subject: [PATCH 06/13] Cleanup; add more tests --- mypy/fixup.py | 4 +- mypy/nodes.py | 5 +- mypy/semanal.py | 6 +- mypy/semanal_namedtuple.py | 6 +- mypy/semanal_newtype.py | 1 + test-data/unit/check-recursive-types.test | 99 ++++++++++++++++++++--- 6 files changed, 99 insertions(+), 22 deletions(-) diff --git a/mypy/fixup.py b/mypy/fixup.py index 56ca8c3f4263..d4c8e2766114 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -338,9 +338,7 @@ def lookup_fully_qualified_alias( if isinstance(node, TypeAlias): return node elif isinstance(node, TypeInfo): - assert node.tuple_type - target = node.tuple_type.copy_modified(fallback=Instance(node, [])) - alias = TypeAlias(target, node.fullname, node.line, node.column) + alias = TypeAlias.from_tuple_type(node) node.tuple_alias = alias return alias else: diff --git a/mypy/nodes.py b/mypy/nodes.py index 15b02e2c15c0..52820ad551e1 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -3260,10 +3260,9 @@ def __init__( @classmethod def from_tuple_type(cls, info: TypeInfo) -> "TypeAlias": + assert info.tuple_type return TypeAlias( - info.tuple_type.copy_modified( - fallback=mypy.types.Instance(info, []) - ), + info.tuple_type.copy_modified(fallback=mypy.types.Instance(info, [])), info.fullname, info.line, info.column, diff --git a/mypy/semanal.py b/mypy/semanal.py index 65236daedf5e..e356b5dbac8e 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1841,13 +1841,13 @@ def configure_tuple_base_class(self, defn: ClassDef, base: TupleType) -> Instanc self.fail("Class has two incompatible bases derived from tuple", defn) defn.has_incompatible_baseclass = True info.tuple_type = base - target = base.copy_modified(fallback=Instance(info, [])) + alias = TypeAlias.from_tuple_type(info) if not info.tuple_alias: - info.tuple_alias = TypeAlias(target, info.fullname, info.line, info.column) + info.tuple_alias = alias else: if has_placeholder(info.tuple_alias.target): self.defer(force_progress=True) - info.tuple_alias.target = target + info.tuple_alias.target = alias.target if base.partial_fallback.type.fullname == "builtins.tuple": # Fallback can only be safely calculated after semantic analysis, since base diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index 557e342d1177..c8a3c929c47c 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -480,11 +480,11 @@ def build_namedtuple_typeinfo( if existing_info: old_tuple_type = existing_info.tuple_type info.tuple_type = tuple_base - target = tuple_base.copy_modified(fallback=Instance(info, [])) + alias = TypeAlias.from_tuple_type(info) if not info.tuple_alias: - info.tuple_alias = TypeAlias(target, info.fullname, info.line, info.column) + info.tuple_alias = alias else: - info.tuple_alias.target = target + info.tuple_alias.target = alias.target info.line = line # For use by mypyc. info.metadata["namedtuple"] = {"fields": items.copy()} diff --git a/mypy/semanal_newtype.py b/mypy/semanal_newtype.py index 529966a5edb2..0788d5bf936d 100644 --- a/mypy/semanal_newtype.py +++ b/mypy/semanal_newtype.py @@ -89,6 +89,7 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> bool: return True # Create the corresponding class definition if the aliased type is subtypeable + assert isinstance(call.analyzed, NewTypeExpr) if isinstance(old_type, TupleType): newtype_class_info = self.build_newtype_typeinfo( name, old_type, old_type.partial_fallback, s.line, call.analyzed.info diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index 3d24c91c9e65..a7e894db7acd 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -90,7 +90,7 @@ else: A = int | list[A] [builtins fixtures/isinstancelist.pyi] --- Tests duplicating some type alias existing tests with recursive aliases enabled +-- Tests duplicating some existing type alias tests with recursive aliases enabled [case testRecursiveAliasesMutual] # flags: --enable-recursive-aliases @@ -427,7 +427,7 @@ x: U reveal_type(x) # N: Revealed type is "Type[Any]" [builtins fixtures/isinstancelist.pyi] -[case testBasicNamedTuple] +[case testBasicRecursiveNamedTuple] # flags: --enable-recursive-aliases from typing import NamedTuple, Optional @@ -441,9 +441,9 @@ if nt.x is not None: y = nt.x[0] # E: Incompatible types in assignment (expression has type "Optional[NT]", variable has type "str") [builtins fixtures/tuple.pyi] -[case testBasicNamedTupleSpecial] +[case testBasicRecursiveNamedTupleSpecial] # flags: --enable-recursive-aliases -from typing import NamedTuple +from typing import NamedTuple, TypeVar, Tuple NT = NamedTuple("NT", [("x", NT), ("y", int)]) nt: NT @@ -453,10 +453,17 @@ reveal_type(nt[0]) # N: Revealed type is "Tuple[Tuple[..., builtins.int, fallba y: str if nt.x is not None: y = nt.x[0] # E: Incompatible types in assignment (expression has type "NT", variable has type "str") -# XXX check join no infinite recursion on fallbacks + +T = TypeVar("T") +def f(a: T, b: T) -> T: ... +tnt: Tuple[NT] + +# Should these be tuple[object] instead? +reveal_type(f(nt, tnt)) # N: Revealed type is "builtins.tuple[Any, ...]" +reveal_type(f(tnt, nt)) # N: Revealed type is "builtins.tuple[Any, ...]" [builtins fixtures/tuple.pyi] -[case testBasicNamedTupleClass] +[case testBasicRecursiveNamedTupleClass] # flags: --enable-recursive-aliases from typing import NamedTuple, Optional @@ -473,7 +480,7 @@ if nt.x is not None: y = nt.x[0] # E: Incompatible types in assignment (expression has type "Optional[NT]", variable has type "str") [builtins fixtures/tuple.pyi] -[case testBasicRegularTupleClass] +[case testRecursiveRegularTupleClass] # flags: --enable-recursive-aliases from typing import Tuple @@ -485,7 +492,7 @@ b, _ = x reveal_type(b.x) # N: Revealed type is "builtins.int" [builtins fixtures/tuple.pyi] -[case testBasicTupleClassesNewType] +[case testRecursiveTupleClassesNewType] # flags: --enable-recursive-aliases from typing import Tuple, NamedTuple, NewType @@ -506,6 +513,78 @@ bnt, _ = y reveal_type(bnt.y) # N: Revealed type is "builtins.int" [builtins fixtures/tuple.pyi] --- TODO: Add more fine-grained tests for various kinds of recursive types +-- Tests duplicating some existing named tuple tests with recursive aliases enabled + +[case testMutuallyRecursiveNamedTuples] +# flags: --enable-recursive-aliases +from typing import Tuple, NamedTuple, TypeVar, Union + +A = NamedTuple('A', [('x', str), ('y', Tuple[B, ...])]) +class B(NamedTuple): + x: A + y: int --- Tests duplicating some named tuple existing tests with recursive aliases enabled +n: A +reveal_type(n) # N: Revealed type is "Tuple[builtins.str, builtins.tuple[Tuple[..., builtins.int, fallback=__main__.B], ...], fallback=__main__.A]" + +T = TypeVar("T") +S = TypeVar("S") +def foo(arg: Tuple[T, S]) -> Union[T, S]: ... +x = foo(n) +y: str = x # E: Incompatible types in assignment (expression has type "Union[str, Tuple[B, ...]]", variable has type "str") +[builtins fixtures/tuple.pyi] + +[case testMutuallyRecursiveNamedTuplesJoin] +# flags: --enable-recursive-aliases +from typing import NamedTuple, Tuple + +class B(NamedTuple): + x: Tuple[A, int] + y: int + +A = NamedTuple('A', [('x', str), ('y', B)]) +n: B +m: A +reveal_type(n.x) # N: Revealed type is "Tuple[Tuple[builtins.str, Tuple[Tuple[..., builtins.int], builtins.int, fallback=__main__.B], fallback=__main__.A], builtins.int]" +reveal_type(m[0]) # N: Revealed type is "builtins.str" +lst = [m, n] +reveal_type(lst[0]) # N: Revealed type is "Tuple[builtins.object, builtins.object]" +[builtins fixtures/tuple.pyi] + +[case testMutuallyRecursiveNamedTuplesClasses] +# flags: --enable-recursive-aliases +from typing import NamedTuple, Tuple + +class B(NamedTuple): + x: A + y: int +class A(NamedTuple): + x: str + y: B + +n: A +reveal_type(n.y[0]) # N: Revealed type is "Tuple[builtins.str, Tuple[Tuple[builtins.str, ..., fallback=__main__.A], builtins.int, fallback=__main__.B], fallback=__main__.A]" + +m: B +n = m.x +n = n.y.x + +t: Tuple[str, B] +t = n +t = m # E: Incompatible types in assignment (expression has type "B", variable has type "Tuple[str, B]") +[builtins fixtures/tuple.pyi] + +[case testMutuallyRecursiveNamedTuplesCalls] +# flags: --enable-recursive-aliases +from typing import NamedTuple + +B = NamedTuple('B', [('x', A), ('y', int)]) +A = NamedTuple('A', [('x', str), ('y', 'B')]) +n: A +def f(m: B) -> None: pass +reveal_type(n) # N: Revealed type is "Tuple[builtins.str, Tuple[..., builtins.int, fallback=__main__.B], fallback=__main__.A]" +reveal_type(f) # N: Revealed type is "def (m: Tuple[Tuple[builtins.str, ..., fallback=__main__.A], builtins.int, fallback=__main__.B])" +f(n) # E: Argument 1 to "f" has incompatible type "A"; expected "B" +[builtins fixtures/tuple.pyi] + +-- TODO: Add more fine-grained tests for various kinds of recursive types From 3aeb47075e029772b227229f1659fc96542534f1 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 9 Aug 2022 20:03:50 +0100 Subject: [PATCH 07/13] More cleanup and tests --- mypy/nodes.py | 10 ++++++++++ mypy/semanal.py | 11 +++-------- mypy/semanal_namedtuple.py | 22 ++++++++-------------- mypy/semanal_newtype.py | 8 +------- mypy/semanal_shared.py | 8 ++++---- mypy/typeanal.py | 3 +++ test-data/unit/check-recursive-types.test | 16 ++++++++++++++-- 7 files changed, 43 insertions(+), 35 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 52820ad551e1..c56147ae0530 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2970,6 +2970,15 @@ def direct_base_classes(self) -> "List[TypeInfo]": """ return [base.type for base in self.bases] + def update_tuple_type(self, typ: "mypy.types.TupleType") -> None: + """Update tuple_type and tuple_alias as needed.""" + self.tuple_type = typ + alias = TypeAlias.from_tuple_type(self) + if not self.tuple_alias: + self.tuple_alias = alias + else: + self.tuple_alias.target = alias.target + def __str__(self) -> str: """Return a string representation of the type. @@ -3260,6 +3269,7 @@ def __init__( @classmethod def from_tuple_type(cls, info: TypeInfo) -> "TypeAlias": + """Generate an alias to the tuple type described by a given TypeInfo.""" assert info.tuple_type return TypeAlias( info.tuple_type.copy_modified(fallback=mypy.types.Instance(info, [])), diff --git a/mypy/semanal.py b/mypy/semanal.py index e356b5dbac8e..7c1b5f9e3fd5 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1840,14 +1840,9 @@ def configure_tuple_base_class(self, defn: ClassDef, base: TupleType) -> Instanc if info.tuple_type and info.tuple_type != base: self.fail("Class has two incompatible bases derived from tuple", defn) defn.has_incompatible_baseclass = True - info.tuple_type = base - alias = TypeAlias.from_tuple_type(info) - if not info.tuple_alias: - info.tuple_alias = alias - else: - if has_placeholder(info.tuple_alias.target): - self.defer(force_progress=True) - info.tuple_alias.target = alias.target + if info.tuple_alias and has_placeholder(info.tuple_alias.target): + self.defer(force_progress=True) + info.update_tuple_type(base) if base.partial_fallback.type.fullname == "builtins.tuple": # Fallback can only be safely calculated after semantic analysis, since base diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index c8a3c929c47c..d6d622593961 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -35,7 +35,6 @@ SymbolTableNode, TempNode, TupleExpr, - TypeAlias, TypeInfo, TypeVarExpr, Var, @@ -52,7 +51,6 @@ TYPED_NAMEDTUPLE_NAMES, AnyType, CallableType, - Instance, LiteralType, TupleType, Type, @@ -170,6 +168,9 @@ def check_namedtuple_classdef( if stmt.type is None: types.append(AnyType(TypeOfAny.unannotated)) else: + # We never allow recursive types at function scope. Although it is + # possible to support this for named tuples, it is still tricky, and + # it would be inconsistent with type aliases. analyzed = self.api.anal_type( stmt.type, allow_placeholder=self.options.enable_recursive_aliases @@ -428,6 +429,7 @@ def parse_namedtuple_fields_with_types( except TypeTranslationError: self.fail("Invalid field type", type_node) return None + # We never allow recursive types at function scope. analyzed = self.api.anal_type( type, allow_placeholder=self.options.enable_recursive_aliases @@ -476,22 +478,13 @@ def build_namedtuple_typeinfo( info = existing_info or self.api.basic_new_typeinfo(name, fallback, line) info.is_named_tuple = True tuple_base = TupleType(types, fallback) - old_tuple_type = None - if existing_info: - old_tuple_type = existing_info.tuple_type - info.tuple_type = tuple_base - alias = TypeAlias.from_tuple_type(info) - if not info.tuple_alias: - info.tuple_alias = alias - else: - info.tuple_alias.target = alias.target + if info.tuple_alias and has_placeholder(info.tuple_alias.target): + self.api.defer(force_progress=True) + info.update_tuple_type(tuple_base) info.line = line # For use by mypyc. info.metadata["namedtuple"] = {"fields": items.copy()} - if old_tuple_type and has_placeholder(old_tuple_type): - self.api.defer(force_progress=True) - # We can't calculate the complete fallback type until after semantic # analysis, since otherwise base classes might be incomplete. Postpone a # callback function that patches the fallback. @@ -528,6 +521,7 @@ def add_field( if self.options.python_version >= (3, 10): add_field(Var("__match_args__", match_args_type), is_initialized_in_class=True) + assert info.tuple_type is not None # Set by update_tuple_type() above. tvd = TypeVarType( SELF_TVAR_NAME, info.fullname + "." + SELF_TVAR_NAME, -1, [], info.tuple_type ) diff --git a/mypy/semanal_newtype.py b/mypy/semanal_newtype.py index 0788d5bf936d..6fe6cd4a4295 100644 --- a/mypy/semanal_newtype.py +++ b/mypy/semanal_newtype.py @@ -24,7 +24,6 @@ RefExpr, StrExpr, SymbolTableNode, - TypeAlias, TypeInfo, Var, ) @@ -94,12 +93,7 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> bool: newtype_class_info = self.build_newtype_typeinfo( name, old_type, old_type.partial_fallback, s.line, call.analyzed.info ) - newtype_class_info.tuple_type = old_type - alias = TypeAlias.from_tuple_type(newtype_class_info) - if not newtype_class_info.tuple_alias: - newtype_class_info.tuple_alias = alias - else: - newtype_class_info.tuple_alias.target = alias.target + newtype_class_info.update_tuple_type(old_type) elif isinstance(old_type, Instance): if old_type.type.is_protocol: self.fail("NewType cannot be used with protocol classes", s) diff --git a/mypy/semanal_shared.py b/mypy/semanal_shared.py index 873ee2788d58..51f77c1b5d33 100644 --- a/mypy/semanal_shared.py +++ b/mypy/semanal_shared.py @@ -108,6 +108,10 @@ def is_future_flag_set(self, flag: str) -> bool: def is_stub_file(self) -> bool: raise NotImplementedError + @abstractmethod + def is_func_scope(self) -> bool: + raise NotImplementedError + @trait class SemanticAnalyzerInterface(SemanticAnalyzerCoreInterface): @@ -211,10 +215,6 @@ def qualified_name(self, n: str) -> str: def is_typeshed_stub_file(self) -> bool: raise NotImplementedError - @abstractmethod - def is_func_scope(self) -> bool: - raise NotImplementedError - def set_callable_name(sig: Type, fdef: FuncDef) -> ProperType: sig = get_proper_type(sig) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 4e4904331a58..22e467f49215 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -393,6 +393,8 @@ def cannot_resolve_type(self, t: UnboundType) -> None: # need access to MessageBuilder here. Also move the similar # message generation logic in semanal.py. self.api.fail(f'Cannot resolve name "{t.name}" (possible cyclic definition)', t) + if self.options.enable_recursive_aliases and self.api.is_func_scope(): + self.note("Recursive types are not allowed at function scope", t) def apply_concatenate_operator(self, t: UnboundType) -> Type: if len(t.args) == 0: @@ -611,6 +613,7 @@ def analyze_type_with_type_info( self.fail("Generic tuple types not supported", ctx) return AnyType(TypeOfAny.from_error) if info.tuple_alias: + # We don't support generic tuple types yet. return TypeAliasType(info.tuple_alias, []) return tup.copy_modified(items=self.anal_array(tup.items), fallback=instance) td = info.typeddict_type diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index a7e894db7acd..e0fe2013e940 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -545,7 +545,7 @@ class B(NamedTuple): A = NamedTuple('A', [('x', str), ('y', B)]) n: B m: A -reveal_type(n.x) # N: Revealed type is "Tuple[Tuple[builtins.str, Tuple[Tuple[..., builtins.int], builtins.int, fallback=__main__.B], fallback=__main__.A], builtins.int]" +s: str = n.x # E: Incompatible types in assignment (expression has type "Tuple[A, int]", variable has type "str") reveal_type(m[0]) # N: Revealed type is "builtins.str" lst = [m, n] reveal_type(lst[0]) # N: Revealed type is "Tuple[builtins.object, builtins.object]" @@ -563,7 +563,7 @@ class A(NamedTuple): y: B n: A -reveal_type(n.y[0]) # N: Revealed type is "Tuple[builtins.str, Tuple[Tuple[builtins.str, ..., fallback=__main__.A], builtins.int, fallback=__main__.B], fallback=__main__.A]" +s: str = n.y[0] # E: Incompatible types in assignment (expression has type "A", variable has type "str") m: B n = m.x @@ -587,4 +587,16 @@ reveal_type(f) # N: Revealed type is "def (m: Tuple[Tuple[builtins.str, ..., fal f(n) # E: Argument 1 to "f" has incompatible type "A"; expected "B" [builtins fixtures/tuple.pyi] +[case testNoRecursiveTuplesAtFunctionScope] +# flags: --enable-recursive-aliases +from typing import NamedTuple, Tuple +def foo() -> None: + class B(NamedTuple): + x: B # E: Cannot resolve name "B" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope + y: int + b: B + reveal_type(b) # N: Revealed type is "Tuple[Any, builtins.int, fallback=__main__.B@4]" +[builtins fixtures/tuple.pyi] + -- TODO: Add more fine-grained tests for various kinds of recursive types From 404d8d10851e7f78a33af4deb43b3f4c2852df00 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 9 Aug 2022 20:24:12 +0100 Subject: [PATCH 08/13] Fix mypyc --- mypy/semanal_shared.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/semanal_shared.py b/mypy/semanal_shared.py index 51f77c1b5d33..2c1d843f4c7a 100644 --- a/mypy/semanal_shared.py +++ b/mypy/semanal_shared.py @@ -84,7 +84,7 @@ def record_incomplete_ref(self) -> None: raise NotImplementedError @abstractmethod - def defer(self, debug_context: Optional[Context] = ..., force_progress: bool = ...) -> None: + def defer(self, debug_context: Optional[Context] = None, force_progress: bool = False) -> None: raise NotImplementedError @abstractmethod From 786f2567d195d81f7e9f238157a984767546149a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 10 Aug 2022 00:52:24 +0100 Subject: [PATCH 09/13] Support recursive TypedDicts --- mypy/fixup.py | 10 ++++- mypy/nodes.py | 44 +++++++++++++++---- mypy/semanal.py | 43 ++++++++++++------ mypy/semanal_namedtuple.py | 2 +- mypy/semanal_typeddict.py | 49 ++++++++++++++++----- mypy/server/astmerge.py | 4 +- mypy/typeanal.py | 7 ++- test-data/unit/check-recursive-types.test | 53 +++++++++++++++++++++++ test-data/unit/fine-grained.test | 29 +++++++++++++ 9 files changed, 203 insertions(+), 38 deletions(-) diff --git a/mypy/fixup.py b/mypy/fixup.py index d4c8e2766114..723eba4f7644 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -338,8 +338,14 @@ def lookup_fully_qualified_alias( if isinstance(node, TypeAlias): return node elif isinstance(node, TypeInfo): - alias = TypeAlias.from_tuple_type(node) - node.tuple_alias = alias + if node.tuple_type: + alias = TypeAlias.from_tuple_type(node) + elif node.typeddict_type: + alias = TypeAlias.from_typeddict_type(node) + else: + assert allow_missing + return missing_alias() + node.special_alias = alias return alias else: # Looks like a missing TypeAlias during an initial daemon load, put something there diff --git a/mypy/nodes.py b/mypy/nodes.py index 5e312b2767de..b7b3a6ef87f3 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2656,7 +2656,7 @@ class is generic then it will be a type constructor of higher kind. "bases", "_promote", "tuple_type", - "tuple_alias", + "special_alias", "is_named_tuple", "typeddict_type", "is_newtype", @@ -2795,8 +2795,16 @@ class is generic then it will be a type constructor of higher kind. # It is useful for plugins to add their data to save in the cache. metadata: Dict[str, JsonDict] - # Store type alias representing this type (for named tuples). - tuple_alias: Optional["TypeAlias"] + # Store type alias representing this type (for named tuples and TypedDicts). + # Although definitions of these types are stored in symbol tables as TypeInfo, + # when a type analyzer will find them, it should construct a TupleType, or + # a TypedDict type. However, we can't use the plain types, since if the definition + # is recursive, this will create an actual recursive structure of types (i.e. as + # internal Python objects) causing infinite recursions everywhere during type checking. + # To overcome this, we create a TypeAlias node, that will point to these types. + # We store this node in the `special_alias` attribute, because it must be the same node + # in case we are doing multiple semantic analysis passes. + special_alias: Optional["TypeAlias"] FLAGS: Final = [ "is_abstract", @@ -2844,7 +2852,7 @@ def __init__(self, names: "SymbolTable", defn: ClassDef, module_name: str) -> No self._promote = [] self.alt_promote = None self.tuple_type = None - self.tuple_alias = None + self.special_alias = None self.is_named_tuple = False self.typeddict_type = None self.is_newtype = False @@ -2976,13 +2984,22 @@ def direct_base_classes(self) -> "List[TypeInfo]": return [base.type for base in self.bases] def update_tuple_type(self, typ: "mypy.types.TupleType") -> None: - """Update tuple_type and tuple_alias as needed.""" + """Update tuple_type and special_alias as needed.""" self.tuple_type = typ alias = TypeAlias.from_tuple_type(self) - if not self.tuple_alias: - self.tuple_alias = alias + if not self.special_alias: + self.special_alias = alias else: - self.tuple_alias.target = alias.target + self.special_alias.target = alias.target + + def update_typeddict_type(self, typ: "mypy.types.TypedDictType") -> None: + """Update typeddict_type and special_alias as needed.""" + self.typeddict_type = typ + alias = TypeAlias.from_typeddict_type(self) + if not self.special_alias: + self.special_alias = alias + else: + self.special_alias.target = alias.target def __str__(self) -> str: """Return a string representation of the type. @@ -3283,6 +3300,17 @@ def from_tuple_type(cls, info: TypeInfo) -> "TypeAlias": info.column, ) + @classmethod + def from_typeddict_type(cls, info: TypeInfo) -> "TypeAlias": + """Generate an alias to the TypedDict type described by a given TypeInfo.""" + assert info.typeddict_type + return TypeAlias( + info.typeddict_type.copy_modified(fallback=mypy.types.Instance(info, [])), + info.fullname, + info.line, + info.column, + ) + @property def name(self) -> str: return self._fullname.split(".")[-1] diff --git a/mypy/semanal.py b/mypy/semanal.py index 1e6388442595..c770e8009f4d 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1378,17 +1378,7 @@ def analyze_class(self, defn: ClassDef) -> None: self.mark_incomplete(defn.name, defn) return - is_typeddict, info = self.typed_dict_analyzer.analyze_typeddict_classdef(defn) - if is_typeddict: - for decorator in defn.decorators: - decorator.accept(self) - if isinstance(decorator, RefExpr): - if decorator.fullname in FINAL_DECORATOR_NAMES: - self.fail("@final cannot be used with TypedDict", decorator) - if info is None: - self.mark_incomplete(defn.name, defn) - else: - self.prepare_class_def(defn, info) + if self.analyze_typeddict_classdef(defn): return if self.analyze_namedtuple_classdef(defn): @@ -1423,6 +1413,28 @@ def analyze_class_body_common(self, defn: ClassDef) -> None: self.apply_class_plugin_hooks(defn) self.leave_class() + def analyze_typeddict_classdef(self, defn: ClassDef) -> bool: + if ( + defn.info + and defn.info.typeddict_type + and not has_placeholder(defn.info.typeddict_type) + ): + # This is a valid TypedDict, and it is fully analyzed. + return True + is_typeddict, info = self.typed_dict_analyzer.analyze_typeddict_classdef(defn) + if is_typeddict: + for decorator in defn.decorators: + decorator.accept(self) + if isinstance(decorator, RefExpr): + if decorator.fullname in FINAL_DECORATOR_NAMES: + self.fail("@final cannot be used with TypedDict", decorator) + if info is None: + self.mark_incomplete(defn.name, defn) + else: + self.prepare_class_def(defn, info) + return True + return False + def analyze_namedtuple_classdef(self, defn: ClassDef) -> bool: """Check if this class can define a named tuple.""" if ( @@ -1841,7 +1853,7 @@ def configure_tuple_base_class(self, defn: ClassDef, base: TupleType) -> Instanc if info.tuple_type and info.tuple_type != base: self.fail("Class has two incompatible bases derived from tuple", defn) defn.has_incompatible_baseclass = True - if info.tuple_alias and has_placeholder(info.tuple_alias.target): + if info.special_alias and has_placeholder(info.special_alias.target): self.defer(force_progress=True) info.update_tuple_type(base) @@ -2661,7 +2673,12 @@ def analyze_namedtuple_assign(self, s: AssignmentStmt) -> bool: def analyze_typeddict_assign(self, s: AssignmentStmt) -> bool: """Check if s defines a typed dict.""" if isinstance(s.rvalue, CallExpr) and isinstance(s.rvalue.analyzed, TypedDictExpr): - return True # This is a valid and analyzed typed dict definition, nothing to do here. + if s.rvalue.analyzed.info.typeddict_type and not has_placeholder( + s.rvalue.analyzed.info.typeddict_type + ): + return ( + True # This is a valid and analyzed typed dict definition, nothing to do here. + ) if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], (NameExpr, MemberExpr)): return False lvalue = s.lvalues[0] diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index d6d622593961..3903c52ab0e7 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -478,7 +478,7 @@ def build_namedtuple_typeinfo( info = existing_info or self.api.basic_new_typeinfo(name, fallback, line) info.is_named_tuple = True tuple_base = TupleType(types, fallback) - if info.tuple_alias and has_placeholder(info.tuple_alias.target): + if info.special_alias and has_placeholder(info.special_alias.target): self.api.defer(force_progress=True) info.update_tuple_type(tuple_base) info.line = line diff --git a/mypy/semanal_typeddict.py b/mypy/semanal_typeddict.py index 603eaabcc2d4..2261df76acb3 100644 --- a/mypy/semanal_typeddict.py +++ b/mypy/semanal_typeddict.py @@ -27,7 +27,7 @@ TypeInfo, ) from mypy.options import Options -from mypy.semanal_shared import SemanticAnalyzerInterface +from mypy.semanal_shared import SemanticAnalyzerInterface, has_placeholder from mypy.typeanal import check_for_explicit_any, has_any_from_unimported_type from mypy.types import TPDICT_NAMES, AnyType, RequiredType, Type, TypedDictType, TypeOfAny @@ -66,6 +66,9 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> Tuple[bool, Optional[Typ if base_expr.fullname in TPDICT_NAMES or self.is_typeddict(base_expr): possible = True if possible: + existing_info = None + if isinstance(defn.analyzed, TypedDictExpr): + existing_info = defn.analyzed.info if ( len(defn.base_type_exprs) == 1 and isinstance(defn.base_type_exprs[0], RefExpr) @@ -76,7 +79,7 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> Tuple[bool, Optional[Typ if fields is None: return True, None # Defer info = self.build_typeddict_typeinfo( - defn.name, fields, types, required_keys, defn.line + defn.name, fields, types, required_keys, defn.line, existing_info ) defn.analyzed = TypedDictExpr(info) defn.analyzed.line = defn.line @@ -128,7 +131,9 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> Tuple[bool, Optional[Typ keys.extend(new_keys) types.extend(new_types) required_keys.update(new_required_keys) - info = self.build_typeddict_typeinfo(defn.name, keys, types, required_keys, defn.line) + info = self.build_typeddict_typeinfo( + defn.name, keys, types, required_keys, defn.line, existing_info + ) defn.analyzed = TypedDictExpr(info) defn.analyzed.line = defn.line defn.analyzed.column = defn.column @@ -173,7 +178,12 @@ def analyze_typeddict_classdef_fields( if stmt.type is None: types.append(AnyType(TypeOfAny.unannotated)) else: - analyzed = self.api.anal_type(stmt.type, allow_required=True) + analyzed = self.api.anal_type( + stmt.type, + allow_required=True, + allow_placeholder=self.options.enable_recursive_aliases + and not self.api.is_func_scope(), + ) if analyzed is None: return None, [], set() # Need to defer types.append(analyzed) @@ -232,7 +242,7 @@ def check_typeddict( name, items, types, total, ok = res if not ok: # Error. Construct dummy return value. - info = self.build_typeddict_typeinfo("TypedDict", [], [], set(), call.line) + info = self.build_typeddict_typeinfo("TypedDict", [], [], set(), call.line, None) else: if var_name is not None and name != var_name: self.fail( @@ -254,7 +264,12 @@ def check_typeddict( types = [ # unwrap Required[T] to just T t.item if isinstance(t, RequiredType) else t for t in types # type: ignore[misc] ] - info = self.build_typeddict_typeinfo(name, items, types, required_keys, call.line) + existing_info = None + if isinstance(node.analyzed, TypedDictExpr): + existing_info = node.analyzed.info + info = self.build_typeddict_typeinfo( + name, items, types, required_keys, call.line, existing_info + ) info.line = node.line # Store generated TypeInfo under both names, see semanal_namedtuple for more details. if name != var_name or is_func_scope: @@ -357,7 +372,12 @@ def parse_typeddict_fields_with_types( else: self.fail_typeddict_arg("Invalid field type", field_type_expr) return [], [], False - analyzed = self.api.anal_type(type, allow_required=True) + analyzed = self.api.anal_type( + type, + allow_required=True, + allow_placeholder=self.options.enable_recursive_aliases + and not self.api.is_func_scope(), + ) if analyzed is None: return None types.append(analyzed) @@ -370,7 +390,13 @@ def fail_typeddict_arg( return "", [], [], True, False def build_typeddict_typeinfo( - self, name: str, items: List[str], types: List[Type], required_keys: Set[str], line: int + self, + name: str, + items: List[str], + types: List[Type], + required_keys: Set[str], + line: int, + existing_info: Optional[TypeInfo], ) -> TypeInfo: # Prefer typing then typing_extensions if available. fallback = ( @@ -379,8 +405,11 @@ def build_typeddict_typeinfo( or self.api.named_type_or_none("mypy_extensions._TypedDict", []) ) assert fallback is not None - info = self.api.basic_new_typeinfo(name, fallback, line) - info.typeddict_type = TypedDictType(dict(zip(items, types)), required_keys, fallback) + info = existing_info or self.api.basic_new_typeinfo(name, fallback, line) + typeddict_type = TypedDictType(dict(zip(items, types)), required_keys, fallback) + if info.special_alias and has_placeholder(info.special_alias.target): + self.api.defer(force_progress=True) + info.update_typeddict_type(typeddict_type) return info # Helpers diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index 0ced8bf8ee54..52f67ad900e0 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -364,8 +364,8 @@ def process_type_info(self, info: Optional[TypeInfo]) -> None: self.fixup_type(target) self.fixup_type(info.tuple_type) self.fixup_type(info.typeddict_type) - if info.tuple_alias: - self.fixup_type(info.tuple_alias.target) + if info.special_alias: + self.fixup_type(info.special_alias.target) info.defn.info = self.fixup(info) replace_nodes_in_symbol_table(info.names, self.replacements) for i, item in enumerate(info.mro): diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 39c2cfb7f616..d797c8306515 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -613,9 +613,9 @@ def analyze_type_with_type_info( if args: self.fail("Generic tuple types not supported", ctx) return AnyType(TypeOfAny.from_error) - if info.tuple_alias: + if info.special_alias: # We don't support generic tuple types yet. - return TypeAliasType(info.tuple_alias, []) + return TypeAliasType(info.special_alias, []) return tup.copy_modified(items=self.anal_array(tup.items), fallback=instance) td = info.typeddict_type if td is not None: @@ -624,6 +624,9 @@ def analyze_type_with_type_info( if args: self.fail("Generic TypedDict types not supported", ctx) return AnyType(TypeOfAny.from_error) + if info.special_alias: + # We don't support generic TypedDict types yet. + return TypeAliasType(info.special_alias, []) # Create a named TypedDictType return td.copy_modified( item_types=self.anal_array(list(td.items.values())), fallback=instance diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index a1ce22461fd2..e1db231dd35f 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -611,4 +611,57 @@ def foo() -> None: reveal_type(b) # N: Revealed type is "Tuple[Any, builtins.int, fallback=__main__.B@4]" [builtins fixtures/tuple.pyi] +[case testBasicRecursiveTypedDictClass] +# flags: --enable-recursive-aliases +from typing import TypedDict + +class TD(TypedDict): + x: int + y: TD + +td: TD +reveal_type(td) # N: Revealed type is "TypedDict('__main__.TD', {'x': builtins.int, 'y': ...})" +s: str = td["y"] # E: Incompatible types in assignment (expression has type "TD", variable has type "str") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testBasicRecursiveTypedDictCall] +# flags: --enable-recursive-aliases +from typing import TypedDict + +TD = TypedDict("TD", {"x": int, "y": TD}) +td: TD +reveal_type(td) # N: Revealed type is "TypedDict('__main__.TD', {'x': builtins.int, 'y': ...})" + +TD2 = TypedDict("TD2", {"x": int, "y": TD2}) +td2: TD2 +TD3 = TypedDict("TD3", {"x": str, "y": TD3}) +td3: TD3 + +td = td2 +td = td3 # E: Incompatible types in assignment (expression has type "TD3", variable has type "TD") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testBasicRecursiveTypedDictExtending] +# flags: --enable-recursive-aliases +from typing import TypedDict + +class TDA(TypedDict): + xa: int + ya: TD + +class TDB(TypedDict): + xb: int + yb: TD + +class TD(TDA, TDB): + a: TDA + b: TDB + +td: TD +reveal_type(td) # N: Revealed type is "TypedDict('__main__.TD', {'xb': builtins.int, 'yb': ..., 'xa': builtins.int, 'ya': ..., 'a': TypedDict('__main__.TDA', {'xa': builtins.int, 'ya': ...}), 'b': TypedDict('__main__.TDB', {'xb': builtins.int, 'yb': ...})})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + -- TODO: Add more fine-grained tests for various kinds of recursive types diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 27bd857b70a7..27f627681245 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -3528,6 +3528,35 @@ def foo(x: Point) -> int: == b.py:3: error: Unsupported operand types for + ("int" and "str") +[case testTypedDictUpdate3] +# flags: --enable-recursive-aliases +import b +[file a.py] +from mypy_extensions import TypedDict +from typing import Optional +class Point(TypedDict): + x: Optional[Point] + y: int + z: int +p = Point(dict(x=None, y=1337, z=0)) +[file a.py.2] +from mypy_extensions import TypedDict +from typing import Optional +class Point(TypedDict): + x: Optional[Point] + y: str + z: int +p = Point(dict(x=None, y='lurr', z=0)) +[file b.py] +from a import Point +def foo(x: Point) -> int: + assert x['x'] is not None + return x['x']['z'] + x['x']['y'] +[builtins fixtures/dict.pyi] +[out] +== +b.py:4: error: Unsupported operand types for + ("int" and "str") + [case testBasicAliasUpdate] import b [file a.py] From 2dfd5c2e6277f615c4448dbd4dbd9929bbccf33b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 10 Aug 2022 01:52:24 +0100 Subject: [PATCH 10/13] Fix accidentaly uncovered NewType bug --- mypy/semanal_newtype.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mypy/semanal_newtype.py b/mypy/semanal_newtype.py index 6fe6cd4a4295..e7ab5cc05e4d 100644 --- a/mypy/semanal_newtype.py +++ b/mypy/semanal_newtype.py @@ -81,6 +81,8 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> bool: old_type = get_proper_type(old_type) if not call.analyzed: call.analyzed = NewTypeExpr(var_name, old_type, line=call.line, column=call.column) + else: + call.analyzed.old_type = old_type if old_type is None: if should_defer: # Base type is not ready. @@ -230,6 +232,7 @@ def build_newtype_typeinfo( existing_info: Optional[TypeInfo], ) -> TypeInfo: info = existing_info or self.api.basic_new_typeinfo(name, base_type, line) + info.bases = [base_type] # Update in case there were nested placeholders. info.is_newtype = True # Add __init__ method @@ -250,7 +253,7 @@ def build_newtype_typeinfo( init_func._fullname = info.fullname + ".__init__" info.names["__init__"] = SymbolTableNode(MDEF, init_func) - if info.tuple_type and has_placeholder(info.tuple_type): + if has_placeholder(old_type) or info.tuple_type and has_placeholder(info.tuple_type): self.api.defer(force_progress=True) return info From f377d6b0e7b2cb4ff1e878332e816ebd9bd0e956 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 10 Aug 2022 01:57:50 +0100 Subject: [PATCH 11/13] Add a comment --- mypy/semanal.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mypy/semanal.py b/mypy/semanal.py index c770e8009f4d..d577f5249b2d 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -5522,6 +5522,11 @@ def defer(self, debug_context: Optional[Context] = None, force_progress: bool = """ assert not self.final_iteration, "Must not defer during final iteration" if force_progress: + # Usually, we report progress if we have replaced a placeholder node + # with an actual valid node. However, sometimes we need to update an + # existing node *in-place*. For example, this is used by type aliases + # in context of forward references and/or recursive aliases, and in + # similar situations (recursive named tuples etc). self.progress = True self.deferred = True # Store debug info for this deferral. From a5e06dce72759d34b1dea6efc5ca2227311e8240 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 10 Aug 2022 17:01:44 +0100 Subject: [PATCH 12/13] Fix merge --- mypy/fixup.py | 1 + mypy/server/astmerge.py | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/mypy/fixup.py b/mypy/fixup.py index 0e2ed3a5f62e..37e651fe05ff 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -349,6 +349,7 @@ def lookup_fully_qualified_alias( else: assert allow_missing return missing_alias() + node.special_alias = alias return alias else: # Looks like a missing TypeAlias during an initial daemon load, put something there diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index 7e0fefac7c5b..3bf3f23f2988 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -172,8 +172,8 @@ def replacement_map_from_symbol_table( node.node.names, new_node.node.names, prefix ) replacements.update(type_repl) - if node.node.tuple_alias and new_node.node.tuple_alias: - replacements[new_node.node.tuple_alias] = node.node.tuple_alias + if node.node.special_alias and new_node.node.special_alias: + replacements[new_node.node.special_alias] = node.node.special_alias return replacements @@ -338,10 +338,10 @@ def fixup(self, node: SN) -> SN: new = self.replacements[node] skip_slots: Tuple[str, ...] = () if isinstance(node, TypeInfo) and isinstance(new, TypeInfo): - # Special case: tuple_alias is not exposed in symbol tables, but may appear + # Special case: special_alias is not exposed in symbol tables, but may appear # in external types (e.g. named tuples), so we need to update it manually. - skip_slots = ("tuple_alias",) - replace_object_state(new.tuple_alias, node.tuple_alias) + skip_slots = ("special_alias",) + replace_object_state(new.special_alias, node.special_alias) replace_object_state(new, node, skip_slots=skip_slots) return cast(SN, new) return node @@ -547,7 +547,7 @@ def replace_nodes_in_symbol_table( new = replacements[node.node] old = node.node # Needed for TypeInfo, see comment in fixup() above. - replace_object_state(new, old, skip_slots=("tuple_alias",)) + replace_object_state(new, old, skip_slots=("special_alias",)) node.node = new if isinstance(node.node, (Var, TypeAlias)): # Handle them here just in case these aren't exposed through the AST. From bb94c80ad19d50eaaf28e6e4b0eccf084d5f6371 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 10 Aug 2022 18:26:18 +0100 Subject: [PATCH 13/13] Fix self-check; formatting; more tests --- mypy/semanal.py | 5 +- mypy/semanal_newtype.py | 2 +- test-data/unit/check-incremental.test | 44 +++++++++++ test-data/unit/check-recursive-types.test | 89 +++++++++++++++++++++++ 4 files changed, 136 insertions(+), 4 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 3a441df2c780..2a30783d5bdc 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -2675,9 +2675,8 @@ def analyze_typeddict_assign(self, s: AssignmentStmt) -> bool: if s.rvalue.analyzed.info.typeddict_type and not has_placeholder( s.rvalue.analyzed.info.typeddict_type ): - return ( - True # This is a valid and analyzed typed dict definition, nothing to do here. - ) + # This is a valid and analyzed typed dict definition, nothing to do here. + return True if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], (NameExpr, MemberExpr)): return False lvalue = s.lvalues[0] diff --git a/mypy/semanal_newtype.py b/mypy/semanal_newtype.py index e7ab5cc05e4d..c70329816421 100644 --- a/mypy/semanal_newtype.py +++ b/mypy/semanal_newtype.py @@ -79,7 +79,7 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> bool: old_type, should_defer = self.check_newtype_args(var_name, call, s) old_type = get_proper_type(old_type) - if not call.analyzed: + if not isinstance(call.analyzed, NewTypeExpr): call.analyzed = NewTypeExpr(var_name, old_type, line=call.line, column=call.column) else: call.analyzed.old_type = old_type diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 02e705cfbbac..0cf048bee959 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -5849,3 +5849,47 @@ def f(x: a.N) -> None: [out3] tmp/c.py:4: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int], None], builtins.int]" tmp/c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") + +[case testTypedDictUpdateNonRecursiveToRecursiveCoarse] +# flags: --enable-recursive-aliases +import c +[file a.py] +from b import M +from typing import TypedDict, Optional +class N(TypedDict): + r: Optional[M] + x: int +n: N +[file b.py] +from a import N +from typing import TypedDict +class M(TypedDict): + r: None + x: int +[file b.py.2] +from a import N +from typing import TypedDict, Optional +class M(TypedDict): + r: Optional[N] + x: int +[file c.py] +import a +def f(x: a.N) -> None: + if x["r"] is not None: + s: int = x["r"]["x"] +[file c.py.3] +import a +def f(x: a.N) -> None: + if x["r"] is not None and x["r"]["r"] is not None and x["r"]["r"]["r"] is not None: + reveal_type(x) + s: int = x["r"]["r"]["r"]["r"] +f(a.n) +reveal_type(a.n) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] +[out] +[out2] +[out3] +tmp/c.py:4: note: Revealed type is "TypedDict('a.N', {'r': Union[TypedDict('b.M', {'r': Union[..., None], 'x': builtins.int}), None], 'x': builtins.int})" +tmp/c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") +tmp/c.py:7: note: Revealed type is "TypedDict('a.N', {'r': Union[TypedDict('b.M', {'r': Union[..., None], 'x': builtins.int}), None], 'x': builtins.int})" diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index 343015b10b50..b5a1fe6838b5 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -663,3 +663,92 @@ td: TD reveal_type(td) # N: Revealed type is "TypedDict('__main__.TD', {'xb': builtins.int, 'yb': ..., 'xa': builtins.int, 'ya': ..., 'a': TypedDict('__main__.TDA', {'xa': builtins.int, 'ya': ...}), 'b': TypedDict('__main__.TDB', {'xb': builtins.int, 'yb': ...})})" [builtins fixtures/dict.pyi] [typing fixtures/typing-typeddict.pyi] + +[case testRecursiveTypedDictCreation] +# flags: --enable-recursive-aliases +from typing import TypedDict, Optional + +class TD(TypedDict): + x: int + y: Optional[TD] + +td: TD = {"x": 0, "y": None} +td2: TD = {"x": 0, "y": {"x": 1, "y": {"x": 2, "y": None}}} + +itd = TD(x=0, y=None) +itd2 = TD(x=0, y=TD(x=0, y=TD(x=0, y=None))) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testRecursiveTypedDictMethods] +# flags: --enable-recursive-aliases +from typing import TypedDict + +class TD(TypedDict, total=False): + x: int + y: TD + +td: TD +td["y"] = {"x": 0, "y": {}} +td["y"] = {"x": 0, "y": {"x": 0, "y": 42}} # E: Incompatible types (expression has type "int", TypedDict item "y" has type "TD") + +reveal_type(td.get("y")) # N: Revealed type is "Union[TypedDict('__main__.TD', {'x'?: builtins.int, 'y'?: TypedDict('__main__.TD', {'x'?: builtins.int, 'y'?: ...})}), None]" +s: str = td.get("y") # E: Incompatible types in assignment (expression has type "Optional[TD]", variable has type "str") + +td.update({"x": 0, "y": {"x": 1, "y": {}}}) +td.update({"x": 0, "y": {"x": 1, "y": {"x": 2, "y": 42}}}) # E: Incompatible types (expression has type "int", TypedDict item "y" has type "TD") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testRecursiveTypedDictSubtyping] +# flags: --enable-recursive-aliases +from typing import TypedDict + +class TDA1(TypedDict): + x: int + y: TDA1 +class TDA2(TypedDict): + x: int + y: TDA2 +class TDB(TypedDict): + x: str + y: TDB + +tda1: TDA1 +tda2: TDA2 +tdb: TDB +def fa1(arg: TDA1) -> None: ... +def fa2(arg: TDA2) -> None: ... +def fb(arg: TDB) -> None: ... + +fa1(tda2) +fa2(tda1) +fb(tda1) # E: Argument 1 to "fb" has incompatible type "TDA1"; expected "TDB" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testRecursiveTypedDictJoin] +# flags: --enable-recursive-aliases +from typing import TypedDict, TypeVar + +class TDA1(TypedDict): + x: int + y: TDA1 +class TDA2(TypedDict): + x: int + y: TDA2 +class TDB(TypedDict): + x: str + y: TDB + +tda1: TDA1 +tda2: TDA2 +tdb: TDB + +T = TypeVar("T") +def f(x: T, y: T) -> T: ... +# Join for recursive types is very basic, but just add tests that we don't crash. +reveal_type(f(tda1, tda2)) # N: Revealed type is "TypedDict({'x': builtins.int, 'y': TypedDict('__main__.TDA1', {'x': builtins.int, 'y': ...})})" +reveal_type(f(tda1, tdb)) # N: Revealed type is "TypedDict({})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi]