From f0163c433c546d4e406539e8c9889c5a6917de2f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 9 Aug 2022 12:08:48 +0100 Subject: [PATCH 01/11] Try another approach --- mypy/fixup.py | 6 +++ mypy/messages.py | 5 ++ mypy/nodes.py | 5 ++ mypy/semanal.py | 49 ++++++++---------- mypy/semanal_namedtuple.py | 49 +++++++++++++++--- mypy/semanal_shared.py | 18 ++++++- mypy/server/astmerge.py | 2 + mypy/typeanal.py | 2 + test-data/unit/check-recursive-types.test | 61 ++++++++++++++++++++++- 9 files changed, 161 insertions(+), 36 deletions(-) diff --git a/mypy/fixup.py b/mypy/fixup.py index 08a17e541d44..56ca8c3f4263 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -337,6 +337,12 @@ def lookup_fully_qualified_alias( node = stnode.node if stnode else None if isinstance(node, TypeAlias): return node + elif isinstance(node, TypeInfo): + assert node.tuple_type + target = node.tuple_type.copy_modified(fallback=Instance(node, [])) + alias = TypeAlias(target, node.fullname, node.line, node.column) + node.tuple_alias = alias + return alias else: # Looks like a missing TypeAlias during an initial daemon load, put something there assert ( diff --git a/mypy/messages.py b/mypy/messages.py index d27dad0df5b7..8214455a12c6 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -2292,6 +2292,11 @@ def visit_instance(self, t: Instance) -> None: self.instances.append(t) super().visit_instance(t) + def visit_type_alias_type(self, t: TypeAliasType) -> None: + if t.alias and not t.is_recursive: + t.alias.target.accept(self) + super().visit_type_alias_type(t) + def find_type_overlaps(*types: Type) -> Set[str]: """Return a set of fullnames that share a short name and appear in either type. diff --git a/mypy/nodes.py b/mypy/nodes.py index 606a2073219f..b49412e45be9 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2651,6 +2651,7 @@ class is generic then it will be a type constructor of higher kind. "bases", "_promote", "tuple_type", + "tuple_alias", "is_named_tuple", "typeddict_type", "is_newtype", @@ -2789,6 +2790,9 @@ class is generic then it will be a type constructor of higher kind. # It is useful for plugins to add their data to save in the cache. metadata: Dict[str, JsonDict] + # Store type alias representing this type (for named tuples). + tuple_alias: Optional["TypeAlias"] + FLAGS: Final = [ "is_abstract", "is_enum", @@ -2835,6 +2839,7 @@ def __init__(self, names: "SymbolTable", defn: ClassDef, module_name: str) -> No self._promote = [] self.alt_promote = None self.tuple_type = None + self.tuple_alias = None self.is_named_tuple = False self.typeddict_type = None self.is_newtype = False diff --git a/mypy/semanal.py b/mypy/semanal.py index 3ac0af8ba11e..04c8e51304fd 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -221,11 +221,11 @@ PRIORITY_FALLBACKS, SemanticAnalyzerInterface, calculate_tuple_fallback, + has_placeholder, set_callable_name as set_callable_name, ) from mypy.semanal_typeddict import TypedDictAnalyzer from mypy.tvar_scope import TypeVarLikeScope -from mypy.type_visitor import TypeQuery from mypy.typeanal import ( TypeAnalyser, TypeVarLikeList, @@ -1424,7 +1424,12 @@ def analyze_class_body_common(self, defn: ClassDef) -> None: def analyze_namedtuple_classdef(self, defn: ClassDef) -> bool: """Check if this class can define a named tuple.""" - if defn.info and defn.info.is_named_tuple: + if ( + defn.info + and defn.info.is_named_tuple + and defn.info.tuple_type + and not has_placeholder(defn.info.tuple_type) + ): # Don't reprocess everything. We just need to process methods defined # in the named tuple class body. is_named_tuple, info = True, defn.info # type: bool, Optional[TypeInfo] @@ -1436,6 +1441,8 @@ def analyze_namedtuple_classdef(self, defn: ClassDef) -> bool: if info is None: self.mark_incomplete(defn.name, defn) else: + if info.tuple_type and has_placeholder(info.tuple_type): + self.defer(force_progress=True) self.prepare_class_def(defn, info) with self.scope.class_scope(defn.info): with self.named_tuple_analyzer.save_namedtuple_body(info): @@ -1784,7 +1791,7 @@ def configure_base_classes( info.tuple_type = None for base, base_expr in bases: if isinstance(base, TupleType): - actual_base = self.configure_tuple_base_class(defn, base, base_expr) + actual_base = self.configure_tuple_base_class(defn, base) base_types.append(actual_base) elif isinstance(base, Instance): if base.type.is_newtype: @@ -1827,9 +1834,7 @@ def configure_base_classes( return self.calculate_class_mro(defn, self.object_type) - def configure_tuple_base_class( - self, defn: ClassDef, base: TupleType, base_expr: Expression - ) -> Instance: + def configure_tuple_base_class(self, defn: ClassDef, base: TupleType) -> Instance: info = defn.info # There may be an existing valid tuple type from previous semanal iterations. @@ -1838,10 +1843,6 @@ def configure_tuple_base_class( self.fail("Class has two incompatible bases derived from tuple", defn) defn.has_incompatible_baseclass = True info.tuple_type = base - if isinstance(base_expr, CallExpr): - defn.analyzed = NamedTupleExpr(base.partial_fallback.type) - defn.analyzed.line = defn.line - defn.analyzed.column = defn.column if base.partial_fallback.type.fullname == "builtins.tuple": # Fallback can only be safely calculated after semantic analysis, since base @@ -2626,7 +2627,10 @@ def analyze_enum_assign(self, s: AssignmentStmt) -> bool: def analyze_namedtuple_assign(self, s: AssignmentStmt) -> bool: """Check if s defines a namedtuple.""" if isinstance(s.rvalue, CallExpr) and isinstance(s.rvalue.analyzed, NamedTupleExpr): - return True # This is a valid and analyzed named tuple definition, nothing to do here. + if s.rvalue.analyzed.info.tuple_type and not has_placeholder( + s.rvalue.analyzed.info.tuple_type + ): + return True # This is a valid and analyzed named tuple definition, nothing to do here. if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], (NameExpr, MemberExpr)): return False lvalue = s.lvalues[0] @@ -3025,6 +3029,9 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool: # unless using PEP 613 `cls: TypeAlias = A` return False + if isinstance(s.rvalue, CallExpr) and s.rvalue.analyzed: + return False + existing = self.current_symbol_table().get(lvalue.name) # Third rule: type aliases can't be re-defined. For example: # A: Type[float] = int @@ -3157,9 +3164,8 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool: self.cannot_resolve_name(lvalue.name, "name", s) return True else: - self.progress = True # We need to defer so that this change can get propagated to base classes. - self.defer(s) + self.defer(s, force_progress=True) else: self.add_symbol(lvalue.name, alias_node, s) if isinstance(rvalue, RefExpr) and isinstance(rvalue.node, TypeAlias): @@ -5464,7 +5470,7 @@ def tvar_scope_frame(self, frame: TypeVarLikeScope) -> Iterator[None]: yield self.tvar_scope = old_scope - def defer(self, debug_context: Optional[Context] = None) -> None: + def defer(self, debug_context: Optional[Context] = None, force_progress: bool = False) -> None: """Defer current analysis target to be analyzed again. This must be called if something in the current target is @@ -5478,6 +5484,8 @@ def defer(self, debug_context: Optional[Context] = None) -> None: They are usually preferable to a direct defer() call. """ assert not self.final_iteration, "Must not defer during final iteration" + if force_progress: + self.progress = True self.deferred = True # Store debug info for this deferral. line = ( @@ -5979,19 +5987,6 @@ def is_future_flag_set(self, flag: str) -> bool: return self.modules[self.cur_mod_id].is_future_flag_set(flag) -class HasPlaceholders(TypeQuery[bool]): - def __init__(self) -> None: - super().__init__(any) - - def visit_placeholder_type(self, t: PlaceholderType) -> bool: - return True - - -def has_placeholder(typ: Type) -> bool: - """Check if a type contains any placeholder types (recursively).""" - return typ.accept(HasPlaceholders()) - - def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: if isinstance(sig, CallableType): if len(sig.arg_types) == 0: diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index 55e38cdfa11d..f2b49ede330c 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -35,6 +35,7 @@ SymbolTableNode, TempNode, TupleExpr, + TypeAlias, TypeInfo, TypeVarExpr, Var, @@ -44,12 +45,14 @@ PRIORITY_FALLBACKS, SemanticAnalyzerInterface, calculate_tuple_fallback, + has_placeholder, set_callable_name, ) from mypy.types import ( TYPED_NAMEDTUPLE_NAMES, AnyType, CallableType, + Instance, LiteralType, TupleType, Type, @@ -109,8 +112,11 @@ def analyze_namedtuple_classdef( items, types, default_items = result if is_func_scope and "@" not in defn.name: defn.name += "@" + str(defn.line) + existing_info = None + if isinstance(defn.analyzed, NamedTupleExpr): + existing_info = defn.analyzed.info info = self.build_namedtuple_typeinfo( - defn.name, items, types, default_items, defn.line + defn.name, items, types, default_items, defn.line, existing_info ) defn.info = info defn.analyzed = NamedTupleExpr(info, is_typed=True) @@ -164,7 +170,9 @@ def check_namedtuple_classdef( if stmt.type is None: types.append(AnyType(TypeOfAny.unannotated)) else: - analyzed = self.api.anal_type(stmt.type) + analyzed = self.api.anal_type( + stmt.type, allow_placeholder=self.options.enable_recursive_aliases + ) if analyzed is None: # Something is incomplete. We need to defer this named tuple. return None @@ -226,7 +234,7 @@ def check_namedtuple( name += "@" + str(call.line) else: name = var_name = "namedtuple@" + str(call.line) - info = self.build_namedtuple_typeinfo(name, [], [], {}, node.line) + info = self.build_namedtuple_typeinfo(name, [], [], {}, node.line, None) self.store_namedtuple_info(info, var_name, call, is_typed) if name != var_name or is_func_scope: # NOTE: we skip local namespaces since they are not serialized. @@ -262,12 +270,28 @@ def check_namedtuple( } else: default_items = {} - info = self.build_namedtuple_typeinfo(name, items, types, default_items, node.line) + + existing_info = None + if isinstance(node.analyzed, NamedTupleExpr): + existing_info = node.analyzed.info + info = self.build_namedtuple_typeinfo( + name, items, types, default_items, node.line, existing_info + ) + + if isinstance(node.analyzed, NamedTupleExpr) and node.analyzed.info.tuple_type: + if has_placeholder(node.analyzed.info.tuple_type): + self.api.defer(force_progress=True) + # No need to store in symbol tables. + return typename, info + # If var_name is not None (i.e. this is not a base class expression), we always # store the generated TypeInfo under var_name in the current scope, so that # other definitions can use it. if var_name: self.store_namedtuple_info(info, var_name, call, is_typed) + else: + call.analyzed = NamedTupleExpr(info, is_typed=is_typed) + call.analyzed.set_line(call) # There are three cases where we need to store the generated TypeInfo # second time (for the purpose of serialization): # * If there is a name mismatch like One = NamedTuple('Other', [...]) @@ -408,7 +432,9 @@ def parse_namedtuple_fields_with_types( except TypeTranslationError: self.fail("Invalid field type", type_node) return None - analyzed = self.api.anal_type(type) + analyzed = self.api.anal_type( + type, allow_placeholder=self.options.enable_recursive_aliases + ) # Workaround #4987 and avoid introducing a bogus UnboundType if isinstance(analyzed, UnboundType): analyzed = AnyType(TypeOfAny.from_error) @@ -428,6 +454,7 @@ def build_namedtuple_typeinfo( types: List[Type], default_items: Mapping[str, Expression], line: int, + existing_info: Optional[TypeInfo], ) -> TypeInfo: strtype = self.api.named_type("builtins.str") implicit_any = AnyType(TypeOfAny.special_form) @@ -448,10 +475,15 @@ def build_namedtuple_typeinfo( literals: List[Type] = [LiteralType(item, strtype) for item in items] match_args_type = TupleType(literals, basetuple_type) - info = self.api.basic_new_typeinfo(name, fallback, line) + info = existing_info or self.api.basic_new_typeinfo(name, fallback, line) info.is_named_tuple = True tuple_base = TupleType(types, fallback) info.tuple_type = tuple_base + target = tuple_base.copy_modified(fallback=Instance(info, [])) + if not info.tuple_alias: + info.tuple_alias = TypeAlias(target, info.fullname, info.line, info.column) + else: + info.tuple_alias.target = target info.line = line # For use by mypyc. info.metadata["namedtuple"] = {"fields": items.copy()} @@ -459,7 +491,10 @@ def build_namedtuple_typeinfo( # We can't calculate the complete fallback type until after semantic # analysis, since otherwise base classes might be incomplete. Postpone a # callback function that patches the fallback. - self.api.schedule_patch(PRIORITY_FALLBACKS, lambda: calculate_tuple_fallback(tuple_base)) + if not has_placeholder(tuple_base): + self.api.schedule_patch( + PRIORITY_FALLBACKS, lambda: calculate_tuple_fallback(tuple_base) + ) def add_field( var: Var, is_initialized_in_class: bool = False, is_property: bool = False diff --git a/mypy/semanal_shared.py b/mypy/semanal_shared.py index fd7bc363b077..873ee2788d58 100644 --- a/mypy/semanal_shared.py +++ b/mypy/semanal_shared.py @@ -19,6 +19,7 @@ TypeInfo, ) from mypy.tvar_scope import TypeVarLikeScope +from mypy.type_visitor import TypeQuery from mypy.types import ( TPDICT_FB_NAMES, FunctionLike, @@ -26,6 +27,7 @@ Parameters, ParamSpecFlavor, ParamSpecType, + PlaceholderType, ProperType, TupleType, Type, @@ -82,7 +84,7 @@ def record_incomplete_ref(self) -> None: raise NotImplementedError @abstractmethod - def defer(self) -> None: + def defer(self, debug_context: Optional[Context] = ..., force_progress: bool = ...) -> None: raise NotImplementedError @abstractmethod @@ -147,6 +149,7 @@ def anal_type( allow_tuple_literal: bool = False, allow_unbound_tvars: bool = False, allow_required: bool = False, + allow_placeholder: bool = False, report_invalid_types: bool = True, ) -> Optional[Type]: raise NotImplementedError @@ -297,3 +300,16 @@ def paramspec_kwargs( column=column, prefix=prefix, ) + + +class HasPlaceholders(TypeQuery[bool]): + def __init__(self) -> None: + super().__init__(any) + + def visit_placeholder_type(self, t: PlaceholderType) -> bool: + return True + + +def has_placeholder(typ: Type) -> bool: + """Check if a type contains any placeholder types (recursively).""" + return typ.accept(HasPlaceholders()) diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index d90061b60cf7..0ced8bf8ee54 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -364,6 +364,8 @@ def process_type_info(self, info: Optional[TypeInfo]) -> None: self.fixup_type(target) self.fixup_type(info.tuple_type) self.fixup_type(info.typeddict_type) + if info.tuple_alias: + self.fixup_type(info.tuple_alias.target) info.defn.info = self.fixup(info) replace_nodes_in_symbol_table(info.names, self.replacements) for i, item in enumerate(info.mro): diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 552990a8482b..4e4904331a58 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -610,6 +610,8 @@ def analyze_type_with_type_info( if args: self.fail("Generic tuple types not supported", ctx) return AnyType(TypeOfAny.from_error) + if info.tuple_alias: + return TypeAliasType(info.tuple_alias, []) return tup.copy_modified(items=self.anal_array(tup.items), fallback=instance) td = info.typeddict_type if td is not None: diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index e8b223d08fd9..1ef14e476cc9 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -90,7 +90,7 @@ else: A = int | list[A] [builtins fixtures/isinstancelist.pyi] --- Tests duplicating some existing tests with recursive aliases enabled +-- Tests duplicating some type alias existing tests with recursive aliases enabled [case testRecursiveAliasesMutual] # flags: --enable-recursive-aliases @@ -426,3 +426,62 @@ U = Type[Union[int, U]] # E: Type[...] cannot contain another Type[...] x: U reveal_type(x) # N: Revealed type is "Type[Any]" [builtins fixtures/isinstancelist.pyi] + +[case testBasicNamedTuple] +# flags: --enable-recursive-aliases +from typing import NamedTuple, Optional + +NT = NamedTuple("NT", [("x", Optional[NT]), ("y", int)]) +nt: NT +reveal_type(nt) # N: Revealed type is "Tuple[Union[..., None], builtins.int, fallback=__main__.NT]" +reveal_type(nt.x) # N: Revealed type is "Union[Tuple[Union[..., None], builtins.int, fallback=__main__.NT], None]" +reveal_type(nt[0]) # N: Revealed type is "Union[Tuple[Union[..., None], builtins.int, fallback=__main__.NT], None]" +y: str +if nt.x is not None: + y = nt.x[0] # E: Incompatible types in assignment (expression has type "Optional[NT]", variable has type "str") +[builtins fixtures/tuple.pyi] + +[case testBasicNamedTupleSpecial] +# flags: --enable-recursive-aliases +from typing import NamedTuple + +NT = NamedTuple("NT", [("x", NT), ("y", int)]) +nt: NT +reveal_type(nt) # N: Revealed type is "Tuple[..., builtins.int, fallback=__main__.NT]" +reveal_type(nt.x) # N: Revealed type is "Tuple[Tuple[..., builtins.int, fallback=__main__.NT], builtins.int, fallback=__main__.NT]" +reveal_type(nt[0]) # N: Revealed type is "Tuple[Tuple[..., builtins.int, fallback=__main__.NT], builtins.int, fallback=__main__.NT]" +y: str +if nt.x is not None: + y = nt.x[0] # E: Incompatible types in assignment (expression has type "NT", variable has type "str") +# XXX check join no infinite recursion on fallbacks +[builtins fixtures/tuple.pyi] + +[case testBasicNamedTupleClass] +# flags: --enable-recursive-aliases +from typing import NamedTuple, Optional + +class NT(NamedTuple): + x: Optional[NT] + y: int + +nt: NT +reveal_type(nt) # N: Revealed type is "Tuple[Union[..., None], builtins.int, fallback=__main__.NT]" +reveal_type(nt.x) # N: Revealed type is "Union[Tuple[Union[..., None], builtins.int, fallback=__main__.NT], None]" +reveal_type(nt[0]) # N: Revealed type is "Union[Tuple[Union[..., None], builtins.int, fallback=__main__.NT], None]" +y: str +if nt.x is not None: + y = nt.x[0] # E: Incompatible types in assignment (expression has type "Optional[NT]", variable has type "str") +[builtins fixtures/tuple.pyi] + +[case testBasicRegularTupleClass] +# flags: --enable-recursive-aliases +[builtins fixtures/tuple.pyi] + +[case testBasicTupleClassesNewType] +# flags: --enable-recursive-aliases +# check both regular and named +[builtins fixtures/tuple.pyi] + +-- Add fine-grained tests + +-- Tests duplicating some named tuple existing tests with recursive aliases enabled From dde4b92038f410d244bb55f58da5ac5276f92a52 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 9 Aug 2022 13:45:46 +0100 Subject: [PATCH 02/11] Fix/simplify things --- mypy/semanal.py | 2 -- mypy/semanal_namedtuple.py | 20 ++++++++++++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 04c8e51304fd..2392ec1e3c05 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1441,8 +1441,6 @@ def analyze_namedtuple_classdef(self, defn: ClassDef) -> bool: if info is None: self.mark_incomplete(defn.name, defn) else: - if info.tuple_type and has_placeholder(info.tuple_type): - self.defer(force_progress=True) self.prepare_class_def(defn, info) with self.scope.class_scope(defn.info): with self.named_tuple_analyzer.save_namedtuple_body(info): diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index f2b49ede330c..557e342d1177 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -171,7 +171,9 @@ def check_namedtuple_classdef( types.append(AnyType(TypeOfAny.unannotated)) else: analyzed = self.api.anal_type( - stmt.type, allow_placeholder=self.options.enable_recursive_aliases + stmt.type, + allow_placeholder=self.options.enable_recursive_aliases + and not self.api.is_func_scope(), ) if analyzed is None: # Something is incomplete. We need to defer this named tuple. @@ -278,12 +280,6 @@ def check_namedtuple( name, items, types, default_items, node.line, existing_info ) - if isinstance(node.analyzed, NamedTupleExpr) and node.analyzed.info.tuple_type: - if has_placeholder(node.analyzed.info.tuple_type): - self.api.defer(force_progress=True) - # No need to store in symbol tables. - return typename, info - # If var_name is not None (i.e. this is not a base class expression), we always # store the generated TypeInfo under var_name in the current scope, so that # other definitions can use it. @@ -433,7 +429,9 @@ def parse_namedtuple_fields_with_types( self.fail("Invalid field type", type_node) return None analyzed = self.api.anal_type( - type, allow_placeholder=self.options.enable_recursive_aliases + type, + allow_placeholder=self.options.enable_recursive_aliases + and not self.api.is_func_scope(), ) # Workaround #4987 and avoid introducing a bogus UnboundType if isinstance(analyzed, UnboundType): @@ -478,6 +476,9 @@ def build_namedtuple_typeinfo( info = existing_info or self.api.basic_new_typeinfo(name, fallback, line) info.is_named_tuple = True tuple_base = TupleType(types, fallback) + old_tuple_type = None + if existing_info: + old_tuple_type = existing_info.tuple_type info.tuple_type = tuple_base target = tuple_base.copy_modified(fallback=Instance(info, [])) if not info.tuple_alias: @@ -488,6 +489,9 @@ def build_namedtuple_typeinfo( # For use by mypyc. info.metadata["namedtuple"] = {"fields": items.copy()} + if old_tuple_type and has_placeholder(old_tuple_type): + self.api.defer(force_progress=True) + # We can't calculate the complete fallback type until after semantic # analysis, since otherwise base classes might be incomplete. Postpone a # callback function that patches the fallback. From 50279bacacde4bda41733422d981b1ee9c7f7f40 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 9 Aug 2022 14:13:32 +0100 Subject: [PATCH 03/11] Add more tests; fix fine-grained --- mypy/server/deps.py | 21 ++++++++++++------ test-data/unit/check-recursive-types.test | 27 +++++++++++++++++++++-- test-data/unit/fine-grained.test | 24 ++++++++++++++++++++ 3 files changed, 63 insertions(+), 9 deletions(-) diff --git a/mypy/server/deps.py b/mypy/server/deps.py index 179e430afad5..092eb2968bce 100644 --- a/mypy/server/deps.py +++ b/mypy/server/deps.py @@ -941,18 +941,23 @@ def get_type_triggers(self, typ: Type) -> List[str]: return get_type_triggers(typ, self.use_logical_deps()) -def get_type_triggers(typ: Type, use_logical_deps: bool) -> List[str]: +def get_type_triggers( + typ: Type, use_logical_deps: bool, seen_aliases: Optional[Set[TypeAliasType]] = None +) -> List[str]: """Return all triggers that correspond to a type becoming stale.""" - return typ.accept(TypeTriggersVisitor(use_logical_deps)) + return typ.accept(TypeTriggersVisitor(use_logical_deps, seen_aliases)) class TypeTriggersVisitor(TypeVisitor[List[str]]): - def __init__(self, use_logical_deps: bool) -> None: + def __init__( + self, use_logical_deps: bool, seen_aliases: Optional[Set[TypeAliasType]] = None + ) -> None: self.deps: List[str] = [] + self.seen_aliases: Set[TypeAliasType] = seen_aliases or set() self.use_logical_deps = use_logical_deps def get_type_triggers(self, typ: Type) -> List[str]: - return get_type_triggers(typ, self.use_logical_deps) + return get_type_triggers(typ, self.use_logical_deps, self.seen_aliases) def visit_instance(self, typ: Instance) -> List[str]: trigger = make_trigger(typ.type.fullname) @@ -964,14 +969,16 @@ def visit_instance(self, typ: Instance) -> List[str]: return triggers def visit_type_alias_type(self, typ: TypeAliasType) -> List[str]: + if typ in self.seen_aliases: + return [] + self.seen_aliases.add(typ) assert typ.alias is not None trigger = make_trigger(typ.alias.fullname) triggers = [trigger] for arg in typ.args: triggers.extend(self.get_type_triggers(arg)) - # TODO: Add guard for infinite recursion here. Moreover, now that type aliases - # are its own kind of types we can simplify the logic to rely on intermediate - # dependencies (like for instance types). + # TODO: Now that type aliases are its own kind of types we can simplify + # the logic to rely on intermediate dependencies (like for instance types). triggers.extend(self.get_type_triggers(typ.alias.target)) return triggers diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index 1ef14e476cc9..020128a32462 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -475,13 +475,36 @@ if nt.x is not None: [case testBasicRegularTupleClass] # flags: --enable-recursive-aliases +from typing import Tuple + +x: B +class B(Tuple[B, int]): + x: int + +b, _ = x +reveal_type(b.x) [builtins fixtures/tuple.pyi] [case testBasicTupleClassesNewType] # flags: --enable-recursive-aliases -# check both regular and named +from typing import Tuple, NamedTuple, NewType + +x: C +class B(Tuple[B, int]): + x: int +C = NewType(B) +b, _ = x +reveal_type(b.x) + +y: BNT +class BNT(NamedTuple): + x: BNT + y: int +CNT = NewType(BNT) +bnt, _ = y +reveal_type(bnt.y) [builtins fixtures/tuple.pyi] --- Add fine-grained tests +-- TODO: Add more fine-grained tests for various kinds of recursive types -- Tests duplicating some named tuple existing tests with recursive aliases enabled diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 4a1dc5cc93c7..27bd857b70a7 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -3448,6 +3448,30 @@ f(a.x) [out] == +[case testNamedTupleUpdate5] +# flags: --enable-recursive-aliases +import b +[file a.py] +from typing import NamedTuple, Optional +class N(NamedTuple): + r: Optional[N] + x: int +x = N(None, 1) +[file a.py.2] +from typing import NamedTuple, Optional +class N(NamedTuple): + r: Optional[N] + x: str +x = N(None, 'hi') +[file b.py] +import a +def f(x: a.N) -> None: + pass +f(a.x) +[builtins fixtures/tuple.pyi] +[out] +== + [case testTypedDictRefresh] [builtins fixtures/dict.pyi] import a From 68263eeb68feb6e9e120cf3d66ba93e774325006 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 9 Aug 2022 15:08:43 +0100 Subject: [PATCH 04/11] Fix regular tuple bases and newtypes --- mypy/semanal.py | 7 +++++++ mypy/semanal_newtype.py | 1 + test-data/unit/check-recursive-types.test | 15 ++++++++------- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 2392ec1e3c05..65236daedf5e 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1841,6 +1841,13 @@ def configure_tuple_base_class(self, defn: ClassDef, base: TupleType) -> Instanc self.fail("Class has two incompatible bases derived from tuple", defn) defn.has_incompatible_baseclass = True info.tuple_type = base + target = base.copy_modified(fallback=Instance(info, [])) + if not info.tuple_alias: + info.tuple_alias = TypeAlias(target, info.fullname, info.line, info.column) + else: + if has_placeholder(info.tuple_alias.target): + self.defer(force_progress=True) + info.tuple_alias.target = target if base.partial_fallback.type.fullname == "builtins.tuple": # Fallback can only be safely calculated after semantic analysis, since base diff --git a/mypy/semanal_newtype.py b/mypy/semanal_newtype.py index 11d325b26bb1..e03cc1bcabe1 100644 --- a/mypy/semanal_newtype.py +++ b/mypy/semanal_newtype.py @@ -93,6 +93,7 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> bool: name, old_type, old_type.partial_fallback, s.line ) newtype_class_info.tuple_type = old_type + newtype_class_info.tuple_alias = old_type.partial_fallback.type.tuple_alias elif isinstance(old_type, Instance): if old_type.type.is_protocol: self.fail("NewType cannot be used with protocol classes", s) diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index 020128a32462..ba82fe62482b 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -482,7 +482,7 @@ class B(Tuple[B, int]): x: int b, _ = x -reveal_type(b.x) +reveal_type(b.x) # N:Revealed type is "builtins.int" [builtins fixtures/tuple.pyi] [case testBasicTupleClassesNewType] @@ -492,17 +492,18 @@ from typing import Tuple, NamedTuple, NewType x: C class B(Tuple[B, int]): x: int -C = NewType(B) +C = NewType("C", B) b, _ = x -reveal_type(b.x) +reveal_type(b) # N: Revealed type is "Tuple[..., builtins.int, fallback=__main__.B]" +reveal_type(b.x) # N: Revealed type is "builtins.int" -y: BNT +y: CNT class BNT(NamedTuple): - x: BNT + x: CNT y: int -CNT = NewType(BNT) +CNT = NewType("CNT", BNT) bnt, _ = y -reveal_type(bnt.y) +reveal_type(bnt.y) # N: Revealed type is "builtins.int" [builtins fixtures/tuple.pyi] -- TODO: Add more fine-grained tests for various kinds of recursive types From 786655b3cae74f8081382394b3403e702efc5152 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 9 Aug 2022 18:19:01 +0100 Subject: [PATCH 05/11] Fix newtype tests --- mypy/checkmember.py | 5 +-- mypy/nodes.py | 11 ++++++ mypy/semanal_newtype.py | 43 ++++++++++++++++++----- test-data/unit/check-recursive-types.test | 2 +- 4 files changed, 49 insertions(+), 12 deletions(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 4bd645f1150c..a2f9db117325 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -563,6 +563,7 @@ def analyze_descriptor_access(descriptor_type: Type, mx: MemberContext) -> Type: The return type of the appropriate ``__get__`` overload for the descriptor. """ instance_type = get_proper_type(mx.original_type) + orig_descriptor_type = descriptor_type descriptor_type = get_proper_type(descriptor_type) if isinstance(descriptor_type, UnionType): @@ -571,10 +572,10 @@ def analyze_descriptor_access(descriptor_type: Type, mx: MemberContext) -> Type: [analyze_descriptor_access(typ, mx) for typ in descriptor_type.items] ) elif not isinstance(descriptor_type, Instance): - return descriptor_type + return orig_descriptor_type if not descriptor_type.type.has_readable_member("__get__"): - return descriptor_type + return orig_descriptor_type dunder_get = descriptor_type.type.get_method("__get__") if dunder_get is None: diff --git a/mypy/nodes.py b/mypy/nodes.py index b49412e45be9..15b02e2c15c0 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -3258,6 +3258,17 @@ def __init__( self.eager = eager super().__init__(line, column) + @classmethod + def from_tuple_type(cls, info: TypeInfo) -> "TypeAlias": + return TypeAlias( + info.tuple_type.copy_modified( + fallback=mypy.types.Instance(info, []) + ), + info.fullname, + info.line, + info.column, + ) + @property def name(self) -> str: return self._fullname.split(".")[-1] diff --git a/mypy/semanal_newtype.py b/mypy/semanal_newtype.py index e03cc1bcabe1..529966a5edb2 100644 --- a/mypy/semanal_newtype.py +++ b/mypy/semanal_newtype.py @@ -24,11 +24,12 @@ RefExpr, StrExpr, SymbolTableNode, + TypeAlias, TypeInfo, Var, ) from mypy.options import Options -from mypy.semanal_shared import SemanticAnalyzerInterface +from mypy.semanal_shared import SemanticAnalyzerInterface, has_placeholder from mypy.typeanal import check_for_explicit_any, has_any_from_unimported_type from mypy.types import ( AnyType, @@ -90,14 +91,20 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> bool: # Create the corresponding class definition if the aliased type is subtypeable if isinstance(old_type, TupleType): newtype_class_info = self.build_newtype_typeinfo( - name, old_type, old_type.partial_fallback, s.line + name, old_type, old_type.partial_fallback, s.line, call.analyzed.info ) newtype_class_info.tuple_type = old_type - newtype_class_info.tuple_alias = old_type.partial_fallback.type.tuple_alias + alias = TypeAlias.from_tuple_type(newtype_class_info) + if not newtype_class_info.tuple_alias: + newtype_class_info.tuple_alias = alias + else: + newtype_class_info.tuple_alias.target = alias.target elif isinstance(old_type, Instance): if old_type.type.is_protocol: self.fail("NewType cannot be used with protocol classes", s) - newtype_class_info = self.build_newtype_typeinfo(name, old_type, old_type, s.line) + newtype_class_info = self.build_newtype_typeinfo( + name, old_type, old_type, s.line, call.analyzed.info + ) else: if old_type is not None: message = "Argument 2 to NewType(...) must be subclassable (got {})" @@ -105,7 +112,9 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> bool: # Otherwise the error was already reported. old_type = AnyType(TypeOfAny.from_error) object_type = self.api.named_type("builtins.object") - newtype_class_info = self.build_newtype_typeinfo(name, old_type, object_type, s.line) + newtype_class_info = self.build_newtype_typeinfo( + name, old_type, object_type, s.line, call.analyzed.info + ) newtype_class_info.fallback_to_any = True check_for_explicit_any( @@ -195,9 +204,18 @@ def check_newtype_args( # We want to use our custom error message (see above), so we suppress # the default error message for invalid types here. - old_type = get_proper_type(self.api.anal_type(unanalyzed_type, report_invalid_types=False)) + old_type = get_proper_type( + self.api.anal_type( + unanalyzed_type, + report_invalid_types=False, + allow_placeholder=self.options.enable_recursive_aliases + and not self.api.is_func_scope(), + ) + ) should_defer = False - if old_type is None or isinstance(old_type, PlaceholderType): + if isinstance(old_type, PlaceholderType): + old_type = None + if old_type is None: should_defer = True # The caller of this function assumes that if we return a Type, it's always @@ -209,9 +227,14 @@ def check_newtype_args( return None if has_failed else old_type, should_defer def build_newtype_typeinfo( - self, name: str, old_type: Type, base_type: Instance, line: int + self, + name: str, + old_type: Type, + base_type: Instance, + line: int, + existing_info: Optional[TypeInfo], ) -> TypeInfo: - info = self.api.basic_new_typeinfo(name, base_type, line) + info = existing_info or self.api.basic_new_typeinfo(name, base_type, line) info.is_newtype = True # Add __init__ method @@ -232,6 +255,8 @@ def build_newtype_typeinfo( init_func._fullname = info.fullname + ".__init__" info.names["__init__"] = SymbolTableNode(MDEF, init_func) + if info.tuple_type and has_placeholder(info.tuple_type): + self.api.defer(force_progress=True) return info # Helpers diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index ba82fe62482b..3d24c91c9e65 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -482,7 +482,7 @@ class B(Tuple[B, int]): x: int b, _ = x -reveal_type(b.x) # N:Revealed type is "builtins.int" +reveal_type(b.x) # N: Revealed type is "builtins.int" [builtins fixtures/tuple.pyi] [case testBasicTupleClassesNewType] From 952692bc7b89e58bbcb70e9cd59e1ab4dbe17667 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 9 Aug 2022 19:05:37 +0100 Subject: [PATCH 06/11] Cleanup; add more tests --- mypy/fixup.py | 4 +- mypy/nodes.py | 5 +- mypy/semanal.py | 6 +- mypy/semanal_namedtuple.py | 6 +- mypy/semanal_newtype.py | 1 + test-data/unit/check-recursive-types.test | 99 ++++++++++++++++++++--- 6 files changed, 99 insertions(+), 22 deletions(-) diff --git a/mypy/fixup.py b/mypy/fixup.py index 56ca8c3f4263..d4c8e2766114 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -338,9 +338,7 @@ def lookup_fully_qualified_alias( if isinstance(node, TypeAlias): return node elif isinstance(node, TypeInfo): - assert node.tuple_type - target = node.tuple_type.copy_modified(fallback=Instance(node, [])) - alias = TypeAlias(target, node.fullname, node.line, node.column) + alias = TypeAlias.from_tuple_type(node) node.tuple_alias = alias return alias else: diff --git a/mypy/nodes.py b/mypy/nodes.py index 15b02e2c15c0..52820ad551e1 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -3260,10 +3260,9 @@ def __init__( @classmethod def from_tuple_type(cls, info: TypeInfo) -> "TypeAlias": + assert info.tuple_type return TypeAlias( - info.tuple_type.copy_modified( - fallback=mypy.types.Instance(info, []) - ), + info.tuple_type.copy_modified(fallback=mypy.types.Instance(info, [])), info.fullname, info.line, info.column, diff --git a/mypy/semanal.py b/mypy/semanal.py index 65236daedf5e..e356b5dbac8e 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1841,13 +1841,13 @@ def configure_tuple_base_class(self, defn: ClassDef, base: TupleType) -> Instanc self.fail("Class has two incompatible bases derived from tuple", defn) defn.has_incompatible_baseclass = True info.tuple_type = base - target = base.copy_modified(fallback=Instance(info, [])) + alias = TypeAlias.from_tuple_type(info) if not info.tuple_alias: - info.tuple_alias = TypeAlias(target, info.fullname, info.line, info.column) + info.tuple_alias = alias else: if has_placeholder(info.tuple_alias.target): self.defer(force_progress=True) - info.tuple_alias.target = target + info.tuple_alias.target = alias.target if base.partial_fallback.type.fullname == "builtins.tuple": # Fallback can only be safely calculated after semantic analysis, since base diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index 557e342d1177..c8a3c929c47c 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -480,11 +480,11 @@ def build_namedtuple_typeinfo( if existing_info: old_tuple_type = existing_info.tuple_type info.tuple_type = tuple_base - target = tuple_base.copy_modified(fallback=Instance(info, [])) + alias = TypeAlias.from_tuple_type(info) if not info.tuple_alias: - info.tuple_alias = TypeAlias(target, info.fullname, info.line, info.column) + info.tuple_alias = alias else: - info.tuple_alias.target = target + info.tuple_alias.target = alias.target info.line = line # For use by mypyc. info.metadata["namedtuple"] = {"fields": items.copy()} diff --git a/mypy/semanal_newtype.py b/mypy/semanal_newtype.py index 529966a5edb2..0788d5bf936d 100644 --- a/mypy/semanal_newtype.py +++ b/mypy/semanal_newtype.py @@ -89,6 +89,7 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> bool: return True # Create the corresponding class definition if the aliased type is subtypeable + assert isinstance(call.analyzed, NewTypeExpr) if isinstance(old_type, TupleType): newtype_class_info = self.build_newtype_typeinfo( name, old_type, old_type.partial_fallback, s.line, call.analyzed.info diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index 3d24c91c9e65..a7e894db7acd 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -90,7 +90,7 @@ else: A = int | list[A] [builtins fixtures/isinstancelist.pyi] --- Tests duplicating some type alias existing tests with recursive aliases enabled +-- Tests duplicating some existing type alias tests with recursive aliases enabled [case testRecursiveAliasesMutual] # flags: --enable-recursive-aliases @@ -427,7 +427,7 @@ x: U reveal_type(x) # N: Revealed type is "Type[Any]" [builtins fixtures/isinstancelist.pyi] -[case testBasicNamedTuple] +[case testBasicRecursiveNamedTuple] # flags: --enable-recursive-aliases from typing import NamedTuple, Optional @@ -441,9 +441,9 @@ if nt.x is not None: y = nt.x[0] # E: Incompatible types in assignment (expression has type "Optional[NT]", variable has type "str") [builtins fixtures/tuple.pyi] -[case testBasicNamedTupleSpecial] +[case testBasicRecursiveNamedTupleSpecial] # flags: --enable-recursive-aliases -from typing import NamedTuple +from typing import NamedTuple, TypeVar, Tuple NT = NamedTuple("NT", [("x", NT), ("y", int)]) nt: NT @@ -453,10 +453,17 @@ reveal_type(nt[0]) # N: Revealed type is "Tuple[Tuple[..., builtins.int, fallba y: str if nt.x is not None: y = nt.x[0] # E: Incompatible types in assignment (expression has type "NT", variable has type "str") -# XXX check join no infinite recursion on fallbacks + +T = TypeVar("T") +def f(a: T, b: T) -> T: ... +tnt: Tuple[NT] + +# Should these be tuple[object] instead? +reveal_type(f(nt, tnt)) # N: Revealed type is "builtins.tuple[Any, ...]" +reveal_type(f(tnt, nt)) # N: Revealed type is "builtins.tuple[Any, ...]" [builtins fixtures/tuple.pyi] -[case testBasicNamedTupleClass] +[case testBasicRecursiveNamedTupleClass] # flags: --enable-recursive-aliases from typing import NamedTuple, Optional @@ -473,7 +480,7 @@ if nt.x is not None: y = nt.x[0] # E: Incompatible types in assignment (expression has type "Optional[NT]", variable has type "str") [builtins fixtures/tuple.pyi] -[case testBasicRegularTupleClass] +[case testRecursiveRegularTupleClass] # flags: --enable-recursive-aliases from typing import Tuple @@ -485,7 +492,7 @@ b, _ = x reveal_type(b.x) # N: Revealed type is "builtins.int" [builtins fixtures/tuple.pyi] -[case testBasicTupleClassesNewType] +[case testRecursiveTupleClassesNewType] # flags: --enable-recursive-aliases from typing import Tuple, NamedTuple, NewType @@ -506,6 +513,78 @@ bnt, _ = y reveal_type(bnt.y) # N: Revealed type is "builtins.int" [builtins fixtures/tuple.pyi] --- TODO: Add more fine-grained tests for various kinds of recursive types +-- Tests duplicating some existing named tuple tests with recursive aliases enabled + +[case testMutuallyRecursiveNamedTuples] +# flags: --enable-recursive-aliases +from typing import Tuple, NamedTuple, TypeVar, Union + +A = NamedTuple('A', [('x', str), ('y', Tuple[B, ...])]) +class B(NamedTuple): + x: A + y: int --- Tests duplicating some named tuple existing tests with recursive aliases enabled +n: A +reveal_type(n) # N: Revealed type is "Tuple[builtins.str, builtins.tuple[Tuple[..., builtins.int, fallback=__main__.B], ...], fallback=__main__.A]" + +T = TypeVar("T") +S = TypeVar("S") +def foo(arg: Tuple[T, S]) -> Union[T, S]: ... +x = foo(n) +y: str = x # E: Incompatible types in assignment (expression has type "Union[str, Tuple[B, ...]]", variable has type "str") +[builtins fixtures/tuple.pyi] + +[case testMutuallyRecursiveNamedTuplesJoin] +# flags: --enable-recursive-aliases +from typing import NamedTuple, Tuple + +class B(NamedTuple): + x: Tuple[A, int] + y: int + +A = NamedTuple('A', [('x', str), ('y', B)]) +n: B +m: A +reveal_type(n.x) # N: Revealed type is "Tuple[Tuple[builtins.str, Tuple[Tuple[..., builtins.int], builtins.int, fallback=__main__.B], fallback=__main__.A], builtins.int]" +reveal_type(m[0]) # N: Revealed type is "builtins.str" +lst = [m, n] +reveal_type(lst[0]) # N: Revealed type is "Tuple[builtins.object, builtins.object]" +[builtins fixtures/tuple.pyi] + +[case testMutuallyRecursiveNamedTuplesClasses] +# flags: --enable-recursive-aliases +from typing import NamedTuple, Tuple + +class B(NamedTuple): + x: A + y: int +class A(NamedTuple): + x: str + y: B + +n: A +reveal_type(n.y[0]) # N: Revealed type is "Tuple[builtins.str, Tuple[Tuple[builtins.str, ..., fallback=__main__.A], builtins.int, fallback=__main__.B], fallback=__main__.A]" + +m: B +n = m.x +n = n.y.x + +t: Tuple[str, B] +t = n +t = m # E: Incompatible types in assignment (expression has type "B", variable has type "Tuple[str, B]") +[builtins fixtures/tuple.pyi] + +[case testMutuallyRecursiveNamedTuplesCalls] +# flags: --enable-recursive-aliases +from typing import NamedTuple + +B = NamedTuple('B', [('x', A), ('y', int)]) +A = NamedTuple('A', [('x', str), ('y', 'B')]) +n: A +def f(m: B) -> None: pass +reveal_type(n) # N: Revealed type is "Tuple[builtins.str, Tuple[..., builtins.int, fallback=__main__.B], fallback=__main__.A]" +reveal_type(f) # N: Revealed type is "def (m: Tuple[Tuple[builtins.str, ..., fallback=__main__.A], builtins.int, fallback=__main__.B])" +f(n) # E: Argument 1 to "f" has incompatible type "A"; expected "B" +[builtins fixtures/tuple.pyi] + +-- TODO: Add more fine-grained tests for various kinds of recursive types From 3aeb47075e029772b227229f1659fc96542534f1 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 9 Aug 2022 20:03:50 +0100 Subject: [PATCH 07/11] More cleanup and tests --- mypy/nodes.py | 10 ++++++++++ mypy/semanal.py | 11 +++-------- mypy/semanal_namedtuple.py | 22 ++++++++-------------- mypy/semanal_newtype.py | 8 +------- mypy/semanal_shared.py | 8 ++++---- mypy/typeanal.py | 3 +++ test-data/unit/check-recursive-types.test | 16 ++++++++++++++-- 7 files changed, 43 insertions(+), 35 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 52820ad551e1..c56147ae0530 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2970,6 +2970,15 @@ def direct_base_classes(self) -> "List[TypeInfo]": """ return [base.type for base in self.bases] + def update_tuple_type(self, typ: "mypy.types.TupleType") -> None: + """Update tuple_type and tuple_alias as needed.""" + self.tuple_type = typ + alias = TypeAlias.from_tuple_type(self) + if not self.tuple_alias: + self.tuple_alias = alias + else: + self.tuple_alias.target = alias.target + def __str__(self) -> str: """Return a string representation of the type. @@ -3260,6 +3269,7 @@ def __init__( @classmethod def from_tuple_type(cls, info: TypeInfo) -> "TypeAlias": + """Generate an alias to the tuple type described by a given TypeInfo.""" assert info.tuple_type return TypeAlias( info.tuple_type.copy_modified(fallback=mypy.types.Instance(info, [])), diff --git a/mypy/semanal.py b/mypy/semanal.py index e356b5dbac8e..7c1b5f9e3fd5 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1840,14 +1840,9 @@ def configure_tuple_base_class(self, defn: ClassDef, base: TupleType) -> Instanc if info.tuple_type and info.tuple_type != base: self.fail("Class has two incompatible bases derived from tuple", defn) defn.has_incompatible_baseclass = True - info.tuple_type = base - alias = TypeAlias.from_tuple_type(info) - if not info.tuple_alias: - info.tuple_alias = alias - else: - if has_placeholder(info.tuple_alias.target): - self.defer(force_progress=True) - info.tuple_alias.target = alias.target + if info.tuple_alias and has_placeholder(info.tuple_alias.target): + self.defer(force_progress=True) + info.update_tuple_type(base) if base.partial_fallback.type.fullname == "builtins.tuple": # Fallback can only be safely calculated after semantic analysis, since base diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index c8a3c929c47c..d6d622593961 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -35,7 +35,6 @@ SymbolTableNode, TempNode, TupleExpr, - TypeAlias, TypeInfo, TypeVarExpr, Var, @@ -52,7 +51,6 @@ TYPED_NAMEDTUPLE_NAMES, AnyType, CallableType, - Instance, LiteralType, TupleType, Type, @@ -170,6 +168,9 @@ def check_namedtuple_classdef( if stmt.type is None: types.append(AnyType(TypeOfAny.unannotated)) else: + # We never allow recursive types at function scope. Although it is + # possible to support this for named tuples, it is still tricky, and + # it would be inconsistent with type aliases. analyzed = self.api.anal_type( stmt.type, allow_placeholder=self.options.enable_recursive_aliases @@ -428,6 +429,7 @@ def parse_namedtuple_fields_with_types( except TypeTranslationError: self.fail("Invalid field type", type_node) return None + # We never allow recursive types at function scope. analyzed = self.api.anal_type( type, allow_placeholder=self.options.enable_recursive_aliases @@ -476,22 +478,13 @@ def build_namedtuple_typeinfo( info = existing_info or self.api.basic_new_typeinfo(name, fallback, line) info.is_named_tuple = True tuple_base = TupleType(types, fallback) - old_tuple_type = None - if existing_info: - old_tuple_type = existing_info.tuple_type - info.tuple_type = tuple_base - alias = TypeAlias.from_tuple_type(info) - if not info.tuple_alias: - info.tuple_alias = alias - else: - info.tuple_alias.target = alias.target + if info.tuple_alias and has_placeholder(info.tuple_alias.target): + self.api.defer(force_progress=True) + info.update_tuple_type(tuple_base) info.line = line # For use by mypyc. info.metadata["namedtuple"] = {"fields": items.copy()} - if old_tuple_type and has_placeholder(old_tuple_type): - self.api.defer(force_progress=True) - # We can't calculate the complete fallback type until after semantic # analysis, since otherwise base classes might be incomplete. Postpone a # callback function that patches the fallback. @@ -528,6 +521,7 @@ def add_field( if self.options.python_version >= (3, 10): add_field(Var("__match_args__", match_args_type), is_initialized_in_class=True) + assert info.tuple_type is not None # Set by update_tuple_type() above. tvd = TypeVarType( SELF_TVAR_NAME, info.fullname + "." + SELF_TVAR_NAME, -1, [], info.tuple_type ) diff --git a/mypy/semanal_newtype.py b/mypy/semanal_newtype.py index 0788d5bf936d..6fe6cd4a4295 100644 --- a/mypy/semanal_newtype.py +++ b/mypy/semanal_newtype.py @@ -24,7 +24,6 @@ RefExpr, StrExpr, SymbolTableNode, - TypeAlias, TypeInfo, Var, ) @@ -94,12 +93,7 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> bool: newtype_class_info = self.build_newtype_typeinfo( name, old_type, old_type.partial_fallback, s.line, call.analyzed.info ) - newtype_class_info.tuple_type = old_type - alias = TypeAlias.from_tuple_type(newtype_class_info) - if not newtype_class_info.tuple_alias: - newtype_class_info.tuple_alias = alias - else: - newtype_class_info.tuple_alias.target = alias.target + newtype_class_info.update_tuple_type(old_type) elif isinstance(old_type, Instance): if old_type.type.is_protocol: self.fail("NewType cannot be used with protocol classes", s) diff --git a/mypy/semanal_shared.py b/mypy/semanal_shared.py index 873ee2788d58..51f77c1b5d33 100644 --- a/mypy/semanal_shared.py +++ b/mypy/semanal_shared.py @@ -108,6 +108,10 @@ def is_future_flag_set(self, flag: str) -> bool: def is_stub_file(self) -> bool: raise NotImplementedError + @abstractmethod + def is_func_scope(self) -> bool: + raise NotImplementedError + @trait class SemanticAnalyzerInterface(SemanticAnalyzerCoreInterface): @@ -211,10 +215,6 @@ def qualified_name(self, n: str) -> str: def is_typeshed_stub_file(self) -> bool: raise NotImplementedError - @abstractmethod - def is_func_scope(self) -> bool: - raise NotImplementedError - def set_callable_name(sig: Type, fdef: FuncDef) -> ProperType: sig = get_proper_type(sig) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 4e4904331a58..22e467f49215 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -393,6 +393,8 @@ def cannot_resolve_type(self, t: UnboundType) -> None: # need access to MessageBuilder here. Also move the similar # message generation logic in semanal.py. self.api.fail(f'Cannot resolve name "{t.name}" (possible cyclic definition)', t) + if self.options.enable_recursive_aliases and self.api.is_func_scope(): + self.note("Recursive types are not allowed at function scope", t) def apply_concatenate_operator(self, t: UnboundType) -> Type: if len(t.args) == 0: @@ -611,6 +613,7 @@ def analyze_type_with_type_info( self.fail("Generic tuple types not supported", ctx) return AnyType(TypeOfAny.from_error) if info.tuple_alias: + # We don't support generic tuple types yet. return TypeAliasType(info.tuple_alias, []) return tup.copy_modified(items=self.anal_array(tup.items), fallback=instance) td = info.typeddict_type diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index a7e894db7acd..e0fe2013e940 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -545,7 +545,7 @@ class B(NamedTuple): A = NamedTuple('A', [('x', str), ('y', B)]) n: B m: A -reveal_type(n.x) # N: Revealed type is "Tuple[Tuple[builtins.str, Tuple[Tuple[..., builtins.int], builtins.int, fallback=__main__.B], fallback=__main__.A], builtins.int]" +s: str = n.x # E: Incompatible types in assignment (expression has type "Tuple[A, int]", variable has type "str") reveal_type(m[0]) # N: Revealed type is "builtins.str" lst = [m, n] reveal_type(lst[0]) # N: Revealed type is "Tuple[builtins.object, builtins.object]" @@ -563,7 +563,7 @@ class A(NamedTuple): y: B n: A -reveal_type(n.y[0]) # N: Revealed type is "Tuple[builtins.str, Tuple[Tuple[builtins.str, ..., fallback=__main__.A], builtins.int, fallback=__main__.B], fallback=__main__.A]" +s: str = n.y[0] # E: Incompatible types in assignment (expression has type "A", variable has type "str") m: B n = m.x @@ -587,4 +587,16 @@ reveal_type(f) # N: Revealed type is "def (m: Tuple[Tuple[builtins.str, ..., fal f(n) # E: Argument 1 to "f" has incompatible type "A"; expected "B" [builtins fixtures/tuple.pyi] +[case testNoRecursiveTuplesAtFunctionScope] +# flags: --enable-recursive-aliases +from typing import NamedTuple, Tuple +def foo() -> None: + class B(NamedTuple): + x: B # E: Cannot resolve name "B" (possible cyclic definition) \ + # N: Recursive types are not allowed at function scope + y: int + b: B + reveal_type(b) # N: Revealed type is "Tuple[Any, builtins.int, fallback=__main__.B@4]" +[builtins fixtures/tuple.pyi] + -- TODO: Add more fine-grained tests for various kinds of recursive types From 404d8d10851e7f78a33af4deb43b3f4c2852df00 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 9 Aug 2022 20:24:12 +0100 Subject: [PATCH 08/11] Fix mypyc --- mypy/semanal_shared.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/semanal_shared.py b/mypy/semanal_shared.py index 51f77c1b5d33..2c1d843f4c7a 100644 --- a/mypy/semanal_shared.py +++ b/mypy/semanal_shared.py @@ -84,7 +84,7 @@ def record_incomplete_ref(self) -> None: raise NotImplementedError @abstractmethod - def defer(self, debug_context: Optional[Context] = ..., force_progress: bool = ...) -> None: + def defer(self, debug_context: Optional[Context] = None, force_progress: bool = False) -> None: raise NotImplementedError @abstractmethod From dd3d27bbddc17ea7b8f090be564a70651e2ce776 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 10 Aug 2022 12:40:57 +0100 Subject: [PATCH 09/11] Add and fix more incremental tests --- mypy/fixup.py | 1 + mypy/semanal.py | 5 +- mypy/server/astmerge.py | 3 + mypy/server/aststrip.py | 1 + test-data/unit/check-incremental.test | 105 ++++++++++++++++++++++ test-data/unit/check-recursive-types.test | 2 - test-data/unit/fine-grained.test | 105 ++++++++++++++++++++++ 7 files changed, 217 insertions(+), 5 deletions(-) diff --git a/mypy/fixup.py b/mypy/fixup.py index d4c8e2766114..49df54e16440 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -75,6 +75,7 @@ def visit_type_info(self, info: TypeInfo) -> None: p.accept(self.type_fixer) if info.tuple_type: info.tuple_type.accept(self.type_fixer) + info.update_tuple_type(info.tuple_type) if info.typeddict_type: info.typeddict_type.accept(self.type_fixer) if info.declared_metaclass: diff --git a/mypy/semanal.py b/mypy/semanal.py index 1e6388442595..0e349d26c6fd 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1787,7 +1787,6 @@ def configure_base_classes( base_types: List[Instance] = [] info = defn.info - info.tuple_type = None for base, base_expr in bases: if isinstance(base, TupleType): actual_base = self.configure_tuple_base_class(defn, base) @@ -1838,14 +1837,14 @@ def configure_tuple_base_class(self, defn: ClassDef, base: TupleType) -> Instanc # There may be an existing valid tuple type from previous semanal iterations. # Use equality to check if it is the case. - if info.tuple_type and info.tuple_type != base: + if info.tuple_type and info.tuple_type != base and not has_placeholder(info.tuple_type): self.fail("Class has two incompatible bases derived from tuple", defn) defn.has_incompatible_baseclass = True if info.tuple_alias and has_placeholder(info.tuple_alias.target): self.defer(force_progress=True) info.update_tuple_type(base) - if base.partial_fallback.type.fullname == "builtins.tuple": + if base.partial_fallback.type.fullname == "builtins.tuple" and not has_placeholder(base): # Fallback can only be safely calculated after semantic analysis, since base # classes may be incomplete. Postpone the calculation. self.schedule_patch(PRIORITY_FALLBACKS, lambda: calculate_tuple_fallback(base)) diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index 0ced8bf8ee54..d06de01308a5 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -349,6 +349,9 @@ def fixup_and_reset_typeinfo(self, node: TypeInfo) -> TypeInfo: # old MRO. new = cast(TypeInfo, self.replacements[node]) TypeState.reset_all_subtype_caches_for(new) + # Special case: tuple_alias is not exposed in symbol tables, but may appear + # in external types (e.g. named tuples), so we need to update it manually. + replace_object_state(new.tuple_alias, node.tuple_alias) return self.fixup(node) def fixup_type(self, typ: Optional[Type]) -> None: diff --git a/mypy/server/aststrip.py b/mypy/server/aststrip.py index 516a78c6c21c..4fced532ff2d 100644 --- a/mypy/server/aststrip.py +++ b/mypy/server/aststrip.py @@ -140,6 +140,7 @@ def visit_class_def(self, node: ClassDef) -> None: TypeState.reset_subtype_caches_for(node.info) # Kill the TypeInfo, since there is none before semantic analysis. node.info = CLASSDEF_NO_INFO + # TODO: Should we also do node.analyzed = None here? def save_implicit_attributes(self, node: ClassDef) -> None: """Produce callbacks that re-add attributes defined on self.""" diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index b724ed51d17c..d9b0759e76a7 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -5740,3 +5740,108 @@ class C: nt2: NT2 = NT2(x=1) [builtins fixtures/tuple.pyi] + +[case testNamedTupleUpdateNonRecursiveToRecursiveCoarse] +# flags: --enable-recursive-aliases +import c +[file a.py] +from b import M +from typing import NamedTuple, Optional +class N(NamedTuple): + r: Optional[M] + x: int +[file b.py] +from a import N +from typing import NamedTuple +class M(NamedTuple): + r: None + x: int +[file b.py.2] +from a import N +from typing import NamedTuple, Optional +class M(NamedTuple): + r: Optional[N] + x: int +[file c.py] +import a +def f(x: a.N) -> None: + if x.r is not None: + s: int = x.r.x +[file c.py.3] +import a +def f(x: a.N) -> None: + if x.r is not None and x.r.r is not None and x.r.r.r is not None: + reveal_type(x) + s: int = x.r.r.r.r +[builtins fixtures/tuple.pyi] +[out] +[out2] +[out3] +tmp/c.py:4: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int, fallback=b.M], None], builtins.int, fallback=a.N]" +tmp/c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") + +[case testTupleTypeUpdateNonRecursiveToRecursiveCoarse] +# flags: --enable-recursive-aliases +import c +[file a.py] +from b import M +from typing import Tuple, Optional +class N(Tuple[Optional[M], int]): ... +[file b.py] +from a import N +from typing import Tuple +class M(Tuple[None, int]): ... +[file b.py.2] +from a import N +from typing import Tuple, Optional +class M(Tuple[Optional[N], int]): ... +[file c.py] +import a +def f(x: a.N) -> None: + if x[0] is not None: + s: int = x[0][1] +[file c.py.3] +import a +def f(x: a.N) -> None: + if x[0] is not None and x[0][0] is not None and x[0][0][0] is not None: + reveal_type(x) + s: int = x[0][0][0][0] +[builtins fixtures/tuple.pyi] +[out] +[out2] +[out3] +tmp/c.py:4: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int, fallback=b.M], None], builtins.int, fallback=a.N]" +tmp/c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") + +[case testTypeAliasUpdateNonRecursiveToRecursiveCoarse] +# flags: --enable-recursive-aliases +import c +[file a.py] +from b import M +from typing import Tuple, Optional +N = Tuple[Optional[M], int] +[file b.py] +from a import N +from typing import Tuple +M = Tuple[None, int] +[file b.py.2] +from a import N +from typing import Tuple, Optional +M = Tuple[Optional[N], int] +[file c.py] +import a +def f(x: a.N) -> None: + if x[0] is not None: + s: int = x[0][1] +[file c.py.3] +import a +def f(x: a.N) -> None: + if x[0] is not None and x[0][0] is not None and x[0][0][0] is not None: + reveal_type(x) + s: int = x[0][0][0][0] +[builtins fixtures/tuple.pyi] +[out] +[out2] +[out3] +tmp/c.py:4: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int], None], builtins.int]" +tmp/c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index a1ce22461fd2..ad0a481b4477 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -610,5 +610,3 @@ def foo() -> None: b: B reveal_type(b) # N: Revealed type is "Tuple[Any, builtins.int, fallback=__main__.B@4]" [builtins fixtures/tuple.pyi] - --- TODO: Add more fine-grained tests for various kinds of recursive types diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 27bd857b70a7..ae672bfcd364 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -3472,6 +3472,111 @@ f(a.x) [out] == +[case testNamedTupleUpdateNonRecursiveToRecursiveFine] +# flags: --enable-recursive-aliases +import c +[file a.py] +from b import M +from typing import NamedTuple, Optional +class N(NamedTuple): + r: Optional[M] + x: int +[file b.py] +from a import N +from typing import NamedTuple +class M(NamedTuple): + r: None + x: int +[file b.py.2] +from a import N +from typing import NamedTuple, Optional +class M(NamedTuple): + r: Optional[N] + x: int +[file c.py] +import a +def f(x: a.N) -> None: + if x.r is not None: + s: int = x.r.x +[file c.py.3] +import a +def f(x: a.N) -> None: + if x.r is not None and x.r.r is not None and x.r.r.r is not None: + reveal_type(x) + s: int = x.r.r.r.r +[builtins fixtures/tuple.pyi] +[out] +== +== +c.py:4: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int, fallback=b.M], None], builtins.int, fallback=a.N]" +c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") + +[case testTupleTypeUpdateNonRecursiveToRecursiveFine] +# flags: --enable-recursive-aliases +import c +[file a.py] +from b import M +from typing import Tuple, Optional +class N(Tuple[Optional[M], int]): ... +[file b.py] +from a import N +from typing import Tuple +class M(Tuple[None, int]): ... +[file b.py.2] +from a import N +from typing import Tuple, Optional +class M(Tuple[Optional[N], int]): ... +[file c.py] +import a +def f(x: a.N) -> None: + if x[0] is not None: + s: int = x[0][1] +[file c.py.3] +import a +def f(x: a.N) -> None: + if x[0] is not None and x[0][0] is not None and x[0][0][0] is not None: + reveal_type(x) + s: int = x[0][0][0][0] +[builtins fixtures/tuple.pyi] +[out] +== +== +c.py:4: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int], None], builtins.int]" +c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") + +[case testTypeAliasUpdateNonRecursiveToRecursiveFine] +# flags: --enable-recursive-aliases +import c +[file a.py] +from b import M +from typing import Tuple, Optional +N = Tuple[Optional[M], int] +[file b.py] +from a import N +from typing import Tuple +M = Tuple[None, int] +[file b.py.2] +from a import N +from typing import Tuple, Optional +M = Tuple[Optional[N], int] +[file c.py] +import a +def f(x: a.N) -> None: + if x[0] is not None: + s: int = x[0][1] +[file c.py.3] +import a +def f(x: a.N) -> None: + if x[0] is not None and x[0][0] is not None and x[0][0][0] is not None: + reveal_type(x) + s: int = x[0][0][0][0] +[builtins fixtures/tuple.pyi] +[out] +== +== +.c.py:4: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int], None], builtins.int]" +c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") + [case testTypedDictRefresh] [builtins fixtures/dict.pyi] import a From 51dd8ad74a425fb44d3dad255ff8094e272cc8ca Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 10 Aug 2022 15:39:25 +0100 Subject: [PATCH 10/11] A more principled fix; More CR --- mypy/fixup.py | 2 ++ mypy/server/astmerge.py | 18 ++++++++++++------ mypy/server/aststrip.py | 2 +- mypy/util.py | 6 +++++- test-data/unit/check-incremental.test | 4 ++++ test-data/unit/check-recursive-types.test | 2 +- test-data/unit/fine-grained.test | 6 +++++- 7 files changed, 30 insertions(+), 10 deletions(-) diff --git a/mypy/fixup.py b/mypy/fixup.py index 49df54e16440..ed9361130529 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -339,6 +339,8 @@ def lookup_fully_qualified_alias( if isinstance(node, TypeAlias): return node elif isinstance(node, TypeInfo): + if node.tuple_alias: + return node.tuple_alias alias = TypeAlias.from_tuple_type(node) node.tuple_alias = alias return alias diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index d06de01308a5..8d7dbdf6f98c 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -45,7 +45,7 @@ See the main entry point merge_asts for more details. """ -from typing import Dict, List, Optional, TypeVar, cast +from typing import Dict, List, Optional, Tuple, TypeVar, cast from mypy.nodes import ( MDEF, @@ -172,6 +172,8 @@ def replacement_map_from_symbol_table( node.node.names, new_node.node.names, prefix ) replacements.update(type_repl) + if node.node.tuple_alias and new_node.node.tuple_alias: + replacements[new_node.node.tuple_alias] = node.node.tuple_alias return replacements @@ -334,7 +336,13 @@ def visit_type_alias(self, node: TypeAlias) -> None: def fixup(self, node: SN) -> SN: if node in self.replacements: new = self.replacements[node] - replace_object_state(new, node) + skip_slots: Tuple[str, ...] = () + if isinstance(node, TypeInfo) and isinstance(new, TypeInfo): + # Special case: tuple_alias is not exposed in symbol tables, but may appear + # in external types (e.g. named tuples), so we need to update it manually. + skip_slots = ("tuple_alias",) + replace_object_state(new.tuple_alias, node.tuple_alias) + replace_object_state(new, node, skip_slots=skip_slots) return cast(SN, new) return node @@ -349,9 +357,6 @@ def fixup_and_reset_typeinfo(self, node: TypeInfo) -> TypeInfo: # old MRO. new = cast(TypeInfo, self.replacements[node]) TypeState.reset_all_subtype_caches_for(new) - # Special case: tuple_alias is not exposed in symbol tables, but may appear - # in external types (e.g. named tuples), so we need to update it manually. - replace_object_state(new.tuple_alias, node.tuple_alias) return self.fixup(node) def fixup_type(self, typ: Optional[Type]) -> None: @@ -541,7 +546,8 @@ def replace_nodes_in_symbol_table( if node.node in replacements: new = replacements[node.node] old = node.node - replace_object_state(new, old) + # Needed for TypeInfo, see comment in fixup() above. + replace_object_state(new, old, skip_slots=("tuple_alias",)) node.node = new if isinstance(node.node, (Var, TypeAlias)): # Handle them here just in case these aren't exposed through the AST. diff --git a/mypy/server/aststrip.py b/mypy/server/aststrip.py index 4fced532ff2d..c5de46610005 100644 --- a/mypy/server/aststrip.py +++ b/mypy/server/aststrip.py @@ -140,7 +140,7 @@ def visit_class_def(self, node: ClassDef) -> None: TypeState.reset_subtype_caches_for(node.info) # Kill the TypeInfo, since there is none before semantic analysis. node.info = CLASSDEF_NO_INFO - # TODO: Should we also do node.analyzed = None here? + node.analyzed = None def save_implicit_attributes(self, node: ClassDef) -> None: """Produce callbacks that re-add attributes defined on self.""" diff --git a/mypy/util.py b/mypy/util.py index b783a84facf6..56f943802537 100644 --- a/mypy/util.py +++ b/mypy/util.py @@ -359,7 +359,9 @@ def get_class_descriptors(cls: "Type[object]") -> Sequence[str]: return fields_cache[cls] -def replace_object_state(new: object, old: object, copy_dict: bool = False) -> None: +def replace_object_state( + new: object, old: object, copy_dict: bool = False, skip_slots: Tuple[str, ...] = () +) -> None: """Copy state of old node to the new node. This handles cases where there is __dict__ and/or attribute descriptors @@ -374,6 +376,8 @@ def replace_object_state(new: object, old: object, copy_dict: bool = False) -> N new.__dict__ = old.__dict__ for attr in get_class_descriptors(old.__class__): + if attr in skip_slots: + continue try: if hasattr(old, attr): setattr(new, attr, getattr(old, attr)) diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index d9b0759e76a7..02e705cfbbac 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -5750,6 +5750,7 @@ from typing import NamedTuple, Optional class N(NamedTuple): r: Optional[M] x: int +n: N [file b.py] from a import N from typing import NamedTuple @@ -5773,12 +5774,15 @@ def f(x: a.N) -> None: if x.r is not None and x.r.r is not None and x.r.r.r is not None: reveal_type(x) s: int = x.r.r.r.r +f(a.n) +reveal_type(a.n) [builtins fixtures/tuple.pyi] [out] [out2] [out3] tmp/c.py:4: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int, fallback=b.M], None], builtins.int, fallback=a.N]" tmp/c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") +tmp/c.py:7: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int, fallback=b.M], None], builtins.int, fallback=a.N]" [case testTupleTypeUpdateNonRecursiveToRecursiveCoarse] # flags: --enable-recursive-aliases diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index ad0a481b4477..aa4bd4a7902d 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -470,7 +470,7 @@ T = TypeVar("T") def f(a: T, b: T) -> T: ... tnt: Tuple[NT] -# Should these be tuple[object] instead? +# TODO: these should be tuple[object] instead. reveal_type(f(nt, tnt)) # N: Revealed type is "builtins.tuple[Any, ...]" reveal_type(f(tnt, nt)) # N: Revealed type is "builtins.tuple[Any, ...]" [builtins fixtures/tuple.pyi] diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index ae672bfcd364..48829ff6b941 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -3481,6 +3481,7 @@ from typing import NamedTuple, Optional class N(NamedTuple): r: Optional[M] x: int +n: N [file b.py] from a import N from typing import NamedTuple @@ -3504,12 +3505,15 @@ def f(x: a.N) -> None: if x.r is not None and x.r.r is not None and x.r.r.r is not None: reveal_type(x) s: int = x.r.r.r.r +f(a.n) +reveal_type(a.n) [builtins fixtures/tuple.pyi] [out] == == c.py:4: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int, fallback=b.M], None], builtins.int, fallback=a.N]" c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") +c.py:7: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int, fallback=b.M], None], builtins.int, fallback=a.N]" [case testTupleTypeUpdateNonRecursiveToRecursiveFine] # flags: --enable-recursive-aliases @@ -3541,7 +3545,7 @@ def f(x: a.N) -> None: [out] == == -c.py:4: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int], None], builtins.int]" +c.py:4: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int, fallback=b.M], None], builtins.int, fallback=a.N]" c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") [case testTypeAliasUpdateNonRecursiveToRecursiveFine] From e91f45214d8033fdd2bf39080677bd0195c097df Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 10 Aug 2022 16:04:45 +0100 Subject: [PATCH 11/11] Fix --- test-data/unit/fine-grained.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 48829ff6b941..14acc8d1664e 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -3578,7 +3578,7 @@ def f(x: a.N) -> None: [out] == == -.c.py:4: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int], None], builtins.int]" +c.py:4: note: Revealed type is "Tuple[Union[Tuple[Union[..., None], builtins.int], None], builtins.int]" c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") [case testTypedDictRefresh]