Fixes to tuple fallbacks (#6442)

This changes tuple fallbacks to be calculated on demand during
type checking. This fixes some issues with fallbacks being imprecise.

In the new semantic analyzer, this introduces a new pass just after
the main semantic analysis pass to calculate precise item types for
`tuple` base classes. These can't be calculated on demand since
the base class is a variable-length tuple type (`Instance`) instead
of `TupleType`. We can't calculate these during the main semantic
analysis pass since base classes can be incomplete.

Fixes #6400.

Author: JukkaL

mypy/applytype.py

@@ -1,7 +1,7 @@
 from typing import Dict, Sequence, Optional
 
 import mypy.subtypes
-from mypy.sametypes import is_same_type
+import mypy.sametypes
 from mypy.expandtype import expand_type
 from mypy.types import Type, TypeVarId, TypeVarType, CallableType, AnyType, PartialType
 from mypy.messages import MessageBuilder
@@ -37,7 +37,7 @@ def apply_generic_arguments(callable: CallableType, orig_types: Sequence[Optiona
             if isinstance(type, TypeVarType) and type.values:
                 # Allow substituting T1 for T if every allowed value of T1
                 # is also a legal value of T.
-                if all(any(is_same_type(v, v1) for v in values)
+                if all(any(mypy.sametypes.is_same_type(v, v1) for v in values)
                        for v1 in type.values):
                     continue
             matching = []

mypy/checker.py

@@ -63,6 +63,7 @@
 from mypy.plugin import Plugin, CheckerPluginInterface
 from mypy.sharedparse import BINARY_MAGIC_METHODS
 from mypy.scope import Scope
+from mypy.typeops import tuple_fallback
 from mypy import state
 
 MYPY = False
@@ -1063,7 +1064,9 @@ def check_reverse_op_method(self, defn: FuncItem,
         forward_inst = reverse_type.arg_types[1]
         if isinstance(forward_inst, TypeVarType):
             forward_inst = forward_inst.upper_bound
-        if isinstance(forward_inst, (FunctionLike, TupleType, TypedDictType, LiteralType)):
+        elif isinstance(forward_inst, TupleType):
+            forward_inst = tuple_fallback(forward_inst)
+        elif isinstance(forward_inst, (FunctionLike, TypedDictType, LiteralType)):
             forward_inst = forward_inst.fallback
         if isinstance(forward_inst, TypeType):
             item = forward_inst.item
@@ -1955,7 +1958,7 @@ def lvalue_type_from_base(self, expr_node: Var,
                     self_type = self.scope.active_self_type()
                     assert self_type is not None, "Internal error: base lookup outside class"
                     if isinstance(self_type, TupleType):
-                        instance = self_type.fallback
+                        instance = tuple_fallback(self_type)
                     else:
                         instance = self_type
                     itype = map_instance_to_supertype(instance, base)
@@ -3260,7 +3263,7 @@ def partition_by_callable(self, typ: Type,
         # when we dummy up a new type.
         ityp = typ
         if isinstance(typ, TupleType):
-            ityp = typ.fallback
+            ityp = tuple_fallback(typ)
 
         if isinstance(ityp, Instance):
             method = ityp.type.get_method('__call__')

mypy/checkexpr.py

@@ -59,6 +59,7 @@
 from mypy.visitor import ExpressionVisitor
 from mypy.plugin import Plugin, MethodContext, MethodSigContext, FunctionContext
 from mypy.typeanal import make_optional_type
+from mypy.typeops import tuple_fallback
 
 # Type of callback user for checking individual function arguments. See
 # check_args() below for details.
@@ -347,7 +348,7 @@ def method_fullname(self, object_type: Type, method_name: str) -> Optional[str]:
             info = object_type.fallback.type.get_containing_type_info(method_name)
             type_name = info.fullname() if info is not None else None
         elif isinstance(object_type, TupleType):
-            type_name = object_type.fallback.type.fullname()
+            type_name = tuple_fallback(object_type).type.fullname()
 
         if type_name is not None:
             return '{}.{}'.format(type_name, method_name)
@@ -722,7 +723,7 @@ def check_call(self,
             return self.check_call(item, args, arg_kinds, context, arg_names,
                                    callable_node, arg_messages)
         elif isinstance(callee, TupleType):
-            return self.check_call(callee.fallback, args, arg_kinds, context,
+            return self.check_call(tuple_fallback(callee), args, arg_kinds, context,
                                    arg_names, callable_node, arg_messages, callable_name,
                                    object_type)
         else:
@@ -835,8 +836,9 @@ def analyze_type_type_callee(self, item: Type, context: Context) -> Type:
             if callee:
                 return callee
         # We support Type of namedtuples but not of tuples in general
-        if isinstance(item, TupleType) and item.fallback.type.fullname() != 'builtins.tuple':
-            return self.analyze_type_type_callee(item.fallback, context)
+        if (isinstance(item, TupleType)
+                and tuple_fallback(item).type.fullname() != 'builtins.tuple'):
+            return self.analyze_type_type_callee(tuple_fallback(item), context)
 
         self.msg.unsupported_type_type(item, context)
         return AnyType(TypeOfAny.from_error)
@@ -2666,8 +2668,8 @@ class LongName(Generic[T]): ...
             return self.apply_type_arguments_to_callable(tp, item.args, ctx)
         elif (isinstance(item, TupleType) and
               # Tuple[str, int]() fails at runtime, only named tuples and subclasses work.
-              item.fallback.type.fullname() != 'builtins.tuple'):
-            return type_object_type(item.fallback.type, self.named_type)
+              tuple_fallback(item).type.fullname() != 'builtins.tuple'):
+            return type_object_type(tuple_fallback(item).type, self.named_type)
         elif isinstance(item, AnyType):
             return AnyType(TypeOfAny.from_another_any, source_any=item)
         else:
@@ -2785,7 +2787,8 @@ def visit_tuple_expr(self, e: TupleExpr) -> Type:
                     tt = self.accept(item, type_context_items[j])
                     j += 1
                 items.append(tt)
-        fallback_item = join.join_type_list(items)
+        # This is a partial fallback item type. A precise type will be calculated on demand.
+        fallback_item = AnyType(TypeOfAny.special_form)
         return TupleType(items, self.chk.named_generic_type('builtins.tuple', [fallback_item]))
 
     def visit_dict_expr(self, e: DictExpr) -> Type:
@@ -2973,7 +2976,8 @@ def check_super_arguments(self, e: SuperExpr) -> None:
                     # Could be anything.
                     return
                 if isinstance(item, TupleType):
-                    item = item.fallback  # Handle named tuples and other Tuple[...] subclasses.
+                    # Handle named tuples and other Tuple[...] subclasses.
+                    item = tuple_fallback(item)
                 if not isinstance(item, Instance):
                     # A complicated type object type. Too tricky, give up.
                     # TODO: Do something more clever here.
@@ -2997,7 +3001,7 @@ def check_super_arguments(self, e: SuperExpr) -> None:
                         return
                 if isinstance(instance_type, TupleType):
                     # Needed for named tuples and other Tuple[...] subclasses.
-                    instance_type = instance_type.fallback
+                    instance_type = tuple_fallback(instance_type)
                 if type_info not in instance_type.type.mro:
                     self.chk.fail(message_registry.SUPER_ARG_2_NOT_INSTANCE_OF_ARG_1, e)
             elif isinstance(instance_type, TypeType) or (isinstance(instance_type, FunctionLike)
@@ -3287,7 +3291,9 @@ def has_member(self, typ: Type, member: str) -> bool:
         # these two should be carefully kept in sync.
         if isinstance(typ, TypeVarType):
             typ = typ.upper_bound
-        if isinstance(typ, (TupleType, LiteralType)):
+        if isinstance(typ, TupleType):
+            typ = tuple_fallback(typ)
+        if isinstance(typ, LiteralType):
             typ = typ.fallback
         if isinstance(typ, Instance):
             return typ.type.has_readable_member(member)
@@ -3305,7 +3311,7 @@ def has_member(self, typ: Type, member: str) -> bool:
             if isinstance(item, TypeVarType):
                 item = item.upper_bound
             if isinstance(item, TupleType):
-                item = item.fallback
+                item = tuple_fallback(item)
             if isinstance(item, Instance) and item.type.metaclass_type is not None:
                 return self.has_member(item.type.metaclass_type, member)
             if isinstance(item, AnyType):
@@ -3645,7 +3651,7 @@ def is_typetype_like(typ: Type) -> bool:
         if isinstance(actual, Overloaded):
             actual = actual.items()[0].fallback
         if isinstance(actual, TupleType):
-            actual = actual.fallback
+            actual = tuple_fallback(actual)
         if isinstance(actual, Instance) and formal.type in actual.type.mro:
             # Try performing a quick check as an optimization
             return True

mypy/checkmember.py

@@ -22,6 +22,7 @@
 from mypy import message_registry
 from mypy import subtypes
 from mypy import meet
+from mypy.typeops import tuple_fallback
 
 MYPY = False
 if MYPY:  # import for forward declaration only
@@ -122,7 +123,10 @@ def _analyze_member_access(name: str,
         return analyze_type_callable_member_access(name, typ, mx)
     elif isinstance(typ, TypeType):
         return analyze_type_type_member_access(name, typ, mx)
-    elif isinstance(typ, (TupleType, TypedDictType, LiteralType, FunctionLike)):
+    elif isinstance(typ, TupleType):
+        # Actually look up from the fallback instance type.
+        return _analyze_member_access(name, tuple_fallback(typ), mx)
+    elif isinstance(typ, (TypedDictType, LiteralType, FunctionLike)):
         # Actually look up from the fallback instance type.
         return _analyze_member_access(name, typ.fallback, mx)
     elif isinstance(typ, NoneTyp):
@@ -195,7 +199,7 @@ def analyze_type_callable_member_access(name: str,
     # TODO super?
     ret_type = typ.items()[0].ret_type
     if isinstance(ret_type, TupleType):
-        ret_type = ret_type.fallback
+        ret_type = tuple_fallback(ret_type)
     if isinstance(ret_type, Instance):
         if not mx.is_operator:
             # When Python sees an operator (eg `3 == 4`), it automatically translates that
@@ -235,7 +239,7 @@ def analyze_type_type_member_access(name: str, typ: TypeType, mx: MemberContext)
         if isinstance(typ.item.upper_bound, Instance):
             item = typ.item.upper_bound
     elif isinstance(typ.item, TupleType):
-        item = typ.item.fallback
+        item = tuple_fallback(typ.item)
     elif isinstance(typ.item, FunctionLike) and typ.item.is_type_obj():
         item = typ.item.fallback
     elif isinstance(typ.item, TypeType):
@@ -804,7 +808,7 @@ def map_type_from_supertype(typ: Type,
     # Create the type of self in subtype, of form t[a1, ...].
     inst_type = fill_typevars(sub_info)
     if isinstance(inst_type, TupleType):
-        inst_type = inst_type.fallback
+        inst_type = tuple_fallback(inst_type)
     # Map the type of self to supertype. This gets us a description of the
     # supertype type variables in terms of subtype variables, i.e. t[t1, ...]
     # so that any type variables in tN are to be interpreted in subtype

mypy/constraints.py

@@ -9,7 +9,8 @@
 )
 from mypy.maptype import map_instance_to_supertype
 import mypy.subtypes
-from mypy.sametypes import is_same_type
+import mypy.sametypes
+import mypy.typeops
 from mypy.erasetype import erase_typevars
 from mypy.nodes import COVARIANT, CONTRAVARIANT
 from mypy.argmap import ArgTypeExpander
@@ -199,7 +200,7 @@ def is_same_constraints(x: List[Constraint], y: List[Constraint]) -> bool:
 def is_same_constraint(c1: Constraint, c2: Constraint) -> bool:
     return (c1.type_var == c2.type_var
             and c1.op == c2.op
-            and is_same_type(c1.target, c2.target))
+            and mypy.sametypes.is_same_type(c1.target, c2.target))
 
 
 def simplify_away_incomplete_types(types: List[Type]) -> List[Type]:
@@ -282,7 +283,8 @@ def visit_instance(self, template: Instance) -> List[Constraint]:
         if isinstance(actual, (CallableType, Overloaded)) and template.type.is_protocol:
             if template.type.protocol_members == ['__call__']:
                 # Special case: a generic callback protocol
-                if not any(is_same_type(template, t) for t in template.type.inferring):
+                if not any(mypy.sametypes.is_same_type(template, t)
+                           for t in template.type.inferring):
                     template.type.inferring.append(template)
                     call = mypy.subtypes.find_member('__call__', template, actual)
                     assert call is not None
@@ -340,7 +342,8 @@ def visit_instance(self, template: Instance) -> List[Constraint]:
                     # Note that we use is_protocol_implementation instead of is_subtype
                     # because some type may be considered a subtype of a protocol
                     # due to _promote, but still not implement the protocol.
-                    not any(is_same_type(template, t) for t in template.type.inferring) and
+                    not any(mypy.sametypes.is_same_type(template, t)
+                            for t in template.type.inferring) and
                     mypy.subtypes.is_protocol_implementation(instance, erased)):
                 template.type.inferring.append(template)
                 self.infer_constraints_from_protocol_members(res, instance, template,
@@ -349,7 +352,8 @@ def visit_instance(self, template: Instance) -> List[Constraint]:
                 return res
             elif (instance.type.is_protocol and self.direction == SUBTYPE_OF and
                   # We avoid infinite recursion for structural subtypes also here.
-                  not any(is_same_type(instance, i) for i in instance.type.inferring) and
+                  not any(mypy.sametypes.is_same_type(instance, i)
+                          for i in instance.type.inferring) and
                   mypy.subtypes.is_protocol_implementation(erased, instance)):
                 instance.type.inferring.append(instance)
                 self.infer_constraints_from_protocol_members(res, instance, template,
@@ -370,7 +374,9 @@ def visit_instance(self, template: Instance) -> List[Constraint]:
                 res.extend(cb)
             return res
         elif isinstance(actual, TupleType) and self.direction == SUPERTYPE_OF:
-            return infer_constraints(template, actual.fallback, self.direction)
+            return infer_constraints(template,
+                                     mypy.typeops.tuple_fallback(actual),
+                                     self.direction)
         else:
             return []
 

mypy/erasetype.py

@@ -73,7 +73,7 @@ def visit_overloaded(self, t: Overloaded) -> Type:
         return t.fallback.accept(self)
 
     def visit_tuple_type(self, t: TupleType) -> Type:
-        return t.fallback.accept(self)
+        return t.partial_fallback.accept(self)
 
     def visit_typeddict_type(self, t: TypedDictType) -> Type:
         return t.fallback.accept(self)

mypy/fixup.py

@@ -199,8 +199,8 @@ def visit_tuple_type(self, tt: TupleType) -> None:
         if tt.items:
             for it in tt.items:
                 it.accept(self)
-        if tt.fallback is not None:
-            tt.fallback.accept(self)
+        if tt.partial_fallback is not None:
+            tt.partial_fallback.accept(self)
 
     def visit_typeddict_type(self, tdt: TypedDictType) -> None:
         if tdt.items:

mypy/indirection.py

@@ -85,7 +85,7 @@ def visit_overloaded(self, t: types.Overloaded) -> Set[str]:
         return self._visit(t.items()) | self._visit(t.fallback)
 
     def visit_tuple_type(self, t: types.TupleType) -> Set[str]:
-        return self._visit(t.items) | self._visit(t.fallback)
+        return self._visit(t.items) | self._visit(t.partial_fallback)
 
     def visit_typeddict_type(self, t: types.TypedDictType) -> Set[str]:
         return self._visit(t.items.values()) | self._visit(t.fallback)

mypy/join.py

@@ -14,6 +14,7 @@
     is_protocol_implementation, find_member
 )
 from mypy.nodes import ARG_NAMED, ARG_NAMED_OPT
+import mypy.typeops
 from mypy import state
 
 
@@ -243,7 +244,8 @@ def visit_tuple_type(self, t: TupleType) -> Type:
             items = []  # type: List[Type]
             for i in range(t.length()):
                 items.append(self.join(t.items[i], self.s.items[i]))
-            fallback = join_instances(self.s.fallback, t.fallback)
+            fallback = join_instances(mypy.typeops.tuple_fallback(self.s),
+                                      mypy.typeops.tuple_fallback(t))
             assert isinstance(fallback, Instance)
             return TupleType(items, fallback)
         else:
@@ -299,7 +301,7 @@ def default(self, typ: Type) -> Type:
         elif isinstance(typ, UnboundType):
             return AnyType(TypeOfAny.special_form)
         elif isinstance(typ, TupleType):
-            return self.default(typ.fallback)
+            return self.default(mypy.typeops.tuple_fallback(typ))
         elif isinstance(typ, TypedDictType):
             return self.default(typ.fallback)
         elif isinstance(typ, FunctionLike):

mypy/meet.py

@@ -15,6 +15,7 @@
 )
 from mypy.erasetype import erase_type
 from mypy.maptype import map_instance_to_supertype
+from mypy.typeops import tuple_fallback
 from mypy import state
 
 # TODO Describe this module.
@@ -211,9 +212,9 @@ def is_none_typevar_overlap(t1: Type, t2: Type) -> bool:
     if is_tuple(left) and is_tuple(right):
         return are_tuples_overlapping(left, right, ignore_promotions=ignore_promotions)
     elif isinstance(left, TupleType):
-        left = left.fallback
+        left = tuple_fallback(left)
     elif isinstance(right, TupleType):
-        right = right.fallback
+        right = tuple_fallback(right)
 
     # Next, we handle single-variant types that cannot be inherently partially overlapping,
     # but do require custom logic to inspect.
@@ -515,7 +516,7 @@ def visit_tuple_type(self, t: TupleType) -> Type:
             for i in range(t.length()):
                 items.append(self.meet(t.items[i], self.s.items[i]))
             # TODO: What if the fallbacks are different?
-            return TupleType(items, t.fallback)
+            return TupleType(items, tuple_fallback(t))
         elif isinstance(self.s, Instance):
             # meet(Tuple[t1, t2, <...>], Tuple[s, ...]) == Tuple[meet(t1, s), meet(t2, s), <...>].
             if self.s.type.fullname() == 'builtins.tuple' and self.s.args:

mypy/messages.py

@@ -208,8 +208,8 @@ def format_bare(self, typ: Type, verbosity: int = 0) -> str:
             return typ.name
         elif isinstance(typ, TupleType):
             # Prefer the name of the fallback class (if not tuple), as it's more informative.
-            if typ.fallback.type.fullname() != 'builtins.tuple':
-                return self.format_bare(typ.fallback)
+            if typ.partial_fallback.type.fullname() != 'builtins.tuple':
+                return self.format_bare(typ.partial_fallback)
             items = []
             for t in typ.items:
                 items.append(self.format_bare(t))
@@ -1225,7 +1225,11 @@ def report_protocol_problems(self, subtype: Union[Instance, TupleType, TypedDict
             # This will be only confusing a user even more.
             return
 
-        if isinstance(subtype, (TupleType, TypedDictType)):
+        if isinstance(subtype, TupleType):
+            if not isinstance(subtype.partial_fallback, Instance):
+                return
+            subtype = subtype.partial_fallback
+        elif isinstance(subtype, TypedDictType):
             if not isinstance(subtype.fallback, Instance):
                 return
             subtype = subtype.fallback