Overhaul overload semantics, remove erasure, add union math (#5084)

Overhaul overload semantics, remove erasure, add union math
This pull request:
* Modifies how mypy handles overloads to match the proposal I made in the typing repo.
* Starts removing type erasure from overload checks.
* Adds support for basic union math.
* Makes overloads respect keyword-only args

Author: Michael0x2a

mypy/checker.py

@@ -525,7 +525,9 @@ def find_matching_overload_item(overloaded: Overloaded, template: CallableType)
     for item in items:
         # Return type may be indeterminate in the template, so ignore it when performing a
         # subtype check.
-        if mypy.subtypes.is_callable_subtype(item, template, ignore_return=True):
+        if mypy.subtypes.is_callable_compatible(item, template,
+                                                is_compat=mypy.subtypes.is_subtype,
+                                                ignore_return=True):
             return item
     # Fall back to the first item if we can't find a match. This is totally arbitrary --
     # maybe we should just bail out at this point.

mypy/checkexpr.py

@@ -2,6 +2,7 @@
 from typing import List, Optional, cast, Tuple
 
 from mypy.join import is_similar_callables, combine_similar_callables, join_type_list
+from mypy.sametypes import is_same_type
 from mypy.types import (
     Type, AnyType, TypeVisitor, UnboundType, NoneTyp, TypeVarType, Instance, CallableType,
     TupleType, TypedDictType, ErasedType, TypeList, UnionType, PartialType, DeletedType,
@@ -49,6 +50,40 @@ def narrow_declared_type(declared: Type, narrowed: Type) -> Type:
     return narrowed
 
 
+def is_partially_overlapping_types(t: Type, s: Type) -> bool:
+    """Returns 'true' if the two types are partially, but not completely, overlapping.
+
+    NOTE: This function is only a partial implementation.
+
+    It exists mostly so that overloads correctly handle partial
+    overlaps for the more obvious cases.
+    """
+    # Are unions partially overlapping?
+    if isinstance(t, UnionType) and isinstance(s, UnionType):
+        t_set = set(t.items)
+        s_set = set(s.items)
+        num_same = len(t_set.intersection(s_set))
+        num_diff = len(t_set.symmetric_difference(s_set))
+        return num_same > 0 and num_diff > 0
+
+    # Are tuples partially overlapping?
+    tup_overlap = is_overlapping_tuples(t, s, use_promotions=True)
+    if tup_overlap is not None and tup_overlap:
+        return tup_overlap
+
+    def is_object(t: Type) -> bool:
+        return isinstance(t, Instance) and t.type.fullname() == 'builtins.object'
+
+    # Is either 't' or 's' an unrestricted TypeVar?
+    if isinstance(t, TypeVarType) and is_object(t.upper_bound) and len(t.values) == 0:
+        return True
+
+    if isinstance(s, TypeVarType) and is_object(s.upper_bound) and len(s.values) == 0:
+        return True
+
+    return False
+
+
 def is_overlapping_types(t: Type, s: Type, use_promotions: bool = False) -> bool:
     """Can a value of type t be a value of type s, or vice versa?
 

mypy/constraints.py

@@ -148,6 +148,11 @@ def copy(self) -> 'MessageBuilder':
         new.disable_type_names = self.disable_type_names
         return new
 
+    def clean_copy(self) -> 'MessageBuilder':
+        errors = self.errors.copy()
+        errors.error_info_map = OrderedDict()
+        return MessageBuilder(errors, self.modules)
+
     def add_errors(self, messages: 'MessageBuilder') -> None:
         """Add errors in messages to this builder."""
         if self.disable_count <= 0:
@@ -937,11 +942,19 @@ def incompatible_typevar_value(self,
                                                     self.format(typ)),
                   context)
 
-    def overloaded_signatures_overlap(self, index1: int, index2: int,
-                                      context: Context) -> None:
+    def overloaded_signatures_overlap(self, index1: int, index2: int, context: Context) -> None:
         self.fail('Overloaded function signatures {} and {} overlap with '
                   'incompatible return types'.format(index1, index2), context)
 
+    def overloaded_signature_will_never_match(self, index1: int, index2: int,
+                                              context: Context) -> None:
+        self.fail(
+            'Overloaded function signature {index2} will never be matched: '
+            'signature {index1}\'s parameter type(s) are the same or broader'.format(
+                index1=index1,
+                index2=index2),
+            context)
+
     def overloaded_signatures_arg_specific(self, index1: int, context: Context) -> None:
         self.fail('Overloaded function implementation does not accept all possible arguments '
                   'of signature {}'.format(index1), context)

mypy/meet.py

@@ -203,8 +203,9 @@ def visit_type_var(self, left: TypeVarType) -> bool:
     def visit_callable_type(self, left: CallableType) -> bool:
         right = self.right
         if isinstance(right, CallableType):
-            return is_callable_subtype(
+            return is_callable_compatible(
                 left, right,
+                is_compat=is_subtype,
                 ignore_pos_arg_names=self.ignore_pos_arg_names)
         elif isinstance(right, Overloaded):
             return all(is_subtype(left, item, self.check_type_parameter,
@@ -310,10 +311,12 @@ def visit_overloaded(self, left: Overloaded) -> bool:
                     else:
                         # If this one overlaps with the supertype in any way, but it wasn't
                         # an exact match, then it's a potential error.
-                        if (is_callable_subtype(left_item, right_item, ignore_return=True,
-                                            ignore_pos_arg_names=self.ignore_pos_arg_names) or
-                                is_callable_subtype(right_item, left_item, ignore_return=True,
-                                                ignore_pos_arg_names=self.ignore_pos_arg_names)):
+                        if (is_callable_compatible(left_item, right_item,
+                                    is_compat=is_subtype, ignore_return=True,
+                                    ignore_pos_arg_names=self.ignore_pos_arg_names) or
+                                is_callable_compatible(right_item, left_item,
+                                        is_compat=is_subtype, ignore_return=True,
+                                        ignore_pos_arg_names=self.ignore_pos_arg_names)):
                             # If this is an overload that's already been matched, there's no
                             # problem.
                             if left_item not in matched_overloads:
@@ -568,16 +571,54 @@ def non_method_protocol_members(tp: TypeInfo) -> List[str]:
     return result
 
 
-def is_callable_subtype(left: CallableType, right: CallableType,
-                        ignore_return: bool = False,
-                        ignore_pos_arg_names: bool = False,
-                        use_proper_subtype: bool = False) -> bool:
-    """Is left a subtype of right?"""
+def is_callable_compatible(left: CallableType, right: CallableType,
+                           *,
+                           is_compat: Callable[[Type, Type], bool],
+                           is_compat_return: Optional[Callable[[Type, Type], bool]] = None,
+                           ignore_return: bool = False,
+                           ignore_pos_arg_names: bool = False,
+                           check_args_covariantly: bool = False) -> bool:
+    """Is the left compatible with the right, using the provided compatibility check?
 
-    if use_proper_subtype:
-        is_compat = is_proper_subtype
-    else:
-        is_compat = is_subtype
+    is_compat:
+        The check we want to run against the parameters.
+
+    is_compat_return:
+        The check we want to run against the return type.
+        If None, use the 'is_compat' check.
+
+    check_args_covariantly:
+        If true, check if the left's args is compatible with the right's
+        instead of the other way around (contravariantly).
+
+        This function is mostly used to check if the left is a subtype of the right which
+        is why the default is to check the args contravariantly. However, it's occasionally
+        useful to check the args using some other check, so we leave the variance
+        configurable.
+
+        For example, when checking the validity of overloads, it's useful to see if
+        the first overload alternative has more precise arguments then the second.
+        We would want to check the arguments covariantly in that case.
+
+        Note! The following two function calls are NOT equivalent:
+
+            is_callable_compatible(f, g, is_compat=is_subtype, check_args_covariantly=False)
+            is_callable_compatible(g, f, is_compat=is_subtype, check_args_covariantly=True)
+
+        The two calls are similar in that they both check the function arguments in
+        the same direction: they both run `is_subtype(argument_from_g, argument_from_f)`.
+
+        However, the two calls differ in which direction they check things likee
+        keyword arguments. For example, suppose f and g are defined like so:
+
+            def f(x: int, *y: int) -> int: ...
+            def g(x: int) -> int: ...
+
+        In this case, the first call will succeed and the second will fail: f is a
+        valid stand-in for g but not vice-versa.
+    """
+    if is_compat_return is None:
+        is_compat_return = is_compat
 
     # If either function is implicitly typed, ignore positional arg names too
     if left.implicit or right.implicit:
@@ -607,9 +648,12 @@ def is_callable_subtype(left: CallableType, right: CallableType,
             left = unified
 
     # Check return types.
-    if not ignore_return and not is_compat(left.ret_type, right.ret_type):
+    if not ignore_return and not is_compat_return(left.ret_type, right.ret_type):
         return False
 
+    if check_args_covariantly:
+        is_compat = flip_compat_check(is_compat)
+
     if right.is_ellipsis_args:
         return True
 
@@ -652,7 +696,7 @@ def is_callable_subtype(left: CallableType, right: CallableType,
             # Right has an infinite series of optional positional arguments
             # here.  Get all further positional arguments of left, and make sure
             # they're more general than their corresponding member in this
-            # series.  Also make sure left has its own inifite series of
+            # series.  Also make sure left has its own infinite series of
             # optional positional arguments.
             if not left.is_var_arg:
                 return False
@@ -664,7 +708,7 @@ def is_callable_subtype(left: CallableType, right: CallableType,
                 right_by_position = right.argument_by_position(j)
                 assert right_by_position is not None
                 if not are_args_compatible(left_by_position, right_by_position,
-                                           ignore_pos_arg_names, use_proper_subtype):
+                                           ignore_pos_arg_names, is_compat):
                     return False
                 j += 1
             continue
@@ -687,7 +731,7 @@ def is_callable_subtype(left: CallableType, right: CallableType,
                 right_by_name = right.argument_by_name(name)
                 assert right_by_name is not None
                 if not are_args_compatible(left_by_name, right_by_name,
-                                           ignore_pos_arg_names, use_proper_subtype):
+                                           ignore_pos_arg_names, is_compat):
                     return False
             continue
 
@@ -696,7 +740,8 @@ def is_callable_subtype(left: CallableType, right: CallableType,
         if left_arg is None:
             return False
 
-        if not are_args_compatible(left_arg, right_arg, ignore_pos_arg_names, use_proper_subtype):
+        if not are_args_compatible(left_arg, right_arg,
+                                   ignore_pos_arg_names, is_compat):
             return False
 
     done_with_positional = False
@@ -748,7 +793,7 @@ def are_args_compatible(
         left: FormalArgument,
         right: FormalArgument,
         ignore_pos_arg_names: bool,
-        use_proper_subtype: bool) -> bool:
+        is_compat: Callable[[Type, Type], bool]) -> bool:
     # If right has a specific name it wants this argument to be, left must
     # have the same.
     if right.name is not None and left.name != right.name:
@@ -759,18 +804,20 @@ def are_args_compatible(
     if right.pos is not None and left.pos != right.pos:
         return False
     # Left must have a more general type
-    if use_proper_subtype:
-        if not is_proper_subtype(right.typ, left.typ):
-            return False
-    else:
-        if not is_subtype(right.typ, left.typ):
-            return False
+    if not is_compat(right.typ, left.typ):
+        return False
     # If right's argument is optional, left's must also be.
     if not right.required and left.required:
         return False
     return True
 
 
+def flip_compat_check(is_compat: Callable[[Type, Type], bool]) -> Callable[[Type, Type], bool]:
+    def new_is_compat(left: Type, right: Type) -> bool:
+        return is_compat(right, left)
+    return new_is_compat
+
+
 def unify_generic_callable(type: CallableType, target: CallableType,
                            ignore_return: bool) -> Optional[CallableType]:
     """Try to unify a generic callable type with another callable type.
@@ -913,10 +960,7 @@ def visit_type_var(self, left: TypeVarType) -> bool:
     def visit_callable_type(self, left: CallableType) -> bool:
         right = self.right
         if isinstance(right, CallableType):
-            return is_callable_subtype(
-                left, right,
-                ignore_pos_arg_names=False,
-                use_proper_subtype=True)
+            return is_callable_compatible(left, right, is_compat=is_proper_subtype)
         elif isinstance(right, Overloaded):
             return all(is_proper_subtype(left, item)
                        for item in right.items())

mypy/messages.py

@@ -1,5 +1,6 @@
 """Classes for representing mypy types."""
 
+import sys
 import copy
 from abc import abstractmethod
 from collections import OrderedDict
@@ -750,6 +751,7 @@ def copy_modified(self,
                       line: int = _dummy,
                       column: int = _dummy,
                       is_ellipsis_args: bool = _dummy,
+                      implicit: bool = _dummy,
                       special_sig: Optional[str] = _dummy,
                       from_type_type: bool = _dummy,
                       bound_args: List[Optional[Type]] = _dummy,
@@ -767,7 +769,7 @@ def copy_modified(self,
             column=column if column is not _dummy else self.column,
             is_ellipsis_args=(
                 is_ellipsis_args if is_ellipsis_args is not _dummy else self.is_ellipsis_args),
-            implicit=self.implicit,
+            implicit=implicit if implicit is not _dummy else self.implicit,
             is_classmethod_class=self.is_classmethod_class,
             special_sig=special_sig if special_sig is not _dummy else self.special_sig,
             from_type_type=from_type_type if from_type_type is not _dummy else self.from_type_type,
@@ -807,6 +809,15 @@ def max_fixed_args(self) -> int:
             n -= 1
         return n
 
+    def max_possible_positional_args(self) -> int:
+        """Returns maximum number of positional arguments this method could possibly accept.
+
+        This takes into acount *arg and **kwargs but excludes keyword-only args."""
+        if self.is_var_arg or self.is_kw_arg:
+            return sys.maxsize
+        blacklist = (ARG_NAMED, ARG_NAMED_OPT)
+        return len([kind not in blacklist for kind in self.arg_kinds])
+
     def corresponding_argument(self, model: FormalArgument) -> Optional[FormalArgument]:
         """Return the argument in this function that corresponds to `model`"""