From 14fa27f167d23fab76942e8d943daae6100e225b Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Thu, 7 Nov 2019 19:05:38 -0800 Subject: [PATCH 1/8] Refine parent type when narrowing "lookup" expressions This diff adds support for the following pattern: ```python from typing import Enum, List from enum import Enum class Key(Enum): A = 1 B = 2 class Foo: key: Literal[Key.A] blah: List[int] class Bar: key: Literal[Key.B] something: List[str] x: Union[Foo, Bar] if x.key is Key.A: reveal_type(x) # Revealed type is 'Foo' else: reveal_type(x) # Revealed type is 'Bar' ``` In short, when we do `x.key is Key.A`, we "propagate" the information we discovered about `x.key` up one level to refine the type of `x`. We perform this propagation only when `x` is a Union and only when we are doing member or index lookups into instances, typeddicts, namedtuples, and tuples. For indexing operations, we have one additional limitation: we *must* use a literal expression in order for narrowing to work at all. Using Literal types or Final instances won't work; See https://github.com/python/mypy/issues/7905 for more details. To put it another way, this adds support for tagged unions, I guess. This more or less resolves https://github.com/python/mypy/issues/7344. We currently don't have support for narrowing based on string or int literals, but that's a separate issue and should be resolved by https://github.com/python/mypy/pull/7169 (which I resumed work on earlier this week). --- mypy/checker.py | 184 +++++++++++++++- mypy/checkexpr.py | 3 + mypy/test/testcheck.py | 1 + mypy/typeops.py | 78 ++++++- test-data/unit/check-narrowing.test | 315 ++++++++++++++++++++++++++++ test-data/unit/fixtures/tuple.pyi | 3 +- 6 files changed, 562 insertions(+), 22 deletions(-) create mode 100644 test-data/unit/check-narrowing.test diff --git a/mypy/checker.py b/mypy/checker.py index 557ceb8a71c0..d94e5fe7993b 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -6,7 +6,7 @@ from typing import ( Dict, Set, List, cast, Tuple, TypeVar, Union, Optional, NamedTuple, Iterator, Iterable, - Sequence + Mapping, Sequence ) from typing_extensions import Final @@ -43,11 +43,13 @@ ) import mypy.checkexpr from mypy.checkmember import ( - analyze_descriptor_access, type_object_type, + analyze_member_access, analyze_descriptor_access, type_object_type, ) from mypy.typeops import ( map_type_from_supertype, bind_self, erase_to_bound, make_simplified_union, - erase_def_to_union_or_bound, erase_to_union_or_bound, + erase_def_to_union_or_bound, erase_to_union_or_bound, coerce_to_literal, + try_getting_str_literals_from_type, try_getting_int_literals_from_type, + tuple_fallback, is_singleton_type, try_expanding_enum_to_union, true_only, false_only, function_type, ) from mypy import message_registry @@ -72,9 +74,6 @@ from mypy.plugin import Plugin, CheckerPluginInterface from mypy.sharedparse import BINARY_MAGIC_METHODS from mypy.scope import Scope -from mypy.typeops import ( - tuple_fallback, coerce_to_literal, is_singleton_type, try_expanding_enum_to_union -) from mypy import state, errorcodes as codes from mypy.traverser import has_return_statement, all_return_statements from mypy.errorcodes import ErrorCode @@ -3709,6 +3708,12 @@ def find_isinstance_check(self, node: Expression Guaranteed to not return None, None. (But may return {}, {}) """ + if_map, else_map = self.find_isinstance_check_helper(node) + new_if_map = self.propagate_up_typemap_info(self.type_map, if_map) + new_else_map = self.propagate_up_typemap_info(self.type_map, else_map) + return new_if_map, new_else_map + + def find_isinstance_check_helper(self, node: Expression) -> Tuple[TypeMap, TypeMap]: type_map = self.type_map if is_true_literal(node): return {}, None @@ -3835,28 +3840,185 @@ def find_isinstance_check(self, node: Expression else None) return if_map, else_map elif isinstance(node, OpExpr) and node.op == 'and': - left_if_vars, left_else_vars = self.find_isinstance_check(node.left) - right_if_vars, right_else_vars = self.find_isinstance_check(node.right) + left_if_vars, left_else_vars = self.find_isinstance_check_helper(node.left) + right_if_vars, right_else_vars = self.find_isinstance_check_helper(node.right) # (e1 and e2) is true if both e1 and e2 are true, # and false if at least one of e1 and e2 is false. return (and_conditional_maps(left_if_vars, right_if_vars), or_conditional_maps(left_else_vars, right_else_vars)) elif isinstance(node, OpExpr) and node.op == 'or': - left_if_vars, left_else_vars = self.find_isinstance_check(node.left) - right_if_vars, right_else_vars = self.find_isinstance_check(node.right) + left_if_vars, left_else_vars = self.find_isinstance_check_helper(node.left) + right_if_vars, right_else_vars = self.find_isinstance_check_helper(node.right) # (e1 or e2) is true if at least one of e1 or e2 is true, # and false if both e1 and e2 are false. return (or_conditional_maps(left_if_vars, right_if_vars), and_conditional_maps(left_else_vars, right_else_vars)) elif isinstance(node, UnaryExpr) and node.op == 'not': - left, right = self.find_isinstance_check(node.expr) + left, right = self.find_isinstance_check_helper(node.expr) return right, left # Not a supported isinstance check return {}, {} + def propagate_up_typemap_info(self, + existing_types: Mapping[Expression, Type], + new_types: TypeMap) -> TypeMap: + """Attempts refining parent expressions of any MemberExpr or IndexExprs in new_types. + + Specifically, this function accepts two mappings of expression to original types: + the original mapping (existing_types), and a new mapping (new_types) intended to + update the original. + + This function iterates through new_types and attempts to use the information to try + refining any parent types that happen to be unions. + + For example, suppose there are two types "A = Tuple[int, int]" and "B = Tuple[str, str]". + Next, suppose that 'new_types' specifies the expression 'foo[0]' has a refined type + of 'int' and that 'foo' was previously deduced to be of type Union[A, B]. + + Then, this function will observe that since A[0] is an int and B[0] is not, the type of + 'foo' can be further refined from Union[A, B] into just B. + + We perform this kind of "parent narrowing" for member lookup expressions and indexing + expressions into tuples, namedtuples, and typeddicts. This narrowing is also performed + only once, for the immediate parents of any "lookup" expressions in `new_types`. + + We return the newly refined map. This map is guaranteed to be a superset of 'new_types'. + """ + if new_types is None: + return None + output_map = {} + for expr, expr_type in new_types.items(): + # The original inferred type should always be present in the output map, of course + output_map[expr] = expr_type + + # Next, try using this information to refine the parent type, if applicable. + # Note that we currently refine just the immediate parent. + # + # TODO: Should we also try recursively refining any parents of the parents? + # + # One quick-and-dirty way of doing this would be to have the caller repeatedly run + # this function until we reach fixpoint; another way would be to modify + # 'refine_parent_type' to run in a loop. Both approaches seem expensive though. + new_mapping = self.refine_parent_type(existing_types, expr, expr_type) + for parent_expr, proposed_parent_type in new_mapping.items(): + # We don't try inferring anything if we've already inferred something for + # the parent expression. + # TODO: Consider picking the narrower type instead of always discarding this? + if parent_expr in new_types: + continue + output_map[parent_expr] = proposed_parent_type + return output_map + + def refine_parent_type(self, + existing_types: Mapping[Expression, Type], + expr: Expression, + expr_type: Type) -> Mapping[Expression, Type]: + """Checks if the given expr is a 'lookup operation' into a union and refines the parent type + based on the 'expr_type'. + + For more details about what a 'lookup operation' is and how we use the expr_type to refine + the parent type, see the docstring in 'propagate_up_typemap_info'. + """ + + # First, check if this expression is one that's attempting to + # "lookup" some key in the parent type. If so, save the parent type + # and create function that will try replaying the same lookup + # operation against arbitrary types. + if isinstance(expr, MemberExpr): + parent_expr = expr.expr + parent_type = existing_types.get(parent_expr) + member_name = expr.name + + def replay_lookup(new_parent_type: ProperType) -> Optional[Type]: + msg_copy = self.msg.clean_copy() + msg_copy.disable_count = 0 + member_type = analyze_member_access( + name=member_name, + typ=new_parent_type, + context=parent_expr, + is_lvalue=False, + is_super=False, + is_operator=False, + msg=msg_copy, + original_type=new_parent_type, + chk=self, + in_literal_context=False, + ) + if msg_copy.is_errors(): + return None + else: + return member_type + elif isinstance(expr, IndexExpr): + parent_expr = expr.base + parent_type = existing_types.get(parent_expr) + + index_type = existing_types.get(expr.index) + if index_type is None: + return {} + + str_literals = try_getting_str_literals_from_type(index_type) + if str_literals is not None: + def replay_lookup(new_parent_type: ProperType) -> Optional[Type]: + if not isinstance(new_parent_type, TypedDictType): + return None + try: + assert str_literals is not None + member_types = [new_parent_type.items[key] for key in str_literals] + except KeyError: + return None + return make_simplified_union(member_types) + else: + int_literals = try_getting_int_literals_from_type(index_type) + if int_literals is not None: + def replay_lookup(new_parent_type: ProperType) -> Optional[Type]: + if not isinstance(new_parent_type, TupleType): + return None + try: + assert int_literals is not None + member_types = [new_parent_type.items[key] for key in int_literals] + except IndexError: + return None + return make_simplified_union(member_types) + else: + return {} + else: + return {} + + # If we somehow didn't previously derive the parent type, abort: + # something went wrong at an earlier stage. + if parent_type is None: + return {} + + # We currently only try refining the parent type if it's a Union. + parent_type = get_proper_type(parent_type) + if not isinstance(parent_type, UnionType): + return {} + + # Take each element in the parent union and replay the original lookup procedure + # to figure out which parents are compatible. + new_parent_types = [] + for item in parent_type.items: + item = get_proper_type(item) + member_type = replay_lookup(item) + if member_type is None: + # We were unable to obtain the member type. So, we give up on refining this + # parent type entirely. + return {} + + if is_overlapping_types(member_type, expr_type): + new_parent_types.append(item) + + # If none of the parent types overlap (if we derived an empty union), something + # went wrong. We should never hit this case, but deriving the uninhabited type or + # reporting an error both seem unhelpful. So we abort. + if not new_parent_types: + return {} + + return {parent_expr: make_simplified_union(new_parent_types)} + # # Helpers # diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index b86801e25f1b..a13c5d11809e 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -2704,6 +2704,9 @@ def visit_index_with_type(self, left_type: Type, e: IndexExpr, index = e.index left_type = get_proper_type(left_type) + # Visit the index, just to make sure we have a type for it available + self.accept(index) + if isinstance(left_type, UnionType): original_type = original_type or left_type return make_simplified_union([self.visit_index_with_type(typ, e, diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 5fd5405ec4e8..2747d1c034d1 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -46,6 +46,7 @@ 'check-isinstance.test', 'check-lists.test', 'check-namedtuple.test', + 'check-narrowing.test', 'check-typeddict.test', 'check-type-aliases.test', 'check-ignore.test', diff --git a/mypy/typeops.py b/mypy/typeops.py index b26aa8b3ea73..aa31fadc6c3e 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -5,17 +5,17 @@ since these may assume that MROs are ready. """ -from typing import cast, Optional, List, Sequence, Set +from typing import cast, Optional, List, Sequence, Set, TypeVar, Type as TypingType import sys from mypy.types import ( TupleType, Instance, FunctionLike, Type, CallableType, TypeVarDef, Overloaded, - TypeVarType, UninhabitedType, FormalArgument, UnionType, NoneType, + TypeVarType, UninhabitedType, FormalArgument, UnionType, NoneType, TypedDictType, AnyType, TypeOfAny, TypeType, ProperType, LiteralType, get_proper_type, get_proper_types, copy_type, TypeAliasType ) from mypy.nodes import ( - FuncBase, FuncItem, OverloadedFuncDef, TypeInfo, TypeVar, ARG_STAR, ARG_STAR2, ARG_POS, + FuncBase, FuncItem, OverloadedFuncDef, TypeInfo, ARG_STAR, ARG_STAR2, ARG_POS, Expression, StrExpr, Var ) from mypy.maptype import map_instance_to_supertype @@ -43,6 +43,25 @@ def tuple_fallback(typ: TupleType) -> Instance: return Instance(info, [join_type_list(typ.items)]) +def try_getting_instance_fallback(typ: ProperType) -> Optional[Instance]: + """Returns the Instance fallback for this type if one exists. + + Otherwise, returns None. + """ + if isinstance(typ, Instance): + return typ + elif isinstance(typ, TupleType): + return tuple_fallback(typ) + elif isinstance(typ, TypedDictType): + return typ.fallback + elif isinstance(typ, FunctionLike): + return typ.fallback + elif isinstance(typ, LiteralType): + return typ.fallback + else: + return None + + def type_object_type_from_function(signature: FunctionLike, info: TypeInfo, def_info: TypeInfo, @@ -475,11 +494,48 @@ def try_getting_str_literals(expr: Expression, typ: Type) -> Optional[List[str]] 2. 'typ' is a LiteralType containing a string 3. 'typ' is a UnionType containing only LiteralType of strings """ - typ = get_proper_type(typ) - if isinstance(expr, StrExpr): return [expr.value] + # TODO: See if we can eliminate this function and call the below one directly + return try_getting_str_literals_from_type(typ) + + +def try_getting_str_literals_from_type(typ: Type) -> Optional[List[str]]: + """If the given expression or type corresponds to a string Literal + or a union of string Literals, returns a list of the underlying strings. + Otherwise, returns None. + + For example, if we had the type 'Literal["foo", "bar"]' as input, this function + would return a list of strings ["foo", "bar"]. + """ + return try_getting_literals_from_type(typ, str, "builtins.str") + + +def try_getting_int_literals_from_type(typ: Type) -> Optional[List[int]]: + """If the given expression or type corresponds to an int Literal + or a union of int Literals, returns a list of the underlying ints. + Otherwise, returns None. + + For example, if we had the type 'Literal[1, 2, 3]' as input, this function + would return a list of ints [1, 2, 3]. + """ + return try_getting_literals_from_type(typ, int, "builtins.int") + + +T = TypeVar('T') + + +def try_getting_literals_from_type(typ: Type, + target_literal_type: TypingType[T], + target_fullname: str) -> Optional[List[T]]: + """If the given expression or type corresponds to a Literal or + union of Literals where the underlying values corresponds to the given + target type, returns a list of those underlying values. Otherwise, + returns None. + """ + typ = get_proper_type(typ) + if isinstance(typ, Instance) and typ.last_known_value is not None: possible_literals = [typ.last_known_value] # type: List[Type] elif isinstance(typ, UnionType): @@ -487,15 +543,17 @@ def try_getting_str_literals(expr: Expression, typ: Type) -> Optional[List[str]] else: possible_literals = [typ] - strings = [] + literals = [] # type: List[T] for lit in get_proper_types(possible_literals): - if isinstance(lit, LiteralType) and lit.fallback.type.fullname() == 'builtins.str': + if isinstance(lit, LiteralType) and lit.fallback.type.fullname() == target_fullname: val = lit.value - assert isinstance(val, str) - strings.append(val) + if isinstance(val, target_literal_type): + literals.append(val) + else: + return None else: return None - return strings + return literals def get_enum_values(typ: Instance) -> List[str]: diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test new file mode 100644 index 000000000000..eb1a8fe48aca --- /dev/null +++ b/test-data/unit/check-narrowing.test @@ -0,0 +1,315 @@ +[case testNarrowingParentWithEnumsBasic] +from enum import Enum +from dataclasses import dataclass +from typing import NamedTuple, Tuple, Union +from typing_extensions import Literal, TypedDict + +class Key(Enum): + A = 1 + B = 2 + C = 3 + +class Object1: + key: Literal[Key.A] + foo: int +class Object2: + key: Literal[Key.B] + bar: str + +@dataclass +class Dataclass1: + key: Literal[Key.A] + foo: int +@dataclass +class Dataclass2: + key: Literal[Key.B] + foo: str + +class NamedTuple1(NamedTuple): + key: Literal[Key.A] + foo: int +class NamedTuple2(NamedTuple): + key: Literal[Key.B] + foo: str + +Tuple1 = Tuple[Literal[Key.A], int] +Tuple2 = Tuple[Literal[Key.B], str] + +x1: Union[Object1, Object2] +if x1.key is Key.A: + reveal_type(x1) # N: Revealed type is '__main__.Object1' + reveal_type(x1.key) # N: Revealed type is 'Literal[__main__.Key.A]' +else: + reveal_type(x1) # N: Revealed type is '__main__.Object2' + reveal_type(x1.key) # N: Revealed type is 'Literal[__main__.Key.B]' + +x2: Union[Dataclass1, Dataclass2] +if x2.key is Key.A: + reveal_type(x2) # N: Revealed type is '__main__.Dataclass1' + reveal_type(x2.key) # N: Revealed type is 'Literal[__main__.Key.A]' +else: + reveal_type(x2) # N: Revealed type is '__main__.Dataclass2' + reveal_type(x2.key) # N: Revealed type is 'Literal[__main__.Key.B]' + +x3: Union[NamedTuple1, NamedTuple2] +if x3.key is Key.A: + reveal_type(x3) # N: Revealed type is 'Tuple[Literal[__main__.Key.A], builtins.int, fallback=__main__.NamedTuple1]' + reveal_type(x3.key) # N: Revealed type is 'Literal[__main__.Key.A]' +else: + reveal_type(x3) # N: Revealed type is 'Tuple[Literal[__main__.Key.B], builtins.str, fallback=__main__.NamedTuple2]' + reveal_type(x3.key) # N: Revealed type is 'Literal[__main__.Key.B]' +if x3[0] is Key.A: + reveal_type(x3) # N: Revealed type is 'Tuple[Literal[__main__.Key.A], builtins.int, fallback=__main__.NamedTuple1]' + reveal_type(x3[0]) # N: Revealed type is 'Literal[__main__.Key.A]' +else: + reveal_type(x3) # N: Revealed type is 'Tuple[Literal[__main__.Key.B], builtins.str, fallback=__main__.NamedTuple2]' + reveal_type(x3[0]) # N: Revealed type is 'Literal[__main__.Key.B]' + +x4: Union[Tuple1, Tuple2] +if x4[0] is Key.A: + reveal_type(x4) # N: Revealed type is 'Tuple[Literal[__main__.Key.A], builtins.int]' + reveal_type(x4[0]) # N: Revealed type is 'Literal[__main__.Key.A]' +else: + reveal_type(x4) # N: Revealed type is 'Tuple[Literal[__main__.Key.B], builtins.str]' + reveal_type(x4[0]) # N: Revealed type is 'Literal[__main__.Key.B]' + +[case testNarrowingParentWithIsInstanceBasic] +from dataclasses import dataclass +from typing import NamedTuple, Tuple, Union +from typing_extensions import TypedDict + +class Object1: + key: int +class Object2: + key: str + +@dataclass +class Dataclass1: + key: int +@dataclass +class Dataclass2: + key: str + +class NamedTuple1(NamedTuple): + key: int +class NamedTuple2(NamedTuple): + key: str + +Tuple1 = Tuple[int] +Tuple2 = Tuple[str] + +x1: Union[Object1, Object2] +if isinstance(x1.key, int): + reveal_type(x1) # N: Revealed type is '__main__.Object1' +else: + reveal_type(x1) # N: Revealed type is '__main__.Object2' + +x2: Union[Dataclass1, Dataclass2] +if isinstance(x2.key, int): + reveal_type(x2) # N: Revealed type is '__main__.Dataclass1' +else: + reveal_type(x2) # N: Revealed type is '__main__.Dataclass2' + +x3: Union[NamedTuple1, NamedTuple2] +if isinstance(x3.key, int): + reveal_type(x3) # N: Revealed type is 'Tuple[builtins.int, fallback=__main__.NamedTuple1]' +else: + reveal_type(x3) # N: Revealed type is 'Tuple[builtins.str, fallback=__main__.NamedTuple2]' +if isinstance(x3[0], int): + reveal_type(x3) # N: Revealed type is 'Tuple[builtins.int, fallback=__main__.NamedTuple1]' +else: + reveal_type(x3) # N: Revealed type is 'Tuple[builtins.str, fallback=__main__.NamedTuple2]' + +x4: Union[Tuple1, Tuple2] +if isinstance(x4[0], int): + reveal_type(x4) # N: Revealed type is 'Tuple[builtins.int]' +else: + reveal_type(x4) # N: Revealed type is 'Tuple[builtins.str]' +[builtins fixtures/isinstance.pyi] + +[case testNarrowingParentMultipleKeys] +# flags: --warn-unreachable +from enum import Enum +from typing import Union +from typing_extensions import Literal + +class Key(Enum): + A = 1 + B = 2 + C = 3 + D = 4 + +class Object1: + key: Literal[Key.A, Key.C] +class Object2: + key: Literal[Key.B, Key.C] + +x: Union[Object1, Object2] +if x.key is Key.A: + reveal_type(x) # N: Revealed type is '__main__.Object1' +else: + reveal_type(x) # N: Revealed type is 'Union[__main__.Object1, __main__.Object2]' + +if x.key is Key.C: + reveal_type(x) # N: Revealed type is 'Union[__main__.Object1, __main__.Object2]' +else: + reveal_type(x) # N: Revealed type is 'Union[__main__.Object1, __main__.Object2]' + +if x.key is Key.D: + reveal_type(x) # E: Statement is unreachable +else: + reveal_type(x) # N: Revealed type is 'Union[__main__.Object1, __main__.Object2]' + +[case testNarrowingParentWithMultipleParents] +from enum import Enum +from typing import Union +from typing_extensions import Literal + +class Key(Enum): + A = 1 + B = 2 + C = 3 + +class Object1: + key: Literal[Key.A] +class Object2: + key: Literal[Key.B] +class Object3: + key: Literal[Key.C] +class Object4: + key: str + +x: Union[Object1, Object2, Object3, Object4] +if x.key is Key.A: + reveal_type(x) # N: Revealed type is '__main__.Object1' +else: + reveal_type(x) # N: Revealed type is 'Union[__main__.Object2, __main__.Object3, __main__.Object4]' + +if isinstance(x.key, str): + reveal_type(x) # N: Revealed type is '__main__.Object4' +else: + reveal_type(x) # N: Revealed type is 'Union[__main__.Object1, __main__.Object2, __main__.Object3]' +[builtins fixtures/isinstance.pyi] + +[case testNarrowingParentsWithGenerics] +from typing import Union, TypeVar, Generic + +T = TypeVar('T') +class Wrapper(Generic[T]): + key: T + +x: Union[Wrapper[int], Wrapper[str]] +if isinstance(x.key, int): + reveal_type(x) # N: Revealed type is '__main__.Wrapper[builtins.int]' +else: + reveal_type(x) # N: Revealed type is '__main__.Wrapper[builtins.str]' +[builtins fixtures/isinstance.pyi] + +[case testNarrowingParentWithParentMixtures] +from enum import Enum +from typing import Union, NamedTuple +from typing_extensions import Literal, TypedDict + +class Key(Enum): + A = 1 + B = 2 + C = 3 + +class KeyedObject: + key: Literal[Key.A] +class KeyedTypedDict(TypedDict): + key: Literal[Key.B] +class KeyedNamedTuple(NamedTuple): + key: Literal[Key.C] + +ok_mixture: Union[KeyedObject, KeyedNamedTuple] +if ok_mixture.key is Key.A: + reveal_type(ok_mixture) # N: Revealed type is '__main__.KeyedObject' +else: + reveal_type(ok_mixture) # N: Revealed type is 'Tuple[Literal[__main__.Key.C], fallback=__main__.KeyedNamedTuple]' + +# Each reveal_type below really ought to be a union -- e.g. we ought short-circuit +# and skip inferring anything about the parents. Currently, we overreach in some +# cases and infer something a bit non-sensical due to how we're normalizing +# "lookup" operations. +# +# This is a bit confusing from a usability standpoint, but is probably fine: +# we don't guarantee sensible results after errors anyways. (And making sure +# these nonsensical lookups result in an error is the main purpose of this +# test case). + +impossible_mixture: Union[KeyedObject, KeyedTypedDict] +if impossible_mixture.key is Key.A: # E: Item "KeyedTypedDict" of "Union[KeyedObject, KeyedTypedDict]" has no attribute "key" + reveal_type(impossible_mixture) # N: Revealed type is 'Union[__main__.KeyedObject, TypedDict('__main__.KeyedTypedDict', {'key': Literal[__main__.Key.B]})]' +else: + reveal_type(impossible_mixture) # N: Revealed type is 'Union[__main__.KeyedObject, TypedDict('__main__.KeyedTypedDict', {'key': Literal[__main__.Key.B]})]' + +if impossible_mixture["key"] is Key.A: # E: Value of type "Union[KeyedObject, KeyedTypedDict]" is not indexable + reveal_type(impossible_mixture) # N: Revealed type is 'Union[__main__.KeyedObject, TypedDict('__main__.KeyedTypedDict', {'key': Literal[__main__.Key.B]})]' +else: + reveal_type(impossible_mixture) # N: Revealed type is 'Union[__main__.KeyedObject, TypedDict('__main__.KeyedTypedDict', {'key': Literal[__main__.Key.B]})]' + +weird_mixture: Union[KeyedTypedDict, KeyedNamedTuple] +if weird_mixture["key"] is Key.B: # E: Invalid tuple index type (actual type "str", expected type "Union[int, slice]") + reveal_type(weird_mixture) # N: Revealed type is 'Union[TypedDict('__main__.KeyedTypedDict', {'key': Literal[__main__.Key.B]}), Tuple[Literal[__main__.Key.C], fallback=__main__.KeyedNamedTuple]]' +else: + reveal_type(weird_mixture) # N: Revealed type is 'Union[TypedDict('__main__.KeyedTypedDict', {'key': Literal[__main__.Key.B]}), Tuple[Literal[__main__.Key.C], fallback=__main__.KeyedNamedTuple]]' + +if weird_mixture[0] is Key.B: # E: TypedDict key must be a string literal; expected one of ('key') + reveal_type(weird_mixture) # N: Revealed type is 'Union[TypedDict('__main__.KeyedTypedDict', {'key': Literal[__main__.Key.B]}), Tuple[Literal[__main__.Key.C], fallback=__main__.KeyedNamedTuple]]' +else: + reveal_type(weird_mixture) # N: Revealed type is 'Union[TypedDict('__main__.KeyedTypedDict', {'key': Literal[__main__.Key.B]}), Tuple[Literal[__main__.Key.C], fallback=__main__.KeyedNamedTuple]]' +[builtins fixtures/slice.pyi] + +[case testNarrowingParentWithProperties] +from enum import Enum +from typing import Union +from typing_extensions import Literal + +class Key(Enum): + A = 1 + B = 2 + C = 3 + +class Object1: + key: Literal[Key.A] + +class Object2: + @property + def key(self) -> Literal[Key.A]: ... + +class Object3: + @property + def key(self) -> Literal[Key.B]: ... + +x: Union[Object1, Object2, Object3] +if x.key is Key.A: + reveal_type(x) # N: Revealed type is 'Union[__main__.Object1, __main__.Object2]' +else: + reveal_type(x) # N: Revealed type is '__main__.Object3' +[builtins fixtures/property.pyi] + +[case testNarrowingParentWithAny] +from enum import Enum +from typing import Union, Any +from typing_extensions import Literal + +class Key(Enum): + A = 1 + B = 2 + C = 3 + +class Object1: + key: Literal[Key.A] + +class Object2: + key: Literal[Key.B] + +x: Union[Object1, Object2, Any] +if x.key is Key.A: + reveal_type(x.key) # N: Revealed type is 'Literal[__main__.Key.A]' + reveal_type(x) # N: Revealed type is 'Union[__main__.Object1, Any]' +else: + # TODO: Is this a bug? Should we skip inferring Any for singleton types? + reveal_type(x.key) # N: Revealed type is 'Union[Any, Literal[__main__.Key.B]]' + reveal_type(x ) # N: Revealed type is 'Union[__main__.Object1, __main__.Object2, Any]' diff --git a/test-data/unit/fixtures/tuple.pyi b/test-data/unit/fixtures/tuple.pyi index 6e000a7699fd..686e2dd55818 100644 --- a/test-data/unit/fixtures/tuple.pyi +++ b/test-data/unit/fixtures/tuple.pyi @@ -21,7 +21,8 @@ class function: pass class ellipsis: pass # We need int and slice for indexing tuples. -class int: pass +class int: + def __neg__(self) -> 'int': pass class slice: pass class bool: pass class str: pass # For convenience From b707f3d89a9172ee87270fc9a31479f6bd00c6e7 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Sat, 9 Nov 2019 15:54:46 -0800 Subject: [PATCH 2/8] Import Type from typing_extensions for Python 3.5.1 --- mypy/typeops.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/typeops.py b/mypy/typeops.py index aa31fadc6c3e..119e72834554 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -5,7 +5,8 @@ since these may assume that MROs are ready. """ -from typing import cast, Optional, List, Sequence, Set, TypeVar, Type as TypingType +from typing import cast, Optional, List, Sequence, Set, TypeVar +from typing_extensions import Type as TypingType import sys from mypy.types import ( From d183f1c100cd85bd5d9636c12c19719970a9dc8b Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Sun, 10 Nov 2019 10:10:08 -0800 Subject: [PATCH 3/8] Remove out-of-date comment --- test-data/unit/check-narrowing.test | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test index eb1a8fe48aca..13694f44ac9a 100644 --- a/test-data/unit/check-narrowing.test +++ b/test-data/unit/check-narrowing.test @@ -228,16 +228,6 @@ if ok_mixture.key is Key.A: else: reveal_type(ok_mixture) # N: Revealed type is 'Tuple[Literal[__main__.Key.C], fallback=__main__.KeyedNamedTuple]' -# Each reveal_type below really ought to be a union -- e.g. we ought short-circuit -# and skip inferring anything about the parents. Currently, we overreach in some -# cases and infer something a bit non-sensical due to how we're normalizing -# "lookup" operations. -# -# This is a bit confusing from a usability standpoint, but is probably fine: -# we don't guarantee sensible results after errors anyways. (And making sure -# these nonsensical lookups result in an error is the main purpose of this -# test case). - impossible_mixture: Union[KeyedObject, KeyedTypedDict] if impossible_mixture.key is Key.A: # E: Item "KeyedTypedDict" of "Union[KeyedObject, KeyedTypedDict]" has no attribute "key" reveal_type(impossible_mixture) # N: Revealed type is 'Union[__main__.KeyedObject, TypedDict('__main__.KeyedTypedDict', {'key': Literal[__main__.Key.B]})]' From 2c2506de0f6312f1958f2d458de76470b0d4ac6c Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Mon, 11 Nov 2019 10:57:02 -0800 Subject: [PATCH 4/8] Respond to code review --- mypy/checker.py | 209 ++++++++++++++-------------- test-data/unit/check-narrowing.test | 103 +++++++++++++- 2 files changed, 210 insertions(+), 102 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index d94e5fe7993b..b539d1d03b17 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3894,15 +3894,8 @@ def propagate_up_typemap_info(self, # The original inferred type should always be present in the output map, of course output_map[expr] = expr_type - # Next, try using this information to refine the parent type, if applicable. - # Note that we currently refine just the immediate parent. - # - # TODO: Should we also try recursively refining any parents of the parents? - # - # One quick-and-dirty way of doing this would be to have the caller repeatedly run - # this function until we reach fixpoint; another way would be to modify - # 'refine_parent_type' to run in a loop. Both approaches seem expensive though. - new_mapping = self.refine_parent_type(existing_types, expr, expr_type) + # Next, try using this information to refine the parent types, if applicable. + new_mapping = self.refine_parent_types(existing_types, expr, expr_type) for parent_expr, proposed_parent_type in new_mapping.items(): # We don't try inferring anything if we've already inferred something for # the parent expression. @@ -3912,112 +3905,126 @@ def propagate_up_typemap_info(self, output_map[parent_expr] = proposed_parent_type return output_map - def refine_parent_type(self, - existing_types: Mapping[Expression, Type], - expr: Expression, - expr_type: Type) -> Mapping[Expression, Type]: - """Checks if the given expr is a 'lookup operation' into a union and refines the parent type - based on the 'expr_type'. + def refine_parent_types(self, + existing_types: Mapping[Expression, Type], + expr: Expression, + expr_type: Type) -> Mapping[Expression, Type]: + """Checks if the given expr is a 'lookup operation' into a union and iteratively refines + the parent types based on the 'expr_type'. + + For example, if 'expr' is an expression like 'a.b.c.d', we'll potentially return refined + types for expressions 'a', 'a.b', and 'a.b.c'. For more details about what a 'lookup operation' is and how we use the expr_type to refine - the parent type, see the docstring in 'propagate_up_typemap_info'. + the parent types of lookup_expr, see the docstring in 'propagate_up_typemap_info'. """ - # First, check if this expression is one that's attempting to - # "lookup" some key in the parent type. If so, save the parent type - # and create function that will try replaying the same lookup - # operation against arbitrary types. - if isinstance(expr, MemberExpr): - parent_expr = expr.expr - parent_type = existing_types.get(parent_expr) - member_name = expr.name - - def replay_lookup(new_parent_type: ProperType) -> Optional[Type]: - msg_copy = self.msg.clean_copy() - msg_copy.disable_count = 0 - member_type = analyze_member_access( - name=member_name, - typ=new_parent_type, - context=parent_expr, - is_lvalue=False, - is_super=False, - is_operator=False, - msg=msg_copy, - original_type=new_parent_type, - chk=self, - in_literal_context=False, - ) - if msg_copy.is_errors(): - return None - else: - return member_type - elif isinstance(expr, IndexExpr): - parent_expr = expr.base - parent_type = existing_types.get(parent_expr) + output = {} # type: Dict[Expression, Type] - index_type = existing_types.get(expr.index) - if index_type is None: - return {} + # Note: parent_expr and parent_type are progressively refined as we crawl up the + # parent lookup chain. + while True: + # First, check if this expression is one that's attempting to + # "lookup" some key in the parent type. If so, save the parent type + # and create function that will try replaying the same lookup + # operation against arbitrary types. + if isinstance(expr, MemberExpr): + parent_expr = expr.expr + parent_type = existing_types.get(parent_expr) + member_name = expr.name - str_literals = try_getting_str_literals_from_type(index_type) - if str_literals is not None: def replay_lookup(new_parent_type: ProperType) -> Optional[Type]: - if not isinstance(new_parent_type, TypedDictType): - return None - try: - assert str_literals is not None - member_types = [new_parent_type.items[key] for key in str_literals] - except KeyError: + msg_copy = self.msg.clean_copy() + msg_copy.disable_count = 0 + member_type = analyze_member_access( + name=member_name, + typ=new_parent_type, + context=parent_expr, + is_lvalue=False, + is_super=False, + is_operator=False, + msg=msg_copy, + original_type=new_parent_type, + chk=self, + in_literal_context=False, + ) + if msg_copy.is_errors(): return None - return make_simplified_union(member_types) - else: - int_literals = try_getting_int_literals_from_type(index_type) - if int_literals is not None: + else: + return member_type + elif isinstance(expr, IndexExpr): + parent_expr = expr.base + parent_type = existing_types.get(parent_expr) + + index_type = existing_types.get(expr.index) + if index_type is None: + return output + + str_literals = try_getting_str_literals_from_type(index_type) + if str_literals is not None: def replay_lookup(new_parent_type: ProperType) -> Optional[Type]: - if not isinstance(new_parent_type, TupleType): + if not isinstance(new_parent_type, TypedDictType): return None try: - assert int_literals is not None - member_types = [new_parent_type.items[key] for key in int_literals] - except IndexError: + assert str_literals is not None + member_types = [new_parent_type.items[key] for key in str_literals] + except KeyError: return None return make_simplified_union(member_types) else: - return {} - else: - return {} - - # If we somehow didn't previously derive the parent type, abort: - # something went wrong at an earlier stage. - if parent_type is None: - return {} - - # We currently only try refining the parent type if it's a Union. - parent_type = get_proper_type(parent_type) - if not isinstance(parent_type, UnionType): - return {} - - # Take each element in the parent union and replay the original lookup procedure - # to figure out which parents are compatible. - new_parent_types = [] - for item in parent_type.items: - item = get_proper_type(item) - member_type = replay_lookup(item) - if member_type is None: - # We were unable to obtain the member type. So, we give up on refining this - # parent type entirely. - return {} - - if is_overlapping_types(member_type, expr_type): - new_parent_types.append(item) - - # If none of the parent types overlap (if we derived an empty union), something - # went wrong. We should never hit this case, but deriving the uninhabited type or - # reporting an error both seem unhelpful. So we abort. - if not new_parent_types: - return {} - - return {parent_expr: make_simplified_union(new_parent_types)} + int_literals = try_getting_int_literals_from_type(index_type) + if int_literals is not None: + def replay_lookup(new_parent_type: ProperType) -> Optional[Type]: + if not isinstance(new_parent_type, TupleType): + return None + try: + assert int_literals is not None + member_types = [new_parent_type.items[key] for key in int_literals] + except IndexError: + return None + return make_simplified_union(member_types) + else: + return output + else: + return output + + # If we somehow didn't previously derive the parent type, abort completely + # with what we have so far: something went wrong at an earlier stage. + if parent_type is None: + return output + + # We currently only try refining the parent type if it's a Union. + # If not, there's no point in trying to refine any further parents + # since we have no further information we can use to refine the lookup + # chain, so we end early as an optimization. + parent_type = get_proper_type(parent_type) + if not isinstance(parent_type, UnionType): + return output + + # Take each element in the parent union and replay the original lookup procedure + # to figure out which parents are compatible. + new_parent_types = [] + for item in parent_type.items: + item = get_proper_type(item) + member_type = replay_lookup(item) + if member_type is None: + # We were unable to obtain the member type. So, we give up on refining this + # parent type entirely and abort. + return output + + if is_overlapping_types(member_type, expr_type): + new_parent_types.append(item) + + # If none of the parent types overlap (if we derived an empty union), something + # went wrong. We should never hit this case, but deriving the uninhabited type or + # reporting an error both seem unhelpful. So we abort. + if not new_parent_types: + return output + + expr = parent_expr + expr_type = output[parent_expr] = make_simplified_union(new_parent_types) + + return output # # Helpers diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test index 13694f44ac9a..abce29cee3c4 100644 --- a/test-data/unit/check-narrowing.test +++ b/test-data/unit/check-narrowing.test @@ -35,6 +35,13 @@ class NamedTuple2(NamedTuple): Tuple1 = Tuple[Literal[Key.A], int] Tuple2 = Tuple[Literal[Key.B], str] +class TypedDict1(TypedDict): + key: Literal[Key.A] + foo: int +class TypedDict2(TypedDict): + key: Literal[Key.B] + foo: str + x1: Union[Object1, Object2] if x1.key is Key.A: reveal_type(x1) # N: Revealed type is '__main__.Object1' @@ -73,6 +80,12 @@ else: reveal_type(x4) # N: Revealed type is 'Tuple[Literal[__main__.Key.B], builtins.str]' reveal_type(x4[0]) # N: Revealed type is 'Literal[__main__.Key.B]' +x5: Union[TypedDict1, TypedDict2] +if x5["key"] is Key.A: + reveal_type(x5) # N: Revealed type is 'TypedDict('__main__.TypedDict1', {'key': Literal[__main__.Key.A], 'foo': builtins.int})' +else: + reveal_type(x5) # N: Revealed type is 'TypedDict('__main__.TypedDict2', {'key': Literal[__main__.Key.B], 'foo': builtins.str})' + [case testNarrowingParentWithIsInstanceBasic] from dataclasses import dataclass from typing import NamedTuple, Tuple, Union @@ -98,6 +111,11 @@ class NamedTuple2(NamedTuple): Tuple1 = Tuple[int] Tuple2 = Tuple[str] +class TypedDict1(TypedDict): + key: int +class TypedDict2(TypedDict): + key: str + x1: Union[Object1, Object2] if isinstance(x1.key, int): reveal_type(x1) # N: Revealed type is '__main__.Object1' @@ -125,6 +143,12 @@ if isinstance(x4[0], int): reveal_type(x4) # N: Revealed type is 'Tuple[builtins.int]' else: reveal_type(x4) # N: Revealed type is 'Tuple[builtins.str]' + +x5: Union[TypedDict1, TypedDict2] +if isinstance(x5["key"], int): + reveal_type(x5) # N: Revealed type is 'TypedDict('__main__.TypedDict1', {'key': builtins.int})' +else: + reveal_type(x5) # N: Revealed type is 'TypedDict('__main__.TypedDict2', {'key': builtins.str})' [builtins fixtures/isinstance.pyi] [case testNarrowingParentMultipleKeys] @@ -302,4 +326,81 @@ if x.key is Key.A: else: # TODO: Is this a bug? Should we skip inferring Any for singleton types? reveal_type(x.key) # N: Revealed type is 'Union[Any, Literal[__main__.Key.B]]' - reveal_type(x ) # N: Revealed type is 'Union[__main__.Object1, __main__.Object2, Any]' + reveal_type(x) # N: Revealed type is 'Union[__main__.Object1, __main__.Object2, Any]' + +[case testNarrowingParentsHierarchy] +from typing import Union +from typing_extensions import Literal +from enum import Enum + +class Key(Enum): + A = 1 + B = 2 + C = 3 + +class Parent1: + child: Union[Child1, Child2] +class Parent2: + child: Union[Child2, Child3] +class Parent3: + child: Union[Child3, Child1] + +class Child1: + main: Literal[Key.A] + same_for_1_and_2: Literal[Key.A] +class Child2: + main: Literal[Key.B] + same_for_1_and_2: Literal[Key.A] +class Child3: + main: Literal[Key.C] + same_for_1_and_2: Literal[Key.B] + +x: Union[Parent1, Parent2, Parent3] +if x.child.main is Key.A: + reveal_type(x) # N: Revealed type is 'Union[__main__.Parent1, __main__.Parent3]' + reveal_type(x.child) # N: Revealed type is '__main__.Child1' +else: + reveal_type(x) # N: Revealed type is 'Union[__main__.Parent1, __main__.Parent2, __main__.Parent3]' + reveal_type(x.child) # N: Revealed type is 'Union[__main__.Child2, __main__.Child3]' + +if x.child.same_for_1_and_2 is Key.A: + reveal_type(x) # N: Revealed type is 'Union[__main__.Parent1, __main__.Parent2, __main__.Parent3]' + reveal_type(x.child) # N: Revealed type is 'Union[__main__.Child1, __main__.Child2]' +else: + reveal_type(x) # N: Revealed type is 'Union[__main__.Parent2, __main__.Parent3]' + reveal_type(x.child) # N: Revealed type is '__main__.Child3' + +y: Union[Parent1, Parent2] +if y.child.main is Key.A: + reveal_type(y) # N: Revealed type is '__main__.Parent1' + reveal_type(y.child) # N: Revealed type is '__main__.Child1' +else: + reveal_type(y) # N: Revealed type is 'Union[__main__.Parent1, __main__.Parent2]' + reveal_type(y.child) # N: Revealed type is 'Union[__main__.Child2, __main__.Child3]' + +if y.child.same_for_1_and_2 is Key.A: + reveal_type(y) # N: Revealed type is 'Union[__main__.Parent1, __main__.Parent2]' + reveal_type(y.child) # N: Revealed type is 'Union[__main__.Child1, __main__.Child2]' +else: + reveal_type(y) # N: Revealed type is '__main__.Parent2' + reveal_type(y.child) # N: Revealed type is '__main__.Child3' + +[case testNarrowingParentsHierarchyGenerics] +from typing import Generic, TypeVar, Union + +T = TypeVar('T') +class Model(Generic[T]): + attr: T +class A: + model: Model[int] +class B: + model: Model[str] + +x: Union[A, B] +if isinstance(x.model.attr, int): + reveal_type(x) # N: Revealed type is '__main__.A' + reveal_type(x.model) # N: Revealed type is '__main__.Model[builtins.int]' +else: + reveal_type(x) # N: Revealed type is '__main__.B' + reveal_type(x.model) # N: Revealed type is '__main__.Model[builtins.str]' +[builtins fixtures/isinstance.pyi] From 0c331479fef1a38c0d0af2ebd29967616d0ce19a Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Mon, 11 Nov 2019 11:18:41 -0800 Subject: [PATCH 5/8] Refactor replay lookup for index exprs --- mypy/checker.py | 51 +++++++++++++++++++++++++++---------------------- mypy/types.py | 9 +++++++++ 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index b539d1d03b17..de662a8e20cf 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -6,9 +6,9 @@ from typing import ( Dict, Set, List, cast, Tuple, TypeVar, Union, Optional, NamedTuple, Iterator, Iterable, - Mapping, Sequence + Mapping, Sequence, Callable ) -from typing_extensions import Final +from typing_extensions import Final, Protocol, Type as TypingType from mypy.errors import Errors, report_internal_error from mypy.nodes import ( @@ -80,6 +80,8 @@ T = TypeVar('T') +T_contra = TypeVar('T_contra', contravariant=True) + DEFAULT_LAST_PASS = 1 # type: Final # Pass numbers start at 0 DeferredNodeType = Union[FuncDef, LambdaExpr, OverloadedFuncDef, Decorator] @@ -140,6 +142,10 @@ ]) +class SupportsLookup(Protocol[T_contra]): + def get(self, key: T_contra) -> Optional[Type]: ... + + class TypeChecker(NodeVisitor[None], CheckerPluginInterface): """Mypy type checker. @@ -3918,6 +3924,21 @@ def refine_parent_types(self, For more details about what a 'lookup operation' is and how we use the expr_type to refine the parent types of lookup_expr, see the docstring in 'propagate_up_typemap_info'. """ + def make_indexing_replay_lookup_func( + target_type: TypingType[SupportsLookup[T]], + keys: List[T], + ) -> Callable[[ProperType], Optional[Type]]: + def replay_lookup(new_parent_type: ProperType) -> Optional[Type]: + if not isinstance(new_parent_type, target_type): + return None + member_types = [] + for key in keys: + member_type = new_parent_type.get(key) + if member_type is None: + return None + member_types.append(member_type) + return make_simplified_union(member_types) + return replay_lookup output = {} # type: Dict[Expression, Type] @@ -3933,18 +3954,18 @@ def refine_parent_types(self, parent_type = existing_types.get(parent_expr) member_name = expr.name - def replay_lookup(new_parent_type: ProperType) -> Optional[Type]: + def replay_lookup(__new_parent_type: ProperType) -> Optional[Type]: msg_copy = self.msg.clean_copy() msg_copy.disable_count = 0 member_type = analyze_member_access( name=member_name, - typ=new_parent_type, + typ=__new_parent_type, context=parent_expr, is_lvalue=False, is_super=False, is_operator=False, msg=msg_copy, - original_type=new_parent_type, + original_type=__new_parent_type, chk=self, in_literal_context=False, ) @@ -3962,27 +3983,11 @@ def replay_lookup(new_parent_type: ProperType) -> Optional[Type]: str_literals = try_getting_str_literals_from_type(index_type) if str_literals is not None: - def replay_lookup(new_parent_type: ProperType) -> Optional[Type]: - if not isinstance(new_parent_type, TypedDictType): - return None - try: - assert str_literals is not None - member_types = [new_parent_type.items[key] for key in str_literals] - except KeyError: - return None - return make_simplified_union(member_types) + replay_lookup = make_indexing_replay_lookup_func(TypedDictType, str_literals) else: int_literals = try_getting_int_literals_from_type(index_type) if int_literals is not None: - def replay_lookup(new_parent_type: ProperType) -> Optional[Type]: - if not isinstance(new_parent_type, TupleType): - return None - try: - assert int_literals is not None - member_types = [new_parent_type.items[key] for key in int_literals] - except IndexError: - return None - return make_simplified_union(member_types) + replay_lookup = make_indexing_replay_lookup_func(TupleType, int_literals) else: return output else: diff --git a/mypy/types.py b/mypy/types.py index b2c689f537e2..459d8f2a172c 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1328,6 +1328,12 @@ def __init__(self, items: List[Type], fallback: Instance, line: int = -1, self.can_be_true = len(self.items) > 0 self.can_be_false = len(self.items) == 0 + def get(self, index: int) -> Optional[Type]: + if index < 0 or index >= len(self.items): + return None + else: + return self.items[index] + def length(self) -> int: return len(self.items) @@ -1403,6 +1409,9 @@ def __init__(self, items: 'OrderedDict[str, Type]', required_keys: Set[str], self.can_be_true = len(self.items) > 0 self.can_be_false = len(self.required_keys) == 0 + def get(self, key: str) -> Optional[Type]: + return self.items.get(key) + def accept(self, visitor: 'TypeVisitor[T]') -> T: return visitor.visit_typeddict_type(self) From 68b4801444b1838d162844e73a25d090a206a443 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Mon, 11 Nov 2019 21:30:00 -0800 Subject: [PATCH 6/8] Respond to second code review --- mypy/checker.py | 11 +++++--- test-data/unit/check-narrowing.test | 41 +++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index de662a8e20cf..96cf726aa7f0 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -79,7 +79,6 @@ from mypy.errorcodes import ErrorCode T = TypeVar('T') - T_contra = TypeVar('T_contra', contravariant=True) DEFAULT_LAST_PASS = 1 # type: Final # Pass numbers start at 0 @@ -3888,8 +3887,10 @@ def propagate_up_typemap_info(self, 'foo' can be further refined from Union[A, B] into just B. We perform this kind of "parent narrowing" for member lookup expressions and indexing - expressions into tuples, namedtuples, and typeddicts. This narrowing is also performed - only once, for the immediate parents of any "lookup" expressions in `new_types`. + expressions into tuples, namedtuples, and typeddicts. We repeat this narrowing + recursively if the parent is also a "lookup expression". So for example, if we have + the expression "foo['bar'].baz[0]", we'd potentially end up refining types for the + expressions "foo", "foo['bar']", and "foo['bar'].baz". We return the newly refined map. This map is guaranteed to be a superset of 'new_types'. """ @@ -3928,7 +3929,9 @@ def make_indexing_replay_lookup_func( target_type: TypingType[SupportsLookup[T]], keys: List[T], ) -> Callable[[ProperType], Optional[Type]]: - def replay_lookup(new_parent_type: ProperType) -> Optional[Type]: + # For more details on why this accepts 'object' instead of Type + # or ProperType, see https://github.com/python/mypy/issues/7932 + def replay_lookup(new_parent_type: object) -> Optional[Type]: if not isinstance(new_parent_type, target_type): return None member_types = [] diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test index abce29cee3c4..e0a0cb660c80 100644 --- a/test-data/unit/check-narrowing.test +++ b/test-data/unit/check-narrowing.test @@ -404,3 +404,44 @@ else: reveal_type(x) # N: Revealed type is '__main__.B' reveal_type(x.model) # N: Revealed type is '__main__.Model[builtins.str]' [builtins fixtures/isinstance.pyi] + +[case testNarrowingParentsHierarchyTypedDict] +# flags: --warn-unreachable +from typing import Union +from typing_extensions import TypedDict, Literal +from enum import Enum + +class Key(Enum): + A = 1 + B = 2 + C = 3 + +class Parent1(TypedDict): + model: Model1 + foo: int + +class Parent2(TypedDict): + model: Model2 + bar: str + +class Model1(TypedDict): + key: Literal[Key.A] + +class Model2(TypedDict): + key: Literal[Key.B] + +x: Union[Parent1, Parent2] +if x["model"]["key"] is Key.A: + reveal_type(x) # N: Revealed type is 'TypedDict('__main__.Parent1', {'model': TypedDict('__main__.Model1', {'key': Literal[__main__.Key.A]}), 'foo': builtins.int})' + reveal_type(x["model"]) # N: Revealed type is 'TypedDict('__main__.Model1', {'key': Literal[__main__.Key.A]})' +else: + reveal_type(x) # N: Revealed type is 'TypedDict('__main__.Parent2', {'model': TypedDict('__main__.Model2', {'key': Literal[__main__.Key.B]}), 'bar': builtins.str})' + reveal_type(x["model"]) # N: Revealed type is 'TypedDict('__main__.Model2', {'key': Literal[__main__.Key.B]})' + +y: Union[Parent1, Parent2] +if y["model"]["key"] is Key.C: + reveal_type(y) # E: Statement is unreachable + reveal_type(y["model"]) +else: + reveal_type(y) # N: Revealed type is 'Union[TypedDict('__main__.Parent1', {'model': TypedDict('__main__.Model1', {'key': Literal[__main__.Key.A]}), 'foo': builtins.int}), TypedDict('__main__.Parent2', {'model': TypedDict('__main__.Model2', {'key': Literal[__main__.Key.B]}), 'bar': builtins.str})]' + reveal_type(y["model"]) # N: Revealed type is 'Union[TypedDict('__main__.Model1', {'key': Literal[__main__.Key.A]}), TypedDict('__main__.Model2', {'key': Literal[__main__.Key.B]})]' From c2b2ccedbb2b18cd7ff10f73dc61c2f6ffa9559e Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Tue, 12 Nov 2019 17:08:02 -0800 Subject: [PATCH 7/8] Revert index lookup helper refactoring --- mypy/checker.py | 56 +++++++++++++++++++++++-------------------------- mypy/types.py | 9 -------- 2 files changed, 26 insertions(+), 39 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 96cf726aa7f0..553a8abd8874 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -6,9 +6,9 @@ from typing import ( Dict, Set, List, cast, Tuple, TypeVar, Union, Optional, NamedTuple, Iterator, Iterable, - Mapping, Sequence, Callable + Mapping, Sequence ) -from typing_extensions import Final, Protocol, Type as TypingType +from typing_extensions import Final from mypy.errors import Errors, report_internal_error from mypy.nodes import ( @@ -79,7 +79,6 @@ from mypy.errorcodes import ErrorCode T = TypeVar('T') -T_contra = TypeVar('T_contra', contravariant=True) DEFAULT_LAST_PASS = 1 # type: Final # Pass numbers start at 0 @@ -141,10 +140,6 @@ ]) -class SupportsLookup(Protocol[T_contra]): - def get(self, key: T_contra) -> Optional[Type]: ... - - class TypeChecker(NodeVisitor[None], CheckerPluginInterface): """Mypy type checker. @@ -3925,24 +3920,6 @@ def refine_parent_types(self, For more details about what a 'lookup operation' is and how we use the expr_type to refine the parent types of lookup_expr, see the docstring in 'propagate_up_typemap_info'. """ - def make_indexing_replay_lookup_func( - target_type: TypingType[SupportsLookup[T]], - keys: List[T], - ) -> Callable[[ProperType], Optional[Type]]: - # For more details on why this accepts 'object' instead of Type - # or ProperType, see https://github.com/python/mypy/issues/7932 - def replay_lookup(new_parent_type: object) -> Optional[Type]: - if not isinstance(new_parent_type, target_type): - return None - member_types = [] - for key in keys: - member_type = new_parent_type.get(key) - if member_type is None: - return None - member_types.append(member_type) - return make_simplified_union(member_types) - return replay_lookup - output = {} # type: Dict[Expression, Type] # Note: parent_expr and parent_type are progressively refined as we crawl up the @@ -3957,18 +3934,18 @@ def replay_lookup(new_parent_type: object) -> Optional[Type]: parent_type = existing_types.get(parent_expr) member_name = expr.name - def replay_lookup(__new_parent_type: ProperType) -> Optional[Type]: + def replay_lookup(new_parent_type: ProperType) -> Optional[Type]: msg_copy = self.msg.clean_copy() msg_copy.disable_count = 0 member_type = analyze_member_access( name=member_name, - typ=__new_parent_type, + typ=new_parent_type, context=parent_expr, is_lvalue=False, is_super=False, is_operator=False, msg=msg_copy, - original_type=__new_parent_type, + original_type=new_parent_type, chk=self, in_literal_context=False, ) @@ -3986,11 +3963,30 @@ def replay_lookup(__new_parent_type: ProperType) -> Optional[Type]: str_literals = try_getting_str_literals_from_type(index_type) if str_literals is not None: - replay_lookup = make_indexing_replay_lookup_func(TypedDictType, str_literals) + # Refactoring these two indexing replay functions is surprisingly + # tricky -- see https://github.com/python/mypy/pull/7917, which + # was blocked by https://github.com/mypyc/mypyc/issues/586 + def replay_lookup(new_parent_type: ProperType) -> Optional[Type]: + if not isinstance(new_parent_type, TypedDictType): + return None + try: + assert str_literals is not None + member_types = [new_parent_type.items[key] for key in str_literals] + except KeyError: + return None + return make_simplified_union(member_types) else: int_literals = try_getting_int_literals_from_type(index_type) if int_literals is not None: - replay_lookup = make_indexing_replay_lookup_func(TupleType, int_literals) + def replay_lookup(new_parent_type: ProperType) -> Optional[Type]: + if not isinstance(new_parent_type, TupleType): + return None + try: + assert int_literals is not None + member_types = [new_parent_type.items[key] for key in int_literals] + except IndexError: + return None + return make_simplified_union(member_types) else: return output else: diff --git a/mypy/types.py b/mypy/types.py index 459d8f2a172c..b2c689f537e2 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1328,12 +1328,6 @@ def __init__(self, items: List[Type], fallback: Instance, line: int = -1, self.can_be_true = len(self.items) > 0 self.can_be_false = len(self.items) == 0 - def get(self, index: int) -> Optional[Type]: - if index < 0 or index >= len(self.items): - return None - else: - return self.items[index] - def length(self) -> int: return len(self.items) @@ -1409,9 +1403,6 @@ def __init__(self, items: 'OrderedDict[str, Type]', required_keys: Set[str], self.can_be_true = len(self.items) > 0 self.can_be_false = len(self.required_keys) == 0 - def get(self, key: str) -> Optional[Type]: - return self.items.get(key) - def accept(self, visitor: 'TypeVisitor[T]') -> T: return visitor.visit_typeddict_type(self) From b4f35eab18dd0e46133261bcb2fb3aeaccac178f Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Tue, 12 Nov 2019 17:30:05 -0800 Subject: [PATCH 8/8] Fix lint error from bad merge --- mypy/checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index 9dff092c8c46..226a341995eb 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -6,7 +6,7 @@ from typing import ( Dict, Set, List, cast, Tuple, TypeVar, Union, Optional, NamedTuple, Iterator, Sequence, - Iterable, Mapping, + Mapping, ) from typing_extensions import Final