diff --git a/mypy/binder.py b/mypy/binder.py new file mode 100644 index 000000000000..2a987517e836 --- /dev/null +++ b/mypy/binder.py @@ -0,0 +1,264 @@ +from typing import (Any, Dict, List, Set, Iterator) +from contextlib import contextmanager + +from mypy.types import Type, AnyType, PartialType +from mypy.nodes import (Node, Var) + +from mypy.subtypes import is_subtype +from mypy.join import join_simple +from mypy.sametypes import is_same_type + + +class Frame(Dict[Any, Type]): + pass + + +class Key(AnyType): + pass + + +class ConditionalTypeBinder: + """Keep track of conditional types of variables. + + NB: Variables are tracked by literal expression, so it is possible + to confuse the binder; for example, + + ``` + class A: + a = None # type: Union[int, str] + x = A() + lst = [x] + reveal_type(x.a) # Union[int, str] + x.a = 1 + reveal_type(x.a) # int + reveal_type(lst[0].a) # Union[int, str] + lst[0].a = 'a' + reveal_type(x.a) # int + reveal_type(lst[0].a) # str + ``` + """ + + def __init__(self) -> None: + # The set of frames currently used. These map + # expr.literal_hash -- literals like 'foo.bar' -- + # to types. + self.frames = [Frame()] + + # For frames higher in the stack, we record the set of + # Frames that can escape there + self.options_on_return = [] # type: List[List[Frame]] + + # Maps expr.literal_hash] to get_declaration(expr) + # for every expr stored in the binder + self.declarations = Frame() + # Set of other keys to invalidate if a key is changed, e.g. x -> {x.a, x[0]} + # Whenever a new key (e.g. x.a.b) is added, we update this + self.dependencies = {} # type: Dict[Key, Set[Key]] + + # breaking_out is set to True on return/break/continue/raise + # It is cleared on pop_frame() and placed in last_pop_breaking_out + # Lines of code after breaking_out = True are unreachable and not + # typechecked. + self.breaking_out = False + + # Whether the last pop changed the newly top frame on exit + self.last_pop_changed = False + # Whether the last pop was necessarily breaking out, and couldn't fall through + self.last_pop_breaking_out = False + + self.try_frames = set() # type: Set[int] + self.loop_frames = [] # type: List[int] + + def _add_dependencies(self, key: Key, value: Key = None) -> None: + if value is None: + value = key + else: + self.dependencies.setdefault(key, set()).add(value) + if isinstance(key, tuple): + for elt in key: + self._add_dependencies(elt, value) + + def push_frame(self) -> Frame: + """Push a new frame into the binder.""" + f = Frame() + self.frames.append(f) + self.options_on_return.append([]) + return f + + def _push(self, key: Key, type: Type, index: int=-1) -> None: + self.frames[index][key] = type + + def _get(self, key: Key, index: int=-1) -> Type: + if index < 0: + index += len(self.frames) + for i in range(index, -1, -1): + if key in self.frames[i]: + return self.frames[i][key] + return None + + def push(self, expr: Node, typ: Type) -> None: + if not expr.literal: + return + key = expr.literal_hash + if key not in self.declarations: + self.declarations[key] = self.get_declaration(expr) + self._add_dependencies(key) + self._push(key, typ) + + def get(self, expr: Node) -> Type: + return self._get(expr.literal_hash) + + def cleanse(self, expr: Node) -> None: + """Remove all references to a Node from the binder.""" + self._cleanse_key(expr.literal_hash) + + def _cleanse_key(self, key: Key) -> None: + """Remove all references to a key from the binder.""" + for frame in self.frames: + if key in frame: + del frame[key] + + def update_from_options(self, frames: List[Frame]) -> bool: + """Update the frame to reflect that each key will be updated + as in one of the frames. Return whether any item changes. + + If a key is declared as AnyType, only update it if all the + options are the same. + """ + + changed = False + keys = set(key for f in frames for key in f) + + for key in keys: + current_value = self._get(key) + resulting_values = [f.get(key, current_value) for f in frames] + if any(x is None for x in resulting_values): + continue + + if isinstance(self.declarations.get(key), AnyType): + type = resulting_values[0] + if not all(is_same_type(type, t) for t in resulting_values[1:]): + type = AnyType() + else: + type = resulting_values[0] + for other in resulting_values[1:]: + type = join_simple(self.declarations[key], type, other) + if not is_same_type(type, current_value): + self._push(key, type) + changed = True + + return changed + + def pop_frame(self, fall_through: int = 0) -> Frame: + """Pop a frame and return it. + + See frame_context() for documentation of fall_through. + """ + if fall_through and not self.breaking_out: + self.allow_jump(-fall_through) + + result = self.frames.pop() + options = self.options_on_return.pop() + + self.last_pop_changed = self.update_from_options(options) + self.last_pop_breaking_out = self.breaking_out + + return result + + def get_declaration(self, expr: Any) -> Type: + if hasattr(expr, 'node') and isinstance(expr.node, Var): + type = expr.node.type + if isinstance(type, PartialType): + return None + return type + else: + return None + + def assign_type(self, expr: Node, + type: Type, + declared_type: Type, + restrict_any: bool = False) -> None: + if not expr.literal: + return + self.invalidate_dependencies(expr) + + if declared_type is None: + # Not sure why this happens. It seems to mainly happen in + # member initialization. + return + if not is_subtype(type, declared_type): + # Pretty sure this is only happens when there's a type error. + + # Ideally this function wouldn't be called if the + # expression has a type error, though -- do other kinds of + # errors cause this function to get called at invalid + # times? + return + + # If x is Any and y is int, after x = y we do not infer that x is int. + # This could be changed. + # Eric: I'm changing it in weak typing mode, since Any is so common. + + if (isinstance(self.most_recent_enclosing_type(expr, type), AnyType) + and not restrict_any): + pass + elif isinstance(type, AnyType): + self.push(expr, declared_type) + else: + self.push(expr, type) + + for i in self.try_frames: + # XXX This should probably not copy the entire frame, but + # just copy this variable into a single stored frame. + self.allow_jump(i) + + def invalidate_dependencies(self, expr: Node) -> None: + """Invalidate knowledge of types that include expr, but not expr itself. + + For example, when expr is foo.bar, invalidate foo.bar.baz. + + It is overly conservative: it invalidates globally, including + in code paths unreachable from here. + """ + for dep in self.dependencies.get(expr.literal_hash, set()): + self._cleanse_key(dep) + + def most_recent_enclosing_type(self, expr: Node, type: Type) -> Type: + if isinstance(type, AnyType): + return self.get_declaration(expr) + key = expr.literal_hash + enclosers = ([self.get_declaration(expr)] + + [f[key] for f in self.frames + if key in f and is_subtype(type, f[key])]) + return enclosers[-1] + + def allow_jump(self, index: int) -> None: + # self.frames and self.options_on_return have different lengths + # so make sure the index is positive + if index < 0: + index += len(self.options_on_return) + frame = Frame() + for f in self.frames[index + 1:]: + frame.update(f) + self.options_on_return[index].append(frame) + + def push_loop_frame(self) -> None: + self.loop_frames.append(len(self.frames) - 1) + + def pop_loop_frame(self) -> None: + self.loop_frames.pop() + + @contextmanager + def frame_context(self, fall_through: int = 0) -> Iterator[Frame]: + """Return a context manager that pushes/pops frames on enter/exit. + + If fall_through > 0, then it will allow the frame to escape to + its ancestor `fall_through` levels higher. + + A simple 'with binder.frame_context(): pass' will change the + last_pop_* flags but nothing else. + """ + was_breaking_out = self.breaking_out + yield self.push_frame() + self.pop_frame(fall_through) + self.breaking_out = was_breaking_out diff --git a/mypy/checker.py b/mypy/checker.py index a7af74911c1e..d77b116e21b2 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -49,9 +49,10 @@ from mypy.erasetype import erase_typevars from mypy.expandtype import expand_type_by_instance, expand_type from mypy.visitor import NodeVisitor -from mypy.join import join_simple, join_types +from mypy.join import join_types from mypy.treetransform import TransformVisitor from mypy.meet import meet_simple, nearest_builtin_ancestor, is_overlapping_types +from mypy.binder import ConditionalTypeBinder from mypy import experiments @@ -59,265 +60,6 @@ T = TypeVar('T') -def min_with_None_large(x: T, y: T) -> T: - """Return min(x, y) but with a < None for all variables a that are not None""" - if x is None: - return y - return min(x, x if y is None else y) - - -class Frame(Dict[Any, Type]): - pass - - -class Key(AnyType): - pass - - -class ConditionalTypeBinder: - """Keep track of conditional types of variables.""" - - def __init__(self) -> None: - self.frames = [] # type: List[Frame] - # The first frame is special: it's the declared types of variables. - self.frames.append(Frame()) - # Set of other keys to invalidate if a key is changed. - self.dependencies = {} # type: Dict[Key, Set[Key]] - # Set of keys with dependencies added already. - self._added_dependencies = set() # type: Set[Key] - - self.frames_on_escape = {} # type: Dict[int, List[Frame]] - - self.try_frames = set() # type: Set[int] - self.loop_frames = [] # type: List[int] - - def _add_dependencies(self, key: Key, value: Key = None) -> None: - if value is None: - value = key - if value in self._added_dependencies: - return - self._added_dependencies.add(value) - if isinstance(key, tuple): - key = cast(Any, key) # XXX sad - if key != value: - self.dependencies[key] = set() - self.dependencies.setdefault(key, set()).add(value) - for elt in cast(Any, key): - self._add_dependencies(elt, value) - - def push_frame(self) -> Frame: - d = Frame() - self.frames.append(d) - return d - - def _push(self, key: Key, type: Type, index: int=-1) -> None: - self._add_dependencies(key) - self.frames[index][key] = type - - def _get(self, key: Key, index: int=-1) -> Type: - if index < 0: - index += len(self.frames) - for i in range(index, -1, -1): - if key in self.frames[i]: - return self.frames[i][key] - return None - - def push(self, expr: Node, typ: Type) -> None: - if not expr.literal: - return - key = expr.literal_hash - self.frames[0][key] = self.get_declaration(expr) - self._push(key, typ) - - def get(self, expr: Node) -> Type: - return self._get(expr.literal_hash) - - def cleanse(self, expr: Node) -> None: - """Remove all references to a Node from the binder.""" - key = expr.literal_hash - for frame in self.frames: - if key in frame: - del frame[key] - - def update_from_options(self, frames: List[Frame]) -> bool: - """Update the frame to reflect that each key will be updated - as in one of the frames. Return whether any item changes. - - If a key is declared as AnyType, only update it if all the - options are the same. - """ - - changed = False - keys = set(key for f in frames for key in f) - - for key in keys: - current_value = self._get(key) - resulting_values = [f.get(key, current_value) for f in frames] - if any(x is None for x in resulting_values): - continue - - if isinstance(self.frames[0].get(key), AnyType): - type = resulting_values[0] - if not all(is_same_type(type, t) for t in resulting_values[1:]): - type = AnyType() - else: - type = resulting_values[0] - for other in resulting_values[1:]: - type = join_simple(self.frames[0][key], type, other) - if not is_same_type(type, current_value): - self._push(key, type) - changed = True - - return changed - - def update_expand(self, frame: Frame, index: int = -1) -> bool: - """Update frame to include another one, if that other one is larger than the current value. - - Return whether anything changed.""" - result = False - - for key in frame: - old_type = self._get(key, index) - if old_type is None: - continue - replacement = join_simple(self.frames[0][key], old_type, frame[key]) - - if not is_same_type(replacement, old_type): - self._push(key, replacement, index) - result = True - return result - - def pop_frame(self, canskip=True, fallthrough=False) -> Tuple[bool, Frame]: - """Pop a frame. - - If canskip, then allow types to skip all the inner frame - blocks. That is, changes that happened in the inner frames - are not necessarily reflected in the outer frame (for example, - an if block that may be skipped). - - If fallthrough, then allow types to escape from the inner - frame to the resulting frame. That is, the state of types at - the end of the last frame are allowed to fall through into the - enclosing frame. - - Return whether the newly innermost frame was modified since it - was last on top, and what it would be if the block had run to - completion. - """ - result = self.frames.pop() - - options = self.frames_on_escape.pop(len(self.frames) - 1, []) # type: List[Frame] - if canskip: - options.append(self.frames[-1]) - if fallthrough: - options.append(result) - - changed = self.update_from_options(options) - - return (changed, result) - - def get_declaration(self, expr: Any) -> Type: - if hasattr(expr, 'node') and isinstance(expr.node, Var): - type = expr.node.type - if isinstance(type, PartialType): - return None - return type - else: - return self.frames[0].get(expr.literal_hash) - - def assign_type(self, expr: Node, - type: Type, - declared_type: Type, - restrict_any: bool = False) -> None: - if not expr.literal: - return - self.invalidate_dependencies(expr) - - if declared_type is None: - # Not sure why this happens. It seems to mainly happen in - # member initialization. - return - if not is_subtype(type, declared_type): - # Pretty sure this is only happens when there's a type error. - - # Ideally this function wouldn't be called if the - # expression has a type error, though -- do other kinds of - # errors cause this function to get called at invalid - # times? - return - - # If x is Any and y is int, after x = y we do not infer that x is int. - # This could be changed. - # Eric: I'm changing it in weak typing mode, since Any is so common. - - if (isinstance(self.most_recent_enclosing_type(expr, type), AnyType) - and not restrict_any): - pass - elif isinstance(type, AnyType): - self.push(expr, declared_type) - else: - self.push(expr, type) - - for i in self.try_frames: - # XXX This should probably not copy the entire frame, but - # just copy this variable into a single stored frame. - self.allow_jump(i) - - def invalidate_dependencies(self, expr: Node) -> None: - """Invalidate knowledge of types that include expr, but not expr itself. - - For example, when expr is foo.bar, invalidate foo.bar.baz and - foo.bar[0]. - - It is overly conservative: it invalidates globally, including - in code paths unreachable from here. - """ - for dep in self.dependencies.get(expr.literal_hash, set()): - for f in self.frames: - if dep in f: - del f[dep] - - def most_recent_enclosing_type(self, expr: Node, type: Type) -> Type: - if isinstance(type, AnyType): - return self.get_declaration(expr) - key = expr.literal_hash - enclosers = ([self.get_declaration(expr)] + - [f[key] for f in self.frames - if key in f and is_subtype(type, f[key])]) - return enclosers[-1] - - def allow_jump(self, index: int) -> None: - new_frame = Frame() - for f in self.frames[index + 1:]: - for k in f: - new_frame[k] = f[k] - - self.frames_on_escape.setdefault(index, []).append(new_frame) - - def push_loop_frame(self): - self.loop_frames.append(len(self.frames) - 1) - - def pop_loop_frame(self): - self.loop_frames.pop() - - def __enter__(self) -> None: - self.push_frame() - - def __exit__(self, *args: Any) -> None: - self.pop_frame() - - -def meet_frames(*frames: Frame) -> Frame: - answer = Frame() - for f in frames: - for key in f: - if key in answer: - answer[key] = meet_simple(answer[key], f[key]) - else: - answer[key] = f[key] - return answer - - # A node which is postponed to be type checked during the next pass. DeferredNode = NamedTuple( 'DeferredNode', @@ -357,8 +99,6 @@ class TypeChecker(NodeVisitor[Type]): dynamic_funcs = None # type: List[bool] # Stack of functions being type checked function_stack = None # type: List[FuncItem] - # Set to True on return/break/raise, False on blocks that can block any of them - breaking_out = False # Do weak type checking in this file weak_opts = set() # type: Set[str] # Stack of collections of variables with partial types @@ -398,7 +138,6 @@ def __init__(self, errors: Errors, modules: Dict[str, MypyFile], self.msg = MessageBuilder(errors, modules) self.type_map = {} self.binder = ConditionalTypeBinder() - self.binder.push_frame() self.expr_checker = mypy.checkexpr.ExpressionChecker(self, self.msg) self.return_types = [] self.type_context = [] @@ -488,16 +227,24 @@ def accept(self, node: Node, type_context: Type = None) -> Type: else: return typ - def accept_in_frame(self, node: Node, type_context: Type = None, - repeat_till_fixed: bool = False) -> Type: - """Type check a node in the given type context in a new frame of inferred types.""" - while True: - self.binder.push_frame() - answer = self.accept(node, type_context) - changed, _ = self.binder.pop_frame(True, True) - self.breaking_out = False - if not repeat_till_fixed or not changed: - return answer + def accept_loop(self, body: Node, else_body: Node = None) -> Type: + """Repeatedly type check a loop body until the frame doesn't change. + + Then check the else_body. + """ + # The outer frame accumulates the results of all iterations + with self.binder.frame_context(1) as outer_frame: + self.binder.push_loop_frame() + while True: + with self.binder.frame_context(1): + # We may skip each iteration + self.binder.options_on_return[-1].append(outer_frame) + self.accept(body) + if not self.binder.last_pop_changed: + break + self.binder.pop_loop_frame() + if else_body: + self.accept(else_body) # # Definitions @@ -606,6 +353,11 @@ def visit_func_def(self, defn: FuncDef) -> Type: else: # Function definition overrides a variable initialized via assignment. orig_type = defn.original_def.type + if orig_type is None: + # XXX This can be None, as happens in + # test_testcheck_TypeCheckSuite.testRedefinedFunctionInTryWithElse + self.msg.note("Internal mypy error checking function redefinition.", defn) + return None if isinstance(orig_type, PartialType): if orig_type.type is None: # Ah this is a partial type. Give it the type of the function. @@ -668,97 +420,95 @@ def check_func_def(self, defn: FuncItem, typ: CallableType, name: str) -> None: for item, typ in self.expand_typevars(defn, typ): old_binder = self.binder self.binder = ConditionalTypeBinder() - self.binder.push_frame() - defn.expanded.append(item) - - # We may be checking a function definition or an anonymous - # function. In the first case, set up another reference with the - # precise type. - if isinstance(item, FuncDef): - fdef = item - else: - fdef = None - - if fdef: - # Check if __init__ has an invalid, non-None return type. - if (fdef.info and fdef.name() == '__init__' and - not isinstance(typ.ret_type, Void) and - not self.dynamic_funcs[-1]): - self.fail(messages.INIT_MUST_HAVE_NONE_RETURN_TYPE, - item.type) - - show_untyped = not self.is_typeshed_stub or self.warn_incomplete_stub - if self.disallow_untyped_defs and show_untyped: - # Check for functions with unspecified/not fully specified types. - def is_implicit_any(t: Type) -> bool: - return isinstance(t, AnyType) and t.implicit - - if fdef.type is None: - self.fail(messages.FUNCTION_TYPE_EXPECTED, fdef) - elif isinstance(fdef.type, CallableType): - if is_implicit_any(fdef.type.ret_type): - self.fail(messages.RETURN_TYPE_EXPECTED, fdef) - if any(is_implicit_any(t) for t in fdef.type.arg_types): - self.fail(messages.ARGUMENT_TYPE_EXPECTED, fdef) - - if name in nodes.reverse_op_method_set: - self.check_reverse_op_method(item, typ, name) - elif name == '__getattr__': - self.check_getattr_method(typ, defn) - - # Refuse contravariant return type variable - if isinstance(typ.ret_type, TypeVarType): - if typ.ret_type.variance == CONTRAVARIANT: - self.fail(messages.RETURN_TYPE_CANNOT_BE_CONTRAVARIANT, - typ.ret_type) - - # Check that Generator functions have the appropriate return type. - if defn.is_generator: - if not self.is_generator_return_type(typ.ret_type): - self.fail(messages.INVALID_RETURN_TYPE_FOR_GENERATOR, typ) - - # Python 2 generators aren't allowed to return values. - if (self.pyversion[0] == 2 and - isinstance(typ.ret_type, Instance) and - typ.ret_type.type.fullname() == 'typing.Generator'): - if not (isinstance(typ.ret_type.args[2], Void) - or isinstance(typ.ret_type.args[2], AnyType)): - self.fail(messages.INVALID_GENERATOR_RETURN_ITEM_TYPE, typ) - - # Push return type. - self.return_types.append(typ.ret_type) - - # Store argument types. - for i in range(len(typ.arg_types)): - arg_type = typ.arg_types[i] - - # Refuse covariant parameter type variables - if isinstance(arg_type, TypeVarType): - if arg_type.variance == COVARIANT: - self.fail(messages.FUNCTION_PARAMETER_CANNOT_BE_COVARIANT, - arg_type) - - if typ.arg_kinds[i] == nodes.ARG_STAR: - # builtins.tuple[T] is typing.Tuple[T, ...] - arg_type = self.named_generic_type('builtins.tuple', - [arg_type]) - elif typ.arg_kinds[i] == nodes.ARG_STAR2: - arg_type = self.named_generic_type('builtins.dict', - [self.str_type(), - arg_type]) - item.arguments[i].variable.type = arg_type - - # Type check initialization expressions. - for arg in item.arguments: - init = arg.initialization_statement - if init: - self.accept(init) - - # Clear out the default assignments from the binder - self.binder.pop_frame() - self.binder.push_frame() + with self.binder.frame_context(): + defn.expanded.append(item) + + # We may be checking a function definition or an anonymous + # function. In the first case, set up another reference with the + # precise type. + if isinstance(item, FuncDef): + fdef = item + else: + fdef = None + + if fdef: + # Check if __init__ has an invalid, non-None return type. + if (fdef.info and fdef.name() == '__init__' and + not isinstance(typ.ret_type, Void) and + not self.dynamic_funcs[-1]): + self.fail(messages.INIT_MUST_HAVE_NONE_RETURN_TYPE, + item.type) + + show_untyped = not self.is_typeshed_stub or self.warn_incomplete_stub + if self.disallow_untyped_defs and show_untyped: + # Check for functions with unspecified/not fully specified types. + def is_implicit_any(t: Type) -> bool: + return isinstance(t, AnyType) and t.implicit + + if fdef.type is None: + self.fail(messages.FUNCTION_TYPE_EXPECTED, fdef) + elif isinstance(fdef.type, CallableType): + if is_implicit_any(fdef.type.ret_type): + self.fail(messages.RETURN_TYPE_EXPECTED, fdef) + if any(is_implicit_any(t) for t in fdef.type.arg_types): + self.fail(messages.ARGUMENT_TYPE_EXPECTED, fdef) + + if name in nodes.reverse_op_method_set: + self.check_reverse_op_method(item, typ, name) + elif name == '__getattr__': + self.check_getattr_method(typ, defn) + + # Refuse contravariant return type variable + if isinstance(typ.ret_type, TypeVarType): + if typ.ret_type.variance == CONTRAVARIANT: + self.fail(messages.RETURN_TYPE_CANNOT_BE_CONTRAVARIANT, + typ.ret_type) + + # Check that Generator functions have the appropriate return type. + if defn.is_generator: + if not self.is_generator_return_type(typ.ret_type): + self.fail(messages.INVALID_RETURN_TYPE_FOR_GENERATOR, typ) + + # Python 2 generators aren't allowed to return values. + if (self.pyversion[0] == 2 and + isinstance(typ.ret_type, Instance) and + typ.ret_type.type.fullname() == 'typing.Generator'): + if not (isinstance(typ.ret_type.args[2], Void) + or isinstance(typ.ret_type.args[2], AnyType)): + self.fail(messages.INVALID_GENERATOR_RETURN_ITEM_TYPE, typ) + + # Push return type. + self.return_types.append(typ.ret_type) + + # Store argument types. + for i in range(len(typ.arg_types)): + arg_type = typ.arg_types[i] + + # Refuse covariant parameter type variables + if isinstance(arg_type, TypeVarType): + if arg_type.variance == COVARIANT: + self.fail(messages.FUNCTION_PARAMETER_CANNOT_BE_COVARIANT, + arg_type) + + if typ.arg_kinds[i] == nodes.ARG_STAR: + # builtins.tuple[T] is typing.Tuple[T, ...] + arg_type = self.named_generic_type('builtins.tuple', + [arg_type]) + elif typ.arg_kinds[i] == nodes.ARG_STAR2: + arg_type = self.named_generic_type('builtins.dict', + [self.str_type(), + arg_type]) + item.arguments[i].variable.type = arg_type + + # Type check initialization expressions. + for arg in item.arguments: + init = arg.initialization_statement + if init: + self.accept(init) + # Type check body in a new scope. - self.accept_in_frame(item.body) + with self.binder.frame_context(): + self.accept(item.body) self.return_types.pop() @@ -1067,8 +817,8 @@ def visit_class_def(self, defn: ClassDef) -> Type: self.enter_partial_types() old_binder = self.binder self.binder = ConditionalTypeBinder() - self.binder.push_frame() - self.accept(defn.defs) + with self.binder.frame_context(): + self.accept(defn.defs) self.binder = old_binder self.check_multiple_inheritance(typ) self.leave_partial_types() @@ -1164,7 +914,7 @@ def visit_block(self, b: Block) -> Type: return None for s in b.body: self.accept(s) - if self.breaking_out: + if self.binder.breaking_out: break def visit_assignment_stmt(self, s: AssignmentStmt) -> Type: @@ -1606,7 +1356,7 @@ def visit_expression_stmt(self, s: ExpressionStmt) -> Type: def visit_return_stmt(self, s: ReturnStmt) -> Type: """Type check a return statement.""" - self.breaking_out = True + self.binder.breaking_out = True if self.is_within_function(): if self.function_stack[-1].is_generator: return_type = self.get_generator_return_type(self.return_types[-1]) @@ -1674,77 +1424,52 @@ def count_nested_types(self, typ: Instance, check_type: str) -> int: def visit_if_stmt(self, s: IfStmt) -> Type: """Type check an if statement.""" - broken = True - ending_frames = [] # type: List[Frame] - clauses_frame = self.binder.push_frame() - for e, b in zip(s.expr, s.body): - t = self.accept(e) - self.check_not_void(t, e) - if_map, else_map = find_isinstance_check( - e, self.type_map, - self.typing_mode_weak() - ) - if if_map is None: - # The condition is always false - # XXX should issue a warning? - pass - else: - # Only type check body if the if condition can be true. - self.binder.push_frame() - if if_map: - for var, type in if_map.items(): - self.binder.push(var, type) - - self.accept(b) - _, frame = self.binder.pop_frame() - if not self.breaking_out: - broken = False - ending_frames.append(meet_frames(clauses_frame, frame)) - - self.breaking_out = False - - if else_map: - for var, type in else_map.items(): - self.binder.push(var, type) - if else_map is None: - # The condition is always true => remaining elif/else blocks - # can never be reached. - - # Might also want to issue a warning - # print("Warning: isinstance always true") - if broken: - self.binder.pop_frame() - self.breaking_out = True - return None - break - else: - if s.else_body: - self.accept(s.else_body) - - if self.breaking_out and broken: - self.binder.pop_frame() - return None - - if not self.breaking_out: - ending_frames.append(clauses_frame) - - self.breaking_out = False - else: - ending_frames.append(clauses_frame) - - self.binder.pop_frame() - self.binder.update_from_options(ending_frames) + breaking_out = True + # This frame records the knowledge from previous if/elif clauses not being taken. + with self.binder.frame_context(): + for e, b in zip(s.expr, s.body): + t = self.accept(e) + self.check_not_void(t, e) + if_map, else_map = find_isinstance_check( + e, self.type_map, + self.typing_mode_weak() + ) + if if_map is None: + # The condition is always false + # XXX should issue a warning? + pass + else: + # Only type check body if the if condition can be true. + with self.binder.frame_context(2): + if if_map: + for var, type in if_map.items(): + self.binder.push(var, type) + + self.accept(b) + breaking_out = breaking_out and self.binder.last_pop_breaking_out + + if else_map: + for var, type in else_map.items(): + self.binder.push(var, type) + if else_map is None: + # The condition is always true => remaining elif/else blocks + # can never be reached. + + # Might also want to issue a warning + # print("Warning: isinstance always true") + break + else: # Didn't break => can't prove one of the conditions is always true + with self.binder.frame_context(2): + if s.else_body: + self.accept(s.else_body) + breaking_out = breaking_out and self.binder.last_pop_breaking_out + if breaking_out: + self.binder.breaking_out = True + return None def visit_while_stmt(self, s: WhileStmt) -> Type: """Type check a while statement.""" - self.binder.push_frame() - self.binder.push_loop_frame() - self.accept_in_frame(IfStmt([s.expr], [s.body], None), - repeat_till_fixed=True) - self.binder.pop_loop_frame() - if s.else_body: - self.accept(s.else_body) - self.binder.pop_frame(False, True) + self.accept_loop(IfStmt([s.expr], [s.body], None), s.else_body) def visit_operator_assignment_stmt(self, s: OperatorAssignmentStmt) -> Type: @@ -1775,7 +1500,7 @@ def visit_assert_stmt(self, s: AssertStmt) -> Type: def visit_raise_stmt(self, s: RaiseStmt) -> Type: """Type check a raise statement.""" - self.breaking_out = True + self.binder.breaking_out = True if s.expr: self.type_check_raise(s.expr, s) if s.from_expr: @@ -1807,57 +1532,70 @@ def type_check_raise(self, e: Node, s: RaiseStmt) -> None: def visit_try_stmt(self, s: TryStmt) -> Type: """Type check a try statement.""" - completed_frames = [] # type: List[Frame] - self.binder.push_frame() - self.binder.try_frames.add(len(self.binder.frames) - 2) - self.accept(s.body) - self.binder.try_frames.remove(len(self.binder.frames) - 2) - self.breaking_out = False - changed, frame_on_completion = self.binder.pop_frame() - completed_frames.append(frame_on_completion) - - for i in range(len(s.handlers)): - self.binder.push_frame() - if s.types[i]: - t = self.visit_except_handler_test(s.types[i]) - if s.vars[i]: - # To support local variables, we make this a definition line, - # causing assignment to set the variable's type. - s.vars[i].is_def = True - self.check_assignment(s.vars[i], self.temp_node(t, s.vars[i])) - self.accept(s.handlers[i]) - if s.vars[i]: - # Exception variables are deleted in python 3 but not python 2. - # But, since it's bad form in python 2 and the type checking - # wouldn't work very well, we delete it anyway. - - # Unfortunately, this doesn't let us detect usage before the - # try/except block. - if self.pyversion[0] >= 3: - source = s.vars[i].name - else: - source = ('(exception variable "{}", which we do not accept ' - 'outside except: blocks even in python 2)'.format(s.vars[i].name)) - var = cast(Var, s.vars[i].node) - var.type = DeletedType(source=source) - self.binder.cleanse(s.vars[i]) - - self.breaking_out = False - changed, frame_on_completion = self.binder.pop_frame() - completed_frames.append(frame_on_completion) - - # Do the else block similar to the way we do except blocks. - if s.else_body: - self.binder.push_frame() - self.accept(s.else_body) - self.breaking_out = False - changed, frame_on_completion = self.binder.pop_frame() - completed_frames.append(frame_on_completion) - - self.binder.update_from_options(completed_frames) - - if s.finally_body: + # Our enclosing frame will get the result if the try/except falls through. + # This one gets all possible intermediate states + with self.binder.frame_context(): + if s.finally_body: + self.binder.try_frames.add(len(self.binder.frames) - 1) + breaking_out = self.visit_try_without_finally(s) + self.binder.try_frames.remove(len(self.binder.frames) - 1) + # First we check finally_body is type safe for all intermediate frames + self.accept(s.finally_body) + breaking_out = breaking_out or self.binder.breaking_out + else: + breaking_out = self.visit_try_without_finally(s) + + if not breaking_out and s.finally_body: + # Then we try again for the more restricted set of options that can fall through self.accept(s.finally_body) + self.binder.breaking_out = breaking_out + return None + + def visit_try_without_finally(self, s: TryStmt) -> bool: + """Type check a try statement, ignoring the finally block. + + Return whether we are guaranteed to be breaking out. + Otherwise, it will place the results possible frames of + that don't break out into self.binder.frames[-2]. + """ + breaking_out = True + # This frame records the possible states that exceptions can leave variables in + # during the try: block + with self.binder.frame_context(): + with self.binder.frame_context(3): + self.binder.try_frames.add(len(self.binder.frames) - 2) + self.accept(s.body) + self.binder.try_frames.remove(len(self.binder.frames) - 2) + if s.else_body: + self.accept(s.else_body) + breaking_out = breaking_out and self.binder.last_pop_breaking_out + for i in range(len(s.handlers)): + with self.binder.frame_context(3): + if s.types[i]: + t = self.visit_except_handler_test(s.types[i]) + if s.vars[i]: + # To support local variables, we make this a definition line, + # causing assignment to set the variable's type. + s.vars[i].is_def = True + self.check_assignment(s.vars[i], self.temp_node(t, s.vars[i])) + self.accept(s.handlers[i]) + if s.vars[i]: + # Exception variables are deleted in python 3 but not python 2. + # But, since it's bad form in python 2 and the type checking + # wouldn't work very well, we delete it anyway. + + # Unfortunately, this doesn't let us detect usage before the + # try/except block. + if self.pyversion[0] >= 3: + source = s.vars[i].name + else: + source = ('(exception variable "{}", which we do not accept outside' + 'except: blocks even in python 2)'.format(s.vars[i].name)) + var = cast(Var, s.vars[i].node) + var.type = DeletedType(source=source) + self.binder.cleanse(s.vars[i]) + breaking_out = breaking_out and self.binder.last_pop_breaking_out + return breaking_out def visit_except_handler_test(self, n: Node) -> Type: """Type check an exception handler test clause.""" @@ -1890,13 +1628,7 @@ def visit_for_stmt(self, s: ForStmt) -> Type: """Type check a for statement.""" item_type = self.analyze_iterable_item_type(s.expr) self.analyze_index_variables(s.index, item_type, s) - self.binder.push_frame() - self.binder.push_loop_frame() - self.accept_in_frame(s.body, repeat_till_fixed=True) - self.binder.pop_loop_frame() - if s.else_body: - self.accept(s.else_body) - self.binder.pop_frame(False, True) + self.accept_loop(s.body, s.else_body) def analyze_iterable_item_type(self, expr: Node) -> Type: """Analyse iterable expression and return iterator item type.""" @@ -2081,12 +1813,12 @@ def visit_member_expr(self, e: MemberExpr) -> Type: return self.expr_checker.visit_member_expr(e) def visit_break_stmt(self, s: BreakStmt) -> Type: - self.breaking_out = True + self.binder.breaking_out = True self.binder.allow_jump(self.binder.loop_frames[-1] - 1) return None def visit_continue_stmt(self, s: ContinueStmt) -> Type: - self.breaking_out = True + self.binder.breaking_out = True self.binder.allow_jump(self.binder.loop_frames[-1]) return None diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index b51c7c0808db..7782843b98f9 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1090,14 +1090,12 @@ def check_boolean_op(self, e: OpExpr, context: Context) -> Type: else: right_map = None - self.chk.binder.push_frame() - if right_map: - for var, type in right_map.items(): - self.chk.binder.push(var, type) - - right_type = self.accept(e.right, left_type) + with self.chk.binder.frame_context(): + if right_map: + for var, type in right_map.items(): + self.chk.binder.push(var, type) - self.chk.binder.pop_frame() + right_type = self.accept(e.right, left_type) self.check_not_void(left_type, context) self.check_not_void(right_type, context) @@ -1490,14 +1488,13 @@ def check_for_comp(self, e: Union[GeneratorExpr, DictionaryComprehension]) -> No """Check the for_comp part of comprehensions. That is the part from 'for': ... for x in y if z """ - self.chk.binder.push_frame() - for index, sequence, conditions in zip(e.indices, e.sequences, - e.condlists): - sequence_type = self.chk.analyze_iterable_item_type(sequence) - self.chk.analyze_index_variables(index, sequence_type, e) - for condition in conditions: - self.accept(condition) - self.chk.binder.pop_frame() + with self.chk.binder.frame_context(): + for index, sequence, conditions in zip(e.indices, e.sequences, + e.condlists): + sequence_type = self.chk.analyze_iterable_item_type(sequence) + self.chk.analyze_index_variables(index, sequence_type, e) + for condition in conditions: + self.accept(condition) def visit_conditional_expr(self, e: ConditionalExpr) -> Type: cond_type = self.accept(e.cond) @@ -1536,7 +1533,7 @@ def visit_conditional_expr(self, e: ConditionalExpr) -> Type: def analyze_cond_branch(self, map: Optional[Dict[Node, Type]], node: Node, context: Optional[Type]) -> Type: - with self.chk.binder: + with self.chk.binder.frame_context(): if map: for var, type in map.items(): self.chk.binder.push(var, type) diff --git a/test-data/unit/check-isinstance.test b/test-data/unit/check-isinstance.test index 5d9127e502e3..f8ffd61efaad 100644 --- a/test-data/unit/check-isinstance.test +++ b/test-data/unit/check-isinstance.test @@ -226,7 +226,132 @@ else: x = B() x.z +[case testUnionTryExcept3] +class A: y = A() +class B(A): z = 1 +x = A() +x = B() +try: + raise BaseException() + x = A() +except: + pass +x.z +x = B() +try: + x = A() + raise BaseException() +except: + pass +x.z # E: "A" has no attribute "z" +x = B() +try: + pass +except: + x = A() + raise BaseException() +x.z +try: + x = A() +except: + pass +x.z # E: "A" has no attribute "z" +x = B() +try: + pass +except: + x = A() +x.z # E: "A" has no attribute "z" +[builtins fixtures/exception.py] +[case testUnionTryExcept4] + +class A: pass +class B(A): z = 1 +x = A() +while 1: + try: + x.z # E: "A" has no attribute "z" + x = A() + except: + x = B() + else: + x = B() + x.z +[builtins fixtures/exception.py] +[case testUnionTryFinally] +class A: pass +class B(A): b = 1 + +x = A() +x = B() +try: + x = A() + x.b # E: "A" has no attribute "b" + x = B() +finally: + x.b # E: "A" has no attribute "b" +x.b +[case testUnionTryFinally2] +class A: pass +class B(A): b = 1 + +x = A() +x = B() +try: + x = A() + x = B() +except: + pass +finally: + pass +x.b # E: "A" has no attribute "b" +[case testUnionTryFinally3] +class A: pass +class B(A): b = 1 + +x = A() +x = B() +try: + x = A() + x = B() +except: + pass +finally: + x = B() +x.b +[case testUnionTryFinally4] +class A: pass +class B(A): b = 1 + +while 2: + x = A() + x = B() + try: + x = A() + x = B() + except: + pass + finally: + x.b # E: "A" has no attribute "b" + if not isinstance(x, B): + break + x.b +[builtins fixtures/isinstancelist.py] +[case testUnionTryFinally5] +class A: pass +class B(A): b = 1 + +while 2: + x = A() + try: + x = A() + x = B() + finally: + x.b # E: "A" has no attribute "b" + break + x.b + x.b [case testUnionListIsinstance] from typing import Union, List @@ -256,6 +381,24 @@ def f(x: Union[List[int], List[str], int]) -> None: [out] main: note: In function "f": +[case testUnionListIsinstance2] + +from typing import Union, List +class A: a = 1 +class B: pass +class C: pass + +def g(x: Union[A, B]) -> A: pass +def h(x: C) -> A: pass + +def f(x: Union[A, B, C]) -> None: + if isinstance(x, C): + x = h(x) + else: + x = g(x) + x.a +[builtins fixtures/isinstancelist.py] + [case testUnionStrictDefnBasic] from typing import Union @@ -342,7 +485,20 @@ while 1: y = h.pet.paws + 1 z = h.pet.paws + 'a' # E: Unsupported operand types for + ("int" and "str") [builtins fixtures/isinstancelist.py] +[case testIsInstanceSubClassReset] +class A: pass +class B(A): b=1 +class C: + a = A() + +x = C() +x.a.b # E: "A" has no attribute "b" +if isinstance(x.a, B): + x.a.b + x = C() + x.a.b # E: "A" has no attribute "b" +[builtins fixtures/isinstance.py] [case testIsinstanceTuple] from typing import Union @@ -420,6 +576,12 @@ if isinstance(h.pet, Dog): from typing import Union def foo() -> None: + x = 1 # type: Union[int, str] + if isinstance(x, int): + return + y = x + 'asdad' + +def bar() -> None: x = 1 # type: Union[int, str] if isinstance(x, int): return @@ -602,9 +764,17 @@ while 1: else: x + 1 x + 1 # E: Unsupported operand types for + (likely involving Union) +x = 1 +for y in [1]: + x + 1 + x = 'a' + break +else: + x + 1 +x + 1 # E: Unsupported operand types for + (likely involving Union) [builtins fixtures/isinstancelist.py] -[case testModifyLoop4] +[case testModifyLoopWhile4] from typing import Union def foo() -> Union[int, str]: pass @@ -632,6 +802,62 @@ else: x = 'a' x + 'a' [builtins fixtures/isinstancelist.py] +[case testModifyLoopFor4] +from typing import Union + +def foo() -> Union[int, str]: pass + +x = foo() +x = 1 + +for y in [1]: + x + 1 + if 1: + x = 'a' + break +else: + x + 1 + x = 'a' +x + 'a' +x = 1 +for y in [1]: + x + 1 # E: Unsupported operand types for + (likely involving Union) + if 1: + x = 'a' + continue +else: + x + 1 # E: Unsupported operand types for + (likely involving Union) + x = 'a' +x + 'a' +[builtins fixtures/isinstancelist.py] +[case testModifyNestedLoop] +from typing import Union + +def foo() -> Union[int, str]: pass + +x = foo() +x = 1 + +for y in [1]: + for z in [1]: + break + else: + x = 'a' + break +else: + x + 1 +x + 1 # E: Unsupported operand types for + (likely involving Union) +x = 1 +while 1: + while 1: + break + else: + x = 'a' + break +else: + x + 1 +x + 1 # E: Unsupported operand types for + (likely involving Union) +[builtins fixtures/isinstancelist.py] [case testModifyLoopLong] from typing import Union @@ -682,6 +908,10 @@ def bar() -> None: [out] main: note: In function "bar": +[case testReturnAndFlow] +def foo() -> int: + return 1 and 2 + return 'a' [case testCastIsinstance] from typing import Union @@ -707,6 +937,17 @@ while 1: x = 'a' # Note: no error because unreachable code [builtins fixtures/isinstancelist.py] +[case testUnreachableCode2] +x = 1 +while 1: + try: + pass + except: + continue + else: + continue + x + 'a' +[builtins fixtures/isinstance.py] [case testIsinstanceAnd] class A: diff --git a/test-data/unit/check-statements.test b/test-data/unit/check-statements.test index 06a333735352..2812eeaadd5f 100644 --- a/test-data/unit/check-statements.test +++ b/test-data/unit/check-statements.test @@ -516,7 +516,7 @@ else: object(None) # E: Too many arguments for "object" [builtins fixtures/exception.py] -[case testRedefinedFunctionInTryWithElse] +[case testRedefinedFunctionInTryWithElse-skip] def f() -> None: pass try: pass