diff --git a/mypy/checker.py b/mypy/checker.py index 71982bd9e459..9423a95293bf 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -12,14 +12,14 @@ OverloadedFuncDef, FuncDef, FuncItem, FuncBase, TypeInfo, ClassDef, GDEF, Block, AssignmentStmt, NameExpr, MemberExpr, IndexExpr, TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, IfStmt, - WhileStmt, OperatorAssignmentStmt, YieldStmt, WithStmt, AssertStmt, + WhileStmt, OperatorAssignmentStmt, WithStmt, AssertStmt, RaiseStmt, TryStmt, ForStmt, DelStmt, CallExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr, FloatExpr, OpExpr, UnaryExpr, CastExpr, SuperExpr, TypeApplication, DictExpr, SliceExpr, FuncExpr, TempNode, SymbolTableNode, Context, ListComprehension, ConditionalExpr, GeneratorExpr, Decorator, SetExpr, PassStmt, TypeVarExpr, PrintStmt, LITERAL_TYPE, BreakStmt, ContinueStmt, ComparisonExpr, StarExpr, - YieldFromExpr, YieldFromStmt, NamedTupleExpr, SetComprehension, + YieldFromExpr, NamedTupleExpr, SetComprehension, DictionaryComprehension, ComplexExpr, EllipsisExpr, TypeAliasExpr, RefExpr, YieldExpr, BackquoteExpr, ImportFrom, ImportAll, ImportBase, CONTRAVARIANT, COVARIANT @@ -444,6 +444,66 @@ def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: self.msg.overloaded_signatures_overlap(i + 1, j + 2, item.func) + def is_generator_return_type(self, typ: Type) -> bool: + return is_subtype(self.named_generic_type('typing.Generator', + [AnyType(), AnyType(), AnyType()]), + typ) + + def get_generator_yield_type(self, return_type: Type) -> Type: + if isinstance(return_type, AnyType): + return AnyType() + elif not self.is_generator_return_type(return_type): + # If the function doesn't have a proper Generator (or superclass) return type, anything + # is permissible. + return AnyType() + elif not isinstance(return_type, Instance): + # Same as above, but written as a separate branch so the typechecker can understand. + return AnyType() + elif return_type.args: + return return_type.args[0] + else: + # If the function's declared supertype of Generator has no type + # parameters (i.e. is `object`), then the yielded values can't + # be accessed so any type is acceptable. + return AnyType() + + def get_generator_receive_type(self, return_type: Type) -> Type: + if isinstance(return_type, AnyType): + return AnyType() + elif not self.is_generator_return_type(return_type): + # If the function doesn't have a proper Generator (or superclass) return type, anything + # is permissible. + return AnyType() + elif not isinstance(return_type, Instance): + # Same as above, but written as a separate branch so the typechecker can understand. + return AnyType() + elif return_type.type.fullname() == 'typing.Generator': + # Generator is the only type which specifies the type of values it can receive. + return return_type.args[1] + else: + # `return_type` is a supertype of Generator, so callers won't be able to send it + # values. + return Void() + + def get_generator_return_type(self, return_type: Type) -> Type: + if isinstance(return_type, AnyType): + return AnyType() + elif not self.is_generator_return_type(return_type): + # If the function doesn't have a proper Generator (or superclass) return type, anything + # is permissible. + return AnyType() + elif not isinstance(return_type, Instance): + # Same as above, but written as a separate branch so the typechecker can understand. + return AnyType() + elif return_type.type.fullname() == 'typing.Generator': + # Generator is the only type which specifies the type of values it returns into + # `yield from` expressions. + return return_type.args[2] + else: + # `return_type` is supertype of Generator, so callers won't be able to see the return + # type when used in a `yield from` expression. + return AnyType() + def visit_func_def(self, defn: FuncDef) -> Type: """Type check a function definition.""" self.check_func_item(defn, name=defn.name()) @@ -554,6 +614,19 @@ def check_func_def(self, defn: FuncItem, typ: CallableType, name: str) -> None: self.fail(messages.RETURN_TYPE_CANNOT_BE_CONTRAVARIANT, typ.ret_type) + # Check that Generator functions have the appropriate return type. + if defn.is_generator: + if not self.is_generator_return_type(typ.ret_type): + self.fail(messages.INVALID_RETURN_TYPE_FOR_GENERATOR, typ) + + # Python 2 generators aren't allowed to return values. + if (self.pyversion[0] == 2 and + isinstance(typ.ret_type, Instance) and + typ.ret_type.type.fullname() == 'typing.Generator'): + if not (isinstance(typ.ret_type.args[2], Void) + or isinstance(typ.ret_type.args[2], AnyType)): + self.fail(messages.INVALID_GENERATOR_RETURN_ITEM_TYPE, typ) + # Push return type. self.return_types.append(typ.ret_type) @@ -1400,36 +1473,37 @@ def visit_return_stmt(self, s: ReturnStmt) -> Type: """Type check a return statement.""" self.breaking_out = True if self.is_within_function(): + if self.function_stack[-1].is_generator: + return_type = self.get_generator_return_type(self.return_types[-1]) + else: + return_type = self.return_types[-1] + if s.expr: # Return with a value. - typ = self.accept(s.expr, self.return_types[-1]) + typ = self.accept(s.expr, return_type) # Returning a value of type Any is always fine. - if not isinstance(typ, AnyType): - if isinstance(self.return_types[-1], Void): - # FuncExpr (lambda) may have a Void return. - # Function returning a value of type None may have a Void return. - if (not isinstance(self.function_stack[-1], FuncExpr) and - not isinstance(typ, NoneTyp)): - self.fail(messages.NO_RETURN_VALUE_EXPECTED, s) - else: - if self.function_stack[-1].is_coroutine: - # Something similar will be needed to mix return and yield. - # If the function is a coroutine, wrap the return type in a Future. - typ = self.wrap_generic_type(cast(Instance, typ), - cast(Instance, self.return_types[-1]), - 'asyncio.futures.Future', s) - self.check_subtype( - typ, self.return_types[-1], s, - messages.INCOMPATIBLE_RETURN_VALUE_TYPE - + ": expected {}, got {}".format(self.return_types[-1], typ) - ) + if isinstance(typ, AnyType): + return None + + if isinstance(return_type, Void): + # Lambdas are allowed to have a Void return. + # Functions returning a value of type None are allowed to have a Void return. + if isinstance(self.function_stack[-1], FuncExpr) or isinstance(typ, NoneTyp): + return None + self.fail(messages.NO_RETURN_VALUE_EXPECTED, s) + else: + self.check_subtype( + typ, return_type, s, + messages.INCOMPATIBLE_RETURN_VALUE_TYPE + + ": expected {}, got {}".format(return_type, typ) + ) else: - # Return without a value. It's valid in a generator and coroutine function. - if (not self.function_stack[-1].is_generator and - not self.function_stack[-1].is_coroutine): - if (not isinstance(self.return_types[-1], Void) and - self.typing_mode_full()): - self.fail(messages.RETURN_VALUE_EXPECTED, s) + # Empty returns are valid in Generators with Any typed returns. + if (self.function_stack[-1].is_generator and isinstance(return_type, AnyType)): + return None + + if (not isinstance(return_type, Void) and self.typing_mode_full()): + self.fail(messages.RETURN_VALUE_EXPECTED, s) def wrap_generic_type(self, typ: Instance, rtyp: Instance, check_type: str, context: Context) -> Type: @@ -1455,89 +1529,6 @@ def count_nested_types(self, typ: Instance, check_type: str) -> int: return c return c - def visit_yield_stmt(self, s: YieldStmt) -> Type: - return_type = self.return_types[-1] - if isinstance(return_type, Instance): - if not is_subtype(self.named_generic_type('typing.Generator', - [AnyType(), AnyType(), AnyType()]), - return_type): - self.fail(messages.INVALID_RETURN_TYPE_FOR_YIELD, s) - return None - if return_type.args: - expected_item_type = return_type.args[0] - else: - # if the declared supertype of `Generator` has no type - # parameters (i.e. is `object`), then the yielded values can't - # be accessed so any type is acceptable. - expected_item_type = AnyType() - elif isinstance(return_type, AnyType): - expected_item_type = AnyType() - else: - self.fail(messages.INVALID_RETURN_TYPE_FOR_YIELD, s) - return None - if s.expr is None: - actual_item_type = Void() # type: Type - else: - actual_item_type = self.accept(s.expr, expected_item_type) - self.check_subtype(actual_item_type, expected_item_type, s, - messages.INCOMPATIBLE_TYPES_IN_YIELD, - 'actual type', 'expected type') - - def visit_yield_from_stmt(self, s: YieldFromStmt) -> Type: - return_type = self.return_types[-1] - type_func = self.accept(s.expr, return_type) - if isinstance(type_func, Instance): - if type_func.type.fullname() == 'asyncio.futures.Future': - # if is a Future, in stmt don't need to do nothing - # because the type Future[Some] jus matters to the main loop - # that python executes, in statement we shouldn't get the Future, - # is just for async purposes. - self.function_stack[-1].is_coroutine = True # Set the function as coroutine - elif is_subtype(type_func, self.named_type('typing.Iterable')): - # If it's and Iterable-Like, let's check the types. - # Maybe just check if have __iter__? (like in analyze_iterable) - self.check_iterable_yield_from(s) - else: - self.msg.yield_from_invalid_operand_type(type_func, s) - elif isinstance(type_func, AnyType): - self.check_iterable_yield_from(s) - else: - self.msg.yield_from_invalid_operand_type(type_func, s) - - def check_iterable_yield_from(self, s: YieldFromStmt) -> Type: - """ - Check that return type is super type of Iterable (Maybe just check if have __iter__?) - and compare it with the type of the expression - """ - expected_item_type = self.return_types[-1] - if isinstance(expected_item_type, Instance): - if not is_subtype(expected_item_type, self.named_type('typing.Iterable')): - self.fail(messages.INVALID_RETURN_TYPE_FOR_YIELD_FROM, s) - return None - elif expected_item_type.args: - expected_item_type = map_instance_to_supertype( - expected_item_type, - self.lookup_typeinfo('typing.Iterable')) - # Take the item inside the iterator. - expected_item_type = expected_item_type.args[0] - elif isinstance(expected_item_type, AnyType): - expected_item_type = AnyType() - else: - self.fail(messages.INVALID_RETURN_TYPE_FOR_YIELD_FROM, s) - return None - if s.expr is None: - actual_item_type = Void() # type: Type - else: - actual_item_type = self.accept(s.expr, expected_item_type) - if hasattr(actual_item_type, 'args') and cast(Instance, actual_item_type).args: - actual_item_type = map_instance_to_supertype( - cast(Instance, actual_item_type), - self.lookup_typeinfo('typing.Iterable')) - actual_item_type = actual_item_type.args[0] # Take the item inside the iterator - self.check_subtype(actual_item_type, expected_item_type, s, - messages.INCOMPATIBLE_TYPES_IN_YIELD_FROM, - 'actual type', 'expected type') - def visit_if_stmt(self, s: IfStmt) -> Type: """Type check an if statement.""" broken = True @@ -1868,27 +1859,45 @@ def visit_call_expr(self, e: CallExpr) -> Type: return self.expr_checker.visit_call_expr(e) def visit_yield_from_expr(self, e: YieldFromExpr) -> Type: - # result = self.expr_checker.visit_yield_from_expr(e) - result = self.accept(e.expr) + return_type = self.return_types[-1] + subexpr_type = self.accept(e.expr, return_type) + iter_type = None # type: Type + + # Check that the expr is an instance of Iterable and get the type of the iterator produced + # by __iter__. + if isinstance(subexpr_type, AnyType): + iter_type = AnyType() + elif (isinstance(subexpr_type, Instance) and + is_subtype(subexpr_type, self.named_type('typing.Iterable'))): + iter_method_type = self.expr_checker.analyze_external_member_access( + '__iter__', + subexpr_type, + AnyType()) + + generic_generator_type = self.named_generic_type('typing.Generator', + [AnyType(), AnyType(), AnyType()]) + iter_type, _ = self.expr_checker.check_call(iter_method_type, [], [], + context=generic_generator_type) + else: + self.msg.yield_from_invalid_operand_type(subexpr_type, e) + iter_type = AnyType() - # If the yield-from isn't typechecked (ie, its Any), don't typecheck - if isinstance(result, AnyType): - return AnyType() + # Check that the iterator's item type matches the type yielded by the Generator function + # containing this `yield from` expression. + expected_item_type = self.get_generator_yield_type(return_type) + actual_item_type = self.get_generator_yield_type(iter_type) - if not isinstance(result, Instance): - self.msg.yield_from_invalid_operand_type(e.expr.accept(self), e) - elif result.type.fullname() == "asyncio.futures.Future": - self.function_stack[-1].is_coroutine = True # Set the function as coroutine - result = result.args[0] # Set the return type as the type inside - elif is_subtype(result, self.named_type('typing.Iterable')): - # TODO - # Check return type Iterator[Some] - # Maybe set result like in the Future - pass + self.check_subtype(actual_item_type, expected_item_type, e, + messages.INCOMPATIBLE_TYPES_IN_YIELD_FROM, + 'actual type', 'expected type') + + # Determine the type of the entire yield from expression. + if (isinstance(iter_type, Instance) and + iter_type.type.fullname() == 'typing.Generator'): + return self.get_generator_return_type(iter_type) else: - # self.msg.yield_from_invalid_operand_type(e.expr, e) - self.msg.yield_from_invalid_operand_type(e.expr.accept(self), e) - return result + # Non-Generators don't return anything from `yield from` expressions. + return Void() def visit_member_expr(self, e: MemberExpr) -> Type: return self.expr_checker.visit_member_expr(e) @@ -1996,7 +2005,17 @@ def visit_backquote_expr(self, e: BackquoteExpr) -> Type: return self.expr_checker.visit_backquote_expr(e) def visit_yield_expr(self, e: YieldExpr) -> Type: - return self.expr_checker.visit_yield_expr(e) + return_type = self.return_types[-1] + expected_item_type = self.get_generator_yield_type(return_type) + if e.expr is None: + if not isinstance(expected_item_type, Void): + self.fail(messages.YIELD_VALUE_EXPECTED, e) + else: + actual_item_type = self.accept(e.expr, expected_item_type) + self.check_subtype(actual_item_type, expected_item_type, e, + messages.INCOMPATIBLE_TYPES_IN_YIELD, + 'actual type', 'expected type') + return self.get_generator_receive_type(return_type) # # Helpers diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 4c6d686c2a52..f94aa8081bc8 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1192,12 +1192,6 @@ def visit_dict_expr(self, e: DictExpr) -> Type: args, [nodes.ARG_POS] * len(args), e)[0] - def visit_yield_expr(self, e: YieldExpr) -> Type: - # TODO: Implement proper type checking of yield expressions. - if e.expr: - self.accept(e.expr) - return AnyType() - def visit_func_expr(self, e: FuncExpr) -> Type: """Type check lambda expression.""" inferred_type = self.infer_lambda_type_using_context(e) diff --git a/mypy/messages.py b/mypy/messages.py index a4cc2afd04e4..1b988baeea4c 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -29,10 +29,11 @@ BOOLEAN_EXPECTED_FOR_NOT = 'Boolean value expected for not operand' INVALID_EXCEPTION = 'Exception must be derived from BaseException' INVALID_EXCEPTION_TYPE = 'Exception type must be derived from BaseException' -INVALID_RETURN_TYPE_FOR_YIELD = \ +INVALID_RETURN_TYPE_FOR_GENERATOR = \ 'The return type of a generator function should be "Generator" or one of its supertypes' -INVALID_RETURN_TYPE_FOR_YIELD_FROM = \ - 'Iterable function return type expected for "yield from"' +INVALID_GENERATOR_RETURN_ITEM_TYPE = \ + 'The return type of a generator function must be None in its third type parameter in Python 2' +YIELD_VALUE_EXPECTED = 'Yield value expected' INCOMPATIBLE_TYPES = 'Incompatible types' INCOMPATIBLE_TYPES_IN_ASSIGNMENT = 'Incompatible types in assignment' INCOMPATIBLE_REDEFINITION = 'Incompatible redefinition' diff --git a/mypy/nodes.py b/mypy/nodes.py index 1feb90b721b7..3273de0dfcb1 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -323,7 +323,6 @@ class FuncItem(FuncBase): # Is this an overload variant of function with more than one overload variant? is_overload = False is_generator = False # Contains a yield statement? - is_coroutine = False # Contains @coroutine or yield from Future is_static = False # Uses @staticmethod? is_class = False # Uses @classmethod? # Variants of function with type variables with values expanded @@ -634,26 +633,6 @@ def accept(self, visitor: NodeVisitor[T]) -> T: return visitor.visit_assert_stmt(self) -class YieldStmt(Node): - expr = None # type: Node - - def __init__(self, expr: Node) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_yield_stmt(self) - - -class YieldFromStmt(Node): - expr = None # type: Node - - def __init__(self, expr: Node) -> None: - self.expr = expr - - def accept(self, visitor: NodeVisitor[T]) -> T: - return visitor.visit_yield_from_stmt(self) - - class DelStmt(Node): expr = None # type: Node diff --git a/mypy/parse.py b/mypy/parse.py index ba3367c684d9..81eaf9dc02aa 100644 --- a/mypy/parse.py +++ b/mypy/parse.py @@ -19,13 +19,13 @@ MypyFile, Import, Node, ImportAll, ImportFrom, FuncDef, OverloadedFuncDef, ClassDef, Decorator, Block, Var, OperatorAssignmentStmt, ExpressionStmt, AssignmentStmt, ReturnStmt, RaiseStmt, AssertStmt, - YieldStmt, DelStmt, BreakStmt, ContinueStmt, PassStmt, GlobalDecl, + DelStmt, BreakStmt, ContinueStmt, PassStmt, GlobalDecl, WhileStmt, ForStmt, IfStmt, TryStmt, WithStmt, CastExpr, TupleExpr, GeneratorExpr, ListComprehension, ListExpr, ConditionalExpr, DictExpr, SetExpr, NameExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr, FloatExpr, CallExpr, SuperExpr, MemberExpr, IndexExpr, SliceExpr, OpExpr, UnaryExpr, FuncExpr, TypeApplication, PrintStmt, ImportBase, ComparisonExpr, - StarExpr, YieldFromStmt, YieldFromExpr, NonlocalDecl, DictionaryComprehension, + StarExpr, YieldFromExpr, NonlocalDecl, DictionaryComprehension, SetComprehension, ComplexExpr, EllipsisExpr, YieldExpr, ExecStmt, Argument, BackquoteExpr ) @@ -879,8 +879,6 @@ def parse_statement(self) -> Tuple[Node, bool]: stmt = self.parse_nonlocal_decl() elif ts == 'assert': stmt = self.parse_assert_stmt() - elif ts == 'yield': - stmt = self.parse_yield_stmt() elif ts == 'del': stmt = self.parse_del_stmt() elif ts == 'with': @@ -963,34 +961,21 @@ def parse_assert_stmt(self) -> AssertStmt: node = AssertStmt(expr) return node - def parse_yield_stmt(self) -> Union[YieldStmt, YieldFromStmt]: - self.expect('yield') + def parse_yield_or_yield_from_expr(self) -> Union[YieldFromExpr, YieldExpr]: + self.expect("yield") expr = None - node = YieldStmt(expr) + node = YieldExpr(expr) # type: Union[YieldFromExpr, YieldExpr] if not isinstance(self.current(), Break): if self.current_str() == "from": self.expect("from") - expr = self.parse_expression() # Here comes when yield from is not assigned - node_from = YieldFromStmt(expr) - return node_from # return here, we've gotted the type + expr = self.parse_expression() # when yield from is assigned to a variable + node = YieldFromExpr(expr) else: - expr = self.parse_expression() - node = YieldStmt(expr) - return node - - def parse_yield_or_yield_from_expr(self) -> Union[YieldFromExpr, YieldExpr]: - self.expect("yield") - node = None # type: Union[YieldFromExpr, YieldExpr] - if self.current_str() == "from": - self.expect("from") - expr = self.parse_expression() # Here comes when yield from is assigned to a variable - node = YieldFromExpr(expr) - else: - if self.current_str() == ')': - node = YieldExpr(None) - else: - expr = self.parse_expression() - node = YieldExpr(expr) + if self.current_str() == ')': + node = YieldExpr(None) + else: + expr = self.parse_expression() + node = YieldExpr(expr) return node def parse_ellipsis(self) -> EllipsisExpr: diff --git a/mypy/semanal.py b/mypy/semanal.py index 048d9df7cd00..9f2959be05fd 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -49,7 +49,7 @@ ClassDef, Var, GDEF, MODULE_REF, FuncItem, Import, ImportFrom, ImportAll, Block, LDEF, NameExpr, MemberExpr, IndexExpr, TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, - RaiseStmt, YieldStmt, AssertStmt, OperatorAssignmentStmt, WhileStmt, + RaiseStmt, AssertStmt, OperatorAssignmentStmt, WhileStmt, ForStmt, BreakStmt, ContinueStmt, IfStmt, TryStmt, WithStmt, DelStmt, GlobalDecl, SuperExpr, DictExpr, CallExpr, RefExpr, OpExpr, UnaryExpr, SliceExpr, CastExpr, TypeApplication, Context, SymbolTable, @@ -57,7 +57,7 @@ FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, StrExpr, PrintStmt, ConditionalExpr, PromoteExpr, ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, - YieldFromStmt, YieldFromExpr, NamedTupleExpr, NonlocalDecl, + YieldFromExpr, NamedTupleExpr, NonlocalDecl, SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr, YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT, INVARIANT, UNBOUND_IMPORTED @@ -1447,7 +1447,6 @@ def visit_decorator(self, dec: Decorator) -> None: self.check_decorated_function_is_method('abstractmethod', dec) elif refers_to_fullname(d, 'asyncio.tasks.coroutine'): removed.append(i) - dec.func.is_coroutine = True elif refers_to_fullname(d, 'builtins.staticmethod'): removed.append(i) dec.func.is_static = True @@ -1504,20 +1503,6 @@ def visit_raise_stmt(self, s: RaiseStmt) -> None: if s.from_expr: s.from_expr.accept(self) - def visit_yield_stmt(self, s: YieldStmt) -> None: - if not self.is_func_scope(): - self.fail("'yield' outside function", s) - else: - self.function_stack[-1].is_generator = True - if s.expr: - s.expr.accept(self) - - def visit_yield_from_stmt(self, s: YieldFromStmt) -> None: - if not self.is_func_scope(): - self.fail("'yield from' outside function", s) - if s.expr: - s.expr.accept(self) - def visit_assert_stmt(self, s: AssertStmt) -> None: if s.expr: s.expr.accept(self) @@ -1681,6 +1666,8 @@ def visit_star_expr(self, expr: StarExpr) -> None: def visit_yield_from_expr(self, e: YieldFromExpr) -> None: if not self.is_func_scope(): # not sure self.fail("'yield from' outside function", e) + else: + self.function_stack[-1].is_generator = True if e.expr: e.expr.accept(self) @@ -1909,6 +1896,10 @@ def visit__promote_expr(self, expr: PromoteExpr) -> None: expr.type = self.anal_type(expr.type) def visit_yield_expr(self, expr: YieldExpr) -> None: + if not self.is_func_scope(): + self.fail("'yield' outside function", expr) + else: + self.function_stack[-1].is_generator = True if expr.expr: expr.expr.accept(self) diff --git a/mypy/test/data/check-expressions.test b/mypy/test/data/check-expressions.test index 2b989bbd846a..ff5861c8aa17 100644 --- a/mypy/test/data/check-expressions.test +++ b/mypy/test/data/check-expressions.test @@ -1322,8 +1322,10 @@ class str: pass def f(x: int) -> None: x = yield f('') x = 1 +[builtins fixtures/for.py] [out] main: note: In function "f": +main:1: error: The return type of a generator function should be "Generator" or one of its supertypes main:2: error: Argument 1 to "f" has incompatible type "str"; expected "int" [case testYieldExpressionWithNone] @@ -1334,6 +1336,33 @@ def f(x: int) -> Iterator[None]: [out] +-- Yield from expression +-- ---------------- + + + +[case testYieldFromIteratorHasNoValue] +from typing import Iterator +def f() -> Iterator[int]: + yield 5 +def g() -> Iterator[int]: + a = yield from f() +[out] +main: note: In function "g": +main:5: error: Function does not return a value + +[case testYieldFromGeneratorHasValue] +from typing import Iterator, Generator +def f() -> Generator[int, None, str]: + yield 5 + return "ham" +def g() -> Iterator[int]: + a = "string" + a = yield from f() +[out] + + + -- dict(x=y, ...) -- -------------- diff --git a/mypy/test/data/check-statements.test b/mypy/test/data/check-statements.test index 3d4450e13569..c923f9f03ee7 100644 --- a/mypy/test/data/check-statements.test +++ b/mypy/test/data/check-statements.test @@ -51,6 +51,42 @@ def g() -> None: [out] main: note: In function "g": +[case testReturnInGenerator] +from typing import Generator +def f() -> Generator[int, None, str]: + yield 1 + return "foo" +[out] + +[case testEmptyReturnInGenerator] +from typing import Generator +def f() -> Generator[int, None, str]: + yield 1 + return # E: Return value expected +[out] +main: note: In function "f": + +[case testEmptyReturnInNoneTypedGenerator] +from typing import Generator +def f() -> Generator[int, None, None]: + yield 1 + return +[out] + +[case testNonEmptyReturnInNoneTypedGenerator] +from typing import Generator +def f() -> Generator[int, None, None]: + yield 1 + return 42 # E: No return value expected +[out] +main: note: In function "f": + +[case testReturnInIterator] +from typing import Iterator +def f() -> Iterator[int]: + yield 1 + return "foo" +[out] -- If statement -- ------------ @@ -675,8 +711,8 @@ def f() -> Any: [case testYieldInFunctionReturningFunction] from typing import Callable -def f() -> Callable[[], None]: - yield object() # E: The return type of a generator function should be "Generator" or one of its supertypes +def f() -> Callable[[], None]: # E: The return type of a generator function should be "Generator" or one of its supertypes + yield object() [out] main: note: In function "f": @@ -687,8 +723,8 @@ def f(): [case testWithInvalidInstanceReturnType] import typing -def f() -> int: - yield 1 # E: The return type of a generator function should be "Generator" or one of its supertypes +def f() -> int: # E: The return type of a generator function should be "Generator" or one of its supertypes + yield 1 [builtins fixtures/for.py] [out] main: note: In function "f": @@ -715,6 +751,22 @@ def f() -> Iterator[None]: yield [builtins fixtures/for.py] +[case testYieldWithNoValueWhenValueRequired] +from typing import Iterator +def f() -> Iterator[int]: + yield # E: Yield value expected +[builtins fixtures/for.py] +[out] +main: note: In function "f": + +[case testYieldWithExplicitNone] +from typing import Iterator +def f() -> Iterator[None]: + yield None # E: Incompatible types in yield (actual type None, expected type None) +[builtins fixtures/for.py] +[out] +main: note: In function "f": + -- Yield from statement -- -------------------- @@ -746,8 +798,8 @@ def f() -> Any: from typing import Iterator, Callable def g() -> Iterator[int]: yield 42 -def f() -> Callable[[], None]: - yield from g() # E: Iterable function return type expected for "yield from" +def f() -> Callable[[], None]: # E: The return type of a generator function should be "Generator" or one of its supertypes + yield from g() [out] main: note: In function "f": @@ -755,8 +807,8 @@ main: note: In function "f": from typing import Iterator def g() -> Iterator[int]: yield 42 -def f() -> int: - yield from g() # E: Iterable function return type expected for "yield from" +def f() -> int: # E: The return type of a generator function should be "Generator" or one of its supertypes + yield from g() [out] main: note: In function "f": diff --git a/mypy/test/data/lib-stub/typing.py b/mypy/test/data/lib-stub/typing.py index 2ca3d8fdac12..07cd6f75010c 100644 --- a/mypy/test/data/lib-stub/typing.py +++ b/mypy/test/data/lib-stub/typing.py @@ -53,6 +53,9 @@ def throw(self, typ: Any, val: Any=None, tb=None) -> None: pass @abstractmethod def close(self) -> None: pass + @abstractmethod + def __iter__(self) -> 'Generator[T, U, V]': pass + class Sequence(Generic[T]): @abstractmethod def __getitem__(self, n: Any) -> T: pass diff --git a/mypy/test/data/parse.test b/mypy/test/data/parse.test index 764915aec6a3..1b320c951530 100644 --- a/mypy/test/data/parse.test +++ b/mypy/test/data/parse.test @@ -1338,11 +1338,12 @@ MypyFile:1( FuncDef:1( f Block:1( - YieldStmt:2( - OpExpr:2( - + - NameExpr(x) - IntExpr(1)))))) + ExpressionStmt:2( + YieldExpr:2( + OpExpr:2( + + + NameExpr(x) + IntExpr(1))))))) [case testYieldFrom] def f(): @@ -1352,10 +1353,11 @@ MypyFile:1( FuncDef:1( f Block:1( - YieldFromStmt:2( - CallExpr:2( - NameExpr(h) - Args()))))) + ExpressionStmt:2( + YieldFromExpr:2( + CallExpr:2( + NameExpr(h) + Args())))))) [case testYieldFromAssignment] def f(): @@ -2074,7 +2076,8 @@ MypyFile:1( FuncDef:1( f Block:1( - YieldStmt:2()))) + ExpressionStmt:2( + YieldExpr:2())))) [case testConditionalExpression] x if y else z diff --git a/mypy/test/data/pythoneval-asyncio.test b/mypy/test/data/pythoneval-asyncio.test index caca0f38e349..b8403f859527 100644 --- a/mypy/test/data/pythoneval-asyncio.test +++ b/mypy/test/data/pythoneval-asyncio.test @@ -13,12 +13,12 @@ print('Imported') Imported [case testSimpleCoroutineSleep] -from typing import Any +from typing import Any, Generator import asyncio from asyncio import Future @asyncio.coroutine -def greet_every_two_seconds() -> 'Future[None]': +def greet_every_two_seconds() -> 'Generator[Any, None, None]': n = 0 while n < 5: print('Prev', n) @@ -44,17 +44,18 @@ Prev 4 After 4 [case testCoroutineCallingOtherCoroutine] +from typing import Generator, Any import asyncio from asyncio import Future @asyncio.coroutine -def compute(x: int, y: int) -> 'Future[int]': +def compute(x: int, y: int) -> 'Generator[Any, None, int]': print("Compute %s + %s ..." % (x, y)) yield from asyncio.sleep(0.1) return x + y # Here the int is wrapped in Future[int] @asyncio.coroutine -def print_sum(x: int, y: int) -> 'Future[None]': +def print_sum(x: int, y: int) -> 'Generator[Any, None, None]': result = yield from compute(x, y) # The type of result will be int (is extracted from Future[int] print("%s + %s = %s" % (x, y, result)) @@ -66,11 +67,12 @@ Compute 1 + 2 ... 1 + 2 = 3 [case testCoroutineChangingFuture] +from typing import Generator, Any import asyncio from asyncio import Future @asyncio.coroutine -def slow_operation(future: 'Future[str]') -> 'Future[None]': +def slow_operation(future: 'Future[str]') -> 'Generator[Any, None, None]': yield from asyncio.sleep(0.1) future.set_result('Future is done!') @@ -85,11 +87,12 @@ Future is done! [case testFunctionAssignedAsCallback] import typing +from typing import Generator, Any import asyncio from asyncio import Future, AbstractEventLoop @asyncio.coroutine -def slow_operation(future: 'Future[str]') -> 'Future[None]': +def slow_operation(future: 'Future[str]') -> 'Generator[Any, None, None]': yield from asyncio.sleep(1) future.set_result('Callback works!') @@ -110,10 +113,11 @@ Callback works! [case testMultipleTasks] import typing +from typing import Generator, Any import asyncio from asyncio import Task, Future @asyncio.coroutine -def factorial(name, number) -> 'Future[None]': +def factorial(name, number) -> 'Generator[Any, None, None]': f = 1 for i in range(2, number+1): print("Task %s: Compute factorial(%s)..." % (name, i)) @@ -142,28 +146,29 @@ Task C: factorial(4) = 24 [case testConcatenatedCoroutines] import typing +from typing import Generator, Any import asyncio from asyncio import Future @asyncio.coroutine -def h4() -> 'Future[int]': +def h4() -> 'Generator[Any, None, int]': x = yield from future return x @asyncio.coroutine -def h3() -> 'Future[int]': +def h3() -> 'Generator[Any, None, int]': x = yield from h4() print("h3: %s" % x) return x @asyncio.coroutine -def h2() -> 'Future[int]': +def h2() -> 'Generator[Any, None, int]': x = yield from h3() print("h2: %s" % x) return x @asyncio.coroutine -def h() -> 'Future[None]': +def h() -> 'Generator[Any, None, None]': x = yield from h2() print("h: %s" % x) @@ -179,19 +184,20 @@ h2: 42 h: 42 Outside 42 -[case testConcantenatedCoroutinesReturningFutures] +[case testConcatenatedCoroutinesReturningFutures] import typing +from typing import Generator, Any import asyncio from asyncio import Future @asyncio.coroutine -def h4() -> 'Future[Future[int]]': +def h4() -> 'Generator[Any, None, Future[int]]': yield from asyncio.sleep(0.1) f = asyncio.Future() #type: Future[int] return f @asyncio.coroutine -def h3() -> 'Future[Future[Future[int]]]': +def h3() -> 'Generator[Any, None, Future[Future[int]]]': x = yield from h4() x.set_result(42) f = asyncio.Future() #type: Future[Future[int]] @@ -199,7 +205,7 @@ def h3() -> 'Future[Future[Future[int]]]': return f @asyncio.coroutine -def h() -> 'Future[None]': +def h() -> 'Generator[Any, None, None]': print("Before") x = yield from h3() y = yield from x @@ -224,6 +230,7 @@ Future> [case testCoroutineWithOwnClass] import typing +from typing import Generator, Any import asyncio from asyncio import Future @@ -232,7 +239,7 @@ class A: self.x = x @asyncio.coroutine -def h() -> 'Future[None]': +def h() -> 'Generator[Any, None, None]': x = yield from future print("h: %s" % x.x) @@ -250,17 +257,17 @@ Outside 42 -- Errors [case testErrorAssigningCoroutineThatDontReturn] -from typing import Any +from typing import Generator, Any import asyncio from asyncio import Future @asyncio.coroutine -def greet() -> 'Future[None]': +def greet() -> 'Generator[Any, None, None]': yield from asyncio.sleep(0.2) print('Hello World') @asyncio.coroutine -def test() -> 'Future[None]': +def test() -> 'Generator[Any, None, None]': yield from greet() x = yield from greet() # Error @@ -274,17 +281,18 @@ _program.py: note: In function "test": _program.py:13: error: Function does not return a value [case testErrorReturnIsNotTheSameType] +from typing import Generator, Any import asyncio from asyncio import Future @asyncio.coroutine -def compute(x: int, y: int) -> 'Future[int]': +def compute(x: int, y: int) -> 'Generator[Any, None, int]': print("Compute %s + %s ..." % (x, y)) yield from asyncio.sleep(0.1) return str(x + y) # Error @asyncio.coroutine -def print_sum(x: int, y: int) -> 'Future[None]': +def print_sum(x: int, y: int) -> 'Generator[Any, None, None]': result = yield from compute(x, y) print("%s + %s = %s" % (x, y, result)) @@ -294,14 +302,15 @@ loop.close() [out] _program.py: note: In function "compute": -_program.py:8: error: Incompatible return value type: expected asyncio.futures.Future[builtins.int], got asyncio.futures.Future[builtins.str] +_program.py:9: error: Incompatible return value type: expected builtins.int, got builtins.str [case testErrorSetFutureDifferentInternalType] +from typing import Generator, Any import asyncio from asyncio import Future @asyncio.coroutine -def slow_operation(future: 'Future[str]') -> 'Future[None]': +def slow_operation(future: 'Future[str]') -> 'Generator[Any, None, None]': yield from asyncio.sleep(1) future.set_result(42) # Error @@ -313,15 +322,16 @@ print(future.result()) loop.close() [out] _program.py: note: In function "slow_operation": -_program.py:7: error: Argument 1 to "set_result" of "Future" has incompatible type "int"; expected "str" +_program.py:8: error: Argument 1 to "set_result" of "Future" has incompatible type "int"; expected "str" [case testErrorUsingDifferentFutureType] +from typing import Any, Generator import asyncio from asyncio import Future @asyncio.coroutine -def slow_operation(future: 'Future[int]') -> 'Future[None]': +def slow_operation(future: 'Future[int]') -> 'Generator[Any, None, None]': yield from asyncio.sleep(1) future.set_result(42) @@ -332,14 +342,15 @@ loop.run_until_complete(future) print(future.result()) loop.close() [out] -_program.py:11: error: Argument 1 to "slow_operation" has incompatible type Future[str]; expected Future[int] +_program.py:12: error: Argument 1 to "slow_operation" has incompatible type Future[str]; expected Future[int] [case testErrorUsingDifferentFutureTypeAndSetFutureDifferentInternalType] +from typing import Generator, Any import asyncio from asyncio import Future asyncio.coroutine -def slow_operation(future: 'Future[int]') -> 'Future[None]': +def slow_operation(future: 'Future[int]') -> 'Generator[Any, None, None]': yield from asyncio.sleep(1) future.set_result('42') #Try to set an str as result to a Future[int] @@ -351,17 +362,18 @@ print(future.result()) loop.close() [out] _program.py: note: In function "slow_operation": -_program.py:7: error: Argument 1 to "set_result" of "Future" has incompatible type "str"; expected "int" +_program.py:8: error: Argument 1 to "set_result" of "Future" has incompatible type "str"; expected "int" _program.py: note: At top level: -_program.py:11: error: Argument 1 to "slow_operation" has incompatible type Future[str]; expected Future[int] +_program.py:12: error: Argument 1 to "slow_operation" has incompatible type Future[str]; expected Future[int] [case testErrorSettingCallbackWithDifferentFutureType] import typing +from typing import Generator, Any import asyncio from asyncio import Future, AbstractEventLoop @asyncio.coroutine -def slow_operation(future: 'Future[str]') -> 'Future[None]': +def slow_operation(future: 'Future[str]') -> 'Generator[Any, None, None]': yield from asyncio.sleep(1) future.set_result('Future is done!') @@ -379,21 +391,22 @@ try: finally: loop.close() [out] -_program.py:17: error: Argument 1 to "add_done_callback" of "Future" has incompatible type Callable[[Future[int]], None]; expected Callable[[Future[str]], Any] +_program.py:18: error: Argument 1 to "add_done_callback" of "Future" has incompatible type Callable[[Future[int]], None]; expected Callable[[Future[str]], Any] [case testErrorOneMoreFutureInReturnType] import typing +from typing import Any, Generator import asyncio from asyncio import Future @asyncio.coroutine -def h4() -> 'Future[Future[int]]': +def h4() -> 'Generator[Any, None, Future[int]]': yield from asyncio.sleep(1) f = asyncio.Future() #type: Future[int] return f @asyncio.coroutine -def h3() -> 'Future[Future[Future[Future[int]]]]': +def h3() -> 'Generator[Any, None, Future[Future[Future[int]]]]': x = yield from h4() x.set_result(42) f = asyncio.Future() #type: Future[Future[int]] @@ -401,7 +414,7 @@ def h3() -> 'Future[Future[Future[Future[int]]]]': return f @asyncio.coroutine -def h() -> 'Future[None]': +def h() -> 'Generator[Any, None, None]': print("Before") x = yield from h3() y = yield from x @@ -415,21 +428,22 @@ loop.run_until_complete(h()) loop.close() [out] _program.py: note: In function "h3": -_program.py:17: error: Incompatible return value type: expected asyncio.futures.Future[asyncio.futures.Future[asyncio.futures.Future[asyncio.futures.Future[builtins.int]]]], got asyncio.futures.Future[asyncio.futures.Future[builtins.int]] +_program.py:18: error: Incompatible return value type: expected asyncio.futures.Future[asyncio.futures.Future[asyncio.futures.Future[builtins.int]]], got asyncio.futures.Future[asyncio.futures.Future[builtins.int]] [case testErrorOneLessFutureInReturnType] import typing +from typing import Any, Generator import asyncio from asyncio import Future @asyncio.coroutine -def h4() -> 'Future[Future[int]]': +def h4() -> 'Generator[Any, None, Future[int]]': yield from asyncio.sleep(1) f = asyncio.Future() #type: Future[int] return f @asyncio.coroutine -def h3() -> 'Future[Future[int]]': +def h3() -> 'Generator[Any, None, Future[int]]': x = yield from h4() x.set_result(42) f = asyncio.Future() #type: Future[Future[int]] @@ -437,7 +451,7 @@ def h3() -> 'Future[Future[int]]': return f @asyncio.coroutine -def h() -> 'Future[None]': +def h() -> 'Generator[Any, None, None]': print("Before") x = yield from h3() y = yield from x @@ -449,10 +463,11 @@ loop.run_until_complete(h()) loop.close() [out] _program.py: note: In function "h3": -_program.py:17: error: Incompatible return value type: expected asyncio.futures.Future[asyncio.futures.Future[builtins.int]], got asyncio.futures.Future[asyncio.futures.Future[builtins.int]] +_program.py:18: error: Incompatible return value type: expected asyncio.futures.Future[builtins.int], got asyncio.futures.Future[asyncio.futures.Future[builtins.int]] [case testErrorAssignmentDifferentType] import typing +from typing import Generator, Any import asyncio from asyncio import Future @@ -465,7 +480,7 @@ class B: self.x = x @asyncio.coroutine -def h() -> 'Future[None]': +def h() -> 'Generator[Any, None, None]': x = yield from future # type: B # Error print("h: %s" % x.x) @@ -476,4 +491,4 @@ loop.run_until_complete(h()) loop.close() [out] _program.py: note: In function "h": -_program.py:15: error: Incompatible types in assignment (expression has type "A", variable has type "B") +_program.py:16: error: Incompatible types in assignment (expression has type "A", variable has type "B") diff --git a/mypy/test/data/semanal-statements.test b/mypy/test/data/semanal-statements.test index 0275289e18ef..7c216e476724 100644 --- a/mypy/test/data/semanal-statements.test +++ b/mypy/test/data/semanal-statements.test @@ -32,8 +32,9 @@ MypyFile:1( f Generator Block:1( - YieldStmt:1( - NameExpr(f [__main__.f]))))) + ExpressionStmt:1( + YieldExpr:1( + NameExpr(f [__main__.f])))))) [case testAssert] assert object diff --git a/mypy/traverser.py b/mypy/traverser.py index d41d4a2d91a0..d91c6e785a91 100644 --- a/mypy/traverser.py +++ b/mypy/traverser.py @@ -6,11 +6,11 @@ from mypy.nodes import ( Block, MypyFile, FuncItem, CallExpr, ClassDef, Decorator, FuncDef, ExpressionStmt, AssignmentStmt, OperatorAssignmentStmt, WhileStmt, - ForStmt, ReturnStmt, AssertStmt, YieldStmt, DelStmt, IfStmt, RaiseStmt, + ForStmt, ReturnStmt, AssertStmt, DelStmt, IfStmt, RaiseStmt, TryStmt, WithStmt, MemberExpr, OpExpr, SliceExpr, CastExpr, UnaryExpr, ListExpr, TupleExpr, DictExpr, SetExpr, IndexExpr, GeneratorExpr, ListComprehension, ConditionalExpr, TypeApplication, - FuncExpr, ComparisonExpr, OverloadedFuncDef, YieldFromStmt, YieldFromExpr, + FuncExpr, ComparisonExpr, OverloadedFuncDef, YieldFromExpr, YieldExpr ) @@ -97,14 +97,6 @@ def visit_assert_stmt(self, o: AssertStmt) -> T: if o.expr is not None: o.expr.accept(self) - def visit_yield_stmt(self, o: YieldStmt) -> T: - if o.expr is not None: - o.expr.accept(self) - - def visit_yield_from_stmt(self, o: YieldFromStmt) -> T: - if o.expr is not None: - o.expr.accept(self) - def visit_del_stmt(self, o: DelStmt) -> T: if o.expr is not None: o.expr.accept(self) diff --git a/mypy/treetransform.py b/mypy/treetransform.py index 74fc2be87f5e..b0b9572762d3 100644 --- a/mypy/treetransform.py +++ b/mypy/treetransform.py @@ -9,14 +9,14 @@ MypyFile, Import, Node, ImportAll, ImportFrom, FuncItem, FuncDef, OverloadedFuncDef, ClassDef, Decorator, Block, Var, OperatorAssignmentStmt, ExpressionStmt, AssignmentStmt, ReturnStmt, - RaiseStmt, AssertStmt, YieldStmt, DelStmt, BreakStmt, ContinueStmt, + RaiseStmt, AssertStmt, DelStmt, BreakStmt, ContinueStmt, PassStmt, GlobalDecl, WhileStmt, ForStmt, IfStmt, TryStmt, WithStmt, CastExpr, TupleExpr, GeneratorExpr, ListComprehension, ListExpr, ConditionalExpr, DictExpr, SetExpr, NameExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr, FloatExpr, CallExpr, SuperExpr, MemberExpr, IndexExpr, SliceExpr, OpExpr, UnaryExpr, FuncExpr, TypeApplication, PrintStmt, SymbolTable, RefExpr, TypeVarExpr, PromoteExpr, - ComparisonExpr, TempNode, StarExpr, YieldFromStmt, + ComparisonExpr, TempNode, StarExpr, YieldFromExpr, NamedTupleExpr, NonlocalDecl, SetComprehension, DictionaryComprehension, ComplexExpr, TypeAliasExpr, EllipsisExpr, YieldExpr, ExecStmt, Argument, BackquoteExpr @@ -240,12 +240,6 @@ def visit_return_stmt(self, node: ReturnStmt) -> Node: def visit_assert_stmt(self, node: AssertStmt) -> Node: return AssertStmt(self.node(node.expr)) - def visit_yield_stmt(self, node: YieldStmt) -> Node: - return YieldStmt(self.node(node.expr)) - - def visit_yield_from_stmt(self, node: YieldFromStmt) -> Node: - return YieldFromStmt(self.node(node.expr)) - def visit_del_stmt(self, node: DelStmt) -> Node: return DelStmt(self.node(node.expr)) diff --git a/mypy/visitor.py b/mypy/visitor.py index 45d4d6bf378e..717e004d2e82 100644 --- a/mypy/visitor.py +++ b/mypy/visitor.py @@ -85,12 +85,6 @@ def visit_return_stmt(self, o: 'mypy.nodes.ReturnStmt') -> T: def visit_assert_stmt(self, o: 'mypy.nodes.AssertStmt') -> T: pass - def visit_yield_stmt(self, o: 'mypy.nodes.YieldStmt') -> T: - pass - - def visit_yield_from_stmt(self, o: 'mypy.nodes.YieldFromStmt') -> T: - pass - def visit_del_stmt(self, o: 'mypy.nodes.DelStmt') -> T: pass diff --git a/typeshed b/typeshed index a080d6cee589..c83f4b57c75e 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit a080d6cee589b9cb7eeeb00d06bd0a824ad6a372 +Subproject commit c83f4b57c75e8d5a37c9bc6da489bedfbf3db819