Yield from

CREDITS

@@ -9,6 +9,7 @@ Contributors (in alphabetical order):
 
   Steven Allen (@Stebalien)
   Reid Barton (@rwbarton)
+  Miguel Garcia (@rockneurotiko)
   Ryan Gonzalez (@kirbyfan64)
   Ashley Hewson (@ashleyh)
   Bob Ippolito (@etrepum)
@@ -17,7 +18,6 @@ Contributors (in alphabetical order):
   Florian Ludwig (@FlorianLudwig)
   Jared Pochtar (@jaredp)
   Eric Price (@ecprice)
-  rockneurotiko (@rockneurotiko)
   Ron Murawski <[email protected]>
   Sebastian Riikonen
   Schuyler Smith

mypy/checker.py

@@ -16,7 +16,7 @@
     TypeApplication, DictExpr, SliceExpr, FuncExpr, TempNode, SymbolTableNode,
     Context, ListComprehension, ConditionalExpr, GeneratorExpr,
     Decorator, SetExpr, PassStmt, TypeVarExpr, UndefinedExpr, PrintStmt,
-    LITERAL_TYPE, BreakStmt, ContinueStmt, ComparisonExpr
+    LITERAL_TYPE, BreakStmt, ContinueStmt, ComparisonExpr, YieldFromExpr, YieldFromStmt
 )
 from mypy.nodes import function_type, method_type
 from mypy import nodes
@@ -1193,18 +1193,42 @@ def visit_return_stmt(self, s: ReturnStmt) -> Type:
                                 not isinstance(typ, NoneTyp)):
                             self.fail(messages.NO_RETURN_VALUE_EXPECTED, s)
                     else:
+                        if self.function_stack[-1].is_coroutine: # Something similar will be needed to mix return and yield
+                            # If the function is a coroutine, wrap the return type in a Future
+                            typ = self.wrap_generic_type(cast(Instance,typ), cast(Instance,self.return_types[-1]), 'asyncio.futures.Future', s)
                         self.check_subtype(
                             typ, self.return_types[-1], s,
                             messages.INCOMPATIBLE_RETURN_VALUE_TYPE
                             + ": expected {}, got {}".format(self.return_types[-1], typ)
                         )
             else:
-                # Return without a value. It's valid in a generator function.
-                if not self.function_stack[-1].is_generator:
+                # Return without a value. It's valid in a generator and coroutine function.
+                if not self.function_stack[-1].is_generator and not self.function_stack[-1].is_coroutine:
                     if (not isinstance(self.return_types[-1], Void) and
                             not self.is_dynamic_function()):
                             self.fail(messages.RETURN_VALUE_EXPECTED, s)
 
+    def wrap_generic_type(self, typ: Instance, rtyp: Instance, check_type: str, context: Context) -> Type:
+        n_diff = self.count_nested_types(rtyp, check_type) - self.count_nested_types(typ, check_type)
+        if n_diff == 1:
+            return self.named_generic_type(check_type, [typ])
+        elif n_diff == 0 or n_diff > 1:
+            self.fail(messages.INCOMPATIBLE_RETURN_VALUE_TYPE
+                + ": expected {}, got {}".format(rtyp, typ), context)
+            return typ
+        return typ
+
+    def count_nested_types(self, typ: Instance, check_type: str) -> int:
+        c = 0
+        while is_subtype(typ, self.named_type(check_type)):
+            c += 1
+            typ = map_instance_to_supertype(self.named_generic_type(check_type, typ.args), self.lookup_typeinfo(check_type))
+            if typ.args:
+                typ = cast(Instance,typ.args[0])
+            else:
+                return c
+        return c
+
     def visit_yield_stmt(self, s: YieldStmt) -> Type:
         return_type = self.return_types[-1]
         if isinstance(return_type, Instance):
@@ -1225,6 +1249,56 @@ def visit_yield_stmt(self, s: YieldStmt) -> Type:
                            messages.INCOMPATIBLE_TYPES_IN_YIELD,
                            'actual type', 'expected type')
 
+    def visit_yield_from_stmt(self, s: YieldFromStmt) -> Type:
+        return_type = self.return_types[-1]
+        type_func = self.accept(s.expr, return_type)
+        if isinstance(type_func, Instance):
+            if type_func.type.fullname() == 'asyncio.futures.Future':
+                # if is a Future, in stmt don't need to do nothing
+                # because the type Future[Some] jus matters to the main loop
+                # that python executes, in statement we shouldn't get the Future,
+                # is just for async purposes.
+                self.function_stack[-1].is_coroutine = True  # Set the function as coroutine
+            elif is_subtype(type_func, self.named_type('typing.Iterable')):
+                # If it's and Iterable-Like, let's check the types.
+                # Maybe just check if have __iter__? (like in analyse_iterable)
+                self.check_iterable_yield_from(s)
+            else:
+                self.msg.yield_from_invalid_operand_type(type_func, s)
+        elif isinstance(type_func, AnyType):
+            self.check_iterable_yield_from(s)
+        else:
+            self.msg.yield_from_invalid_operand_type(type_func, s)
+
+    def check_iterable_yield_from(self, s: YieldFromStmt) -> Type:
+        """
+            Check that return type is super type of Iterable (Maybe just check if have __iter__?)
+            and compare it with the type of the expression
+        """
+        expected_item_type = self.return_types[-1]
+        if isinstance(expected_item_type, Instance):
+            if not is_subtype(expected_item_type, self.named_type('typing.Iterable')):
+                self.fail(messages.INVALID_RETURN_TYPE_FOR_YIELD_FROM, s)
+                return None
+            elif expected_item_type.args:
+                expected_item_type = map_instance_to_supertype(expected_item_type, self.lookup_typeinfo('typing.Iterable'))
+                expected_item_type = expected_item_type.args[0]  # Take the item inside the iterator
+        elif isinstance(expected_item_type, AnyType):
+            expected_item_type = AnyType()
+        else:
+            self.fail(messages.INVALID_RETURN_TYPE_FOR_YIELD_FROM, s)
+            return None
+        if s.expr is None:
+            actual_item_type = Void() # type: Type
+        else:
+            actual_item_type = self.accept(s.expr, expected_item_type)
+            if hasattr(actual_item_type, 'args') and cast(Instance,actual_item_type).args:
+                actual_item_type = map_instance_to_supertype(cast(Instance,actual_item_type), self.lookup_typeinfo('typing.Iterable'))
+                actual_item_type = actual_item_type.args[0]   # Take the item inside the iterator
+        self.check_subtype(actual_item_type, expected_item_type, s,
+                           messages.INCOMPATIBLE_TYPES_IN_YIELD_FROM,
+                           'actual type', 'expected type')
+
     def visit_if_stmt(self, s: IfStmt) -> Type:
         """Type check an if statement."""
         broken = True
@@ -1527,6 +1601,23 @@ def visit_call_expr(self, e: CallExpr) -> Type:
         self.breaking_out = False
         return result
 
+    def visit_yield_from_expr(self, e: YieldFromExpr) -> Type:
+        # result = self.expr_checker.visit_yield_from_expr(e)
+        result = self.accept(e.expr)
+        result_instance = cast(Instance, result)
+        if result_instance.type.fullname() == "asyncio.futures.Future":
+            self.function_stack[-1].is_coroutine = True  # Set the function as coroutine
+            result = result_instance.args[0]  # Set the return type as the type inside
+        elif is_subtype(result, self.named_type('typing.Iterable')):
+            # TODO
+            # Check return type Iterator[Some]
+            # Maybe set result like in the Future
+            pass
+        else:
+            # self.msg.yield_from_invalid_operand_type(e.expr, e)
+            self.msg.yield_from_invalid_operand_type(e.expr.accept(self), e)
+        return result
+
     def visit_member_expr(self, e: MemberExpr) -> Type:
         return self.expr_checker.visit_member_expr(e)
 

mypy/checkstrformat.py

@@ -159,7 +159,7 @@ def check_mapping_str_interpolation(self, specifiers: List[ConversionSpecifier],
                                    'expression has type', 'expected type for mapping is')
 
     def build_replacement_checkers(self, specifiers: List[ConversionSpecifier],
-                                   context: Context) -> List[ Tuple[ Function[[Node], None], 
+                                   context: Context) -> List[ Tuple[ Function[[Node], None],
                                                                      Function[[Type], None] ] ]:
         checkers = []  # type: List[ Tuple[ Function[[Node], None], Function[[Type], None] ] ]
         for specifier in specifiers:
@@ -170,10 +170,10 @@ def build_replacement_checkers(self, specifiers: List[ConversionSpecifier],
         return checkers
 
     def replacement_checkers(self, specifier: ConversionSpecifier,
-                             context: Context) -> List[ Tuple[ Function[[Node], None], 
+                             context: Context) -> List[ Tuple[ Function[[Node], None],
                                                                Function[[Type], None] ] ]:
         """Returns a list of tuples of two functions that check whether a replacement is
-        of the right type for the specifier. The first functions take a node and checks 
+        of the right type for the specifier. The first functions take a node and checks
         its type in the right type context. The second function just checks a type.
         """
         checkers = []  # type: List[ Tuple[ Function[[Node], None], Function[[Type], None] ] ]
@@ -194,7 +194,7 @@ def replacement_checkers(self, specifier: ConversionSpecifier,
             checkers.append(c)
         return checkers
 
-    def checkers_for_star(self, context: Context) -> Tuple[ Function[[Node], None], 
+    def checkers_for_star(self, context: Context) -> Tuple[ Function[[Node], None],
                                                                                  Function[[Type], None] ]:
         """Returns a tuple of check functions that check whether, respectively,
         a node or a type is compatible with a star in a conversion specifier
@@ -211,10 +211,10 @@ def check_node(node: Node) -> None:
 
         return check_node, check_type
 
-    def checkers_for_regular_type(self, type: str, context: Context) -> Tuple[ Function[[Node], None], 
+    def checkers_for_regular_type(self, type: str, context: Context) -> Tuple[ Function[[Node], None],
                                                                                Function[[Type], None] ]:
         """Returns a tuple of check functions that check whether, respectively,
-        a node or a type is compatible with 'type'. Return None in case of an 
+        a node or a type is compatible with 'type'. Return None in case of an
         """
         expected_type = self.conversion_type(type, context)
         if expected_type == None:
@@ -231,7 +231,7 @@ def check_node(node: Node) -> None:
 
         return check_node, check_type
 
-    def checkers_for_c_type(self, type: str, context: Context) -> Tuple[ Function[[Node], None], 
+    def checkers_for_c_type(self, type: str, context: Context) -> Tuple[ Function[[Node], None],
                                                                          Function[[Type], None] ]:
         """Returns a tuple of check functions that check whether, respectively,
         a node or a type is compatible with 'type' that is a character type

mypy/messages.py

@@ -32,9 +32,12 @@
 INVALID_EXCEPTION_TYPE = 'Exception type must be derived from BaseException'
 INVALID_RETURN_TYPE_FOR_YIELD = \
     'Iterator function return type expected for "yield"'
+INVALID_RETURN_TYPE_FOR_YIELD_FROM = \
+    'Iterable function return type expected for "yield from"'
 INCOMPATIBLE_TYPES = 'Incompatible types'
 INCOMPATIBLE_TYPES_IN_ASSIGNMENT = 'Incompatible types in assignment'
 INCOMPATIBLE_TYPES_IN_YIELD = 'Incompatible types in yield'
+INCOMPATIBLE_TYPES_IN_YIELD_FROM = 'Incompatible types in "yield from"'
 INCOMPATIBLE_TYPES_IN_STR_INTERPOLATION = 'Incompatible types in string interpolation'
 INIT_MUST_NOT_HAVE_RETURN_TYPE = 'Cannot define return type for "__init__"'
 GETTER_TYPE_INCOMPATIBLE_WITH_SETTER = \
@@ -671,6 +674,11 @@ def signatures_incompatible(self, method: str, other_method: str,
         self.fail('Signatures of "{}" and "{}" are incompatible'.format(
             method, other_method), context)
 
+    def yield_from_invalid_operand_type(self, expr: Type, context: Context) -> Type:
+        text = self.format(expr) if self.format(expr) != 'object' else expr
+        self.fail('"yield from" can\'t be applied to {}'.format(text), context)
+        return AnyType()
+
 
 def capitalize(s: str) -> str:
     """Capitalize the first character of a string."""
@@ -721,4 +729,4 @@ def callable_name(type: Callable) -> str:
 
 def temp_message_builder() -> MessageBuilder:
     """Return a message builder usable for collecting errors locally."""
-    return MessageBuilder(Errors())
+    return MessageBuilder(Errors())

mypy/nodes.py

@@ -238,6 +238,7 @@ class FuncItem(FuncBase):
     is_overload = False    # Is this an overload variant of function with
                            # more than one overload variant?
     is_generator = False   # Contains a yield statement?
+    is_coroutine = False   # Contains @coroutine or yield from Future
     is_static = False      # Uses @staticmethod?
     is_class = False       # Uses @classmethod?
     expanded = Undefined(List['FuncItem'])  # Variants of function with type
@@ -615,6 +616,16 @@ def accept(self, visitor: NodeVisitor[T]) -> T:
         return visitor.visit_yield_stmt(self)
 
 
+class YieldFromStmt(Node):
+    expr = Undefined(Node)
+
+    def __init__(self, expr: Node) -> None:
+        self.expr = expr
+
+    def accept(self, visitor: NodeVisitor[T]) -> T:
+        return visitor.visit_yield_from_stmt(self)
+
+
 class DelStmt(Node):
     expr = Undefined(Node)
 
@@ -906,6 +917,16 @@ def accept(self, visitor: NodeVisitor[T]) -> T:
         return visitor.visit_call_expr(self)
 
 
+class YieldFromExpr(Node):
+    expr = Undefined(Node)
+
+    def __init__(self, expr: Node) -> None:
+        self.expr = expr
+
+    def accept(self, visitor: NodeVisitor[T]) -> T:
+        return visitor.visit_yield_from_expr(self)
+
+
 class IndexExpr(Node):
     """Index expression x[y].
 

mypy/output.py

@@ -226,6 +226,9 @@ def visit_assert_stmt(self, o):
     def visit_yield_stmt(self, o):
         self.simple_stmt(o, o.expr)
 
+    def visit_yield_from_stmt(self, o):
+        self.simple_stmt(o, o.expr)
+
     def visit_del_stmt(self, o):
         self.simple_stmt(o, o.expr)
 
@@ -355,6 +358,9 @@ def visit_slice_expr(self, o):
         self.token(o.repr.colon2)
         self.node(o.stride)
 
+    def visit_yield_from_expr(self, o):
+        o.expr.accept(self)
+
     def visit_call_expr(self, o):
         r = o.repr
         self.node(o.callee)

mypy/parse.py

@@ -6,7 +6,7 @@
 
 import re
 
-from typing import Undefined, List, Tuple, Any, Set, cast
+from typing import Undefined, List, Tuple, Any, Set, cast, Union
 
 from mypy import lex
 from mypy.lex import (
@@ -23,7 +23,8 @@
     TupleExpr, GeneratorExpr, ListComprehension, ListExpr, ConditionalExpr,
     DictExpr, SetExpr, NameExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr,
     FloatExpr, CallExpr, SuperExpr, MemberExpr, IndexExpr, SliceExpr, OpExpr,
-    UnaryExpr, FuncExpr, TypeApplication, PrintStmt, ImportBase, ComparisonExpr
+    UnaryExpr, FuncExpr, TypeApplication, PrintStmt, ImportBase, ComparisonExpr, YieldFromStmt,
+    YieldFromExpr
 )
 from mypy import nodes
 from mypy import noderepr
@@ -743,6 +744,8 @@ def parse_return_stmt(self) -> ReturnStmt:
         expr = None  # type: Node
         if not isinstance(self.current(), Break):
             expr = self.parse_expression()
+            if isinstance(expr, YieldFromExpr): # "yield from" expressions can't be returned.
+                return None
         br = self.expect_break()
         node = ReturnStmt(expr)
         self.set_repr(node, noderepr.SimpleStmtRepr(return_tok, br))
@@ -771,16 +774,41 @@ def parse_assert_stmt(self) -> AssertStmt:
         self.set_repr(node, noderepr.SimpleStmtRepr(assert_tok, br))
         return node
 
-    def parse_yield_stmt(self) -> YieldStmt:
+    def parse_yield_stmt(self) -> Union[YieldStmt, YieldFromStmt]:
         yield_tok = self.expect('yield')
         expr = None  # type: Node
+        node = YieldStmt(expr)
         if not isinstance(self.current(), Break):
-            expr = self.parse_expression()
+            if isinstance(self.current(), Keyword) and self.current_str() == "from":  # Not go if it's not from
+                from_tok = self.expect("from")
+                expr = self.parse_expression()  # Here comes when yield from is not assigned
+                node_from = YieldFromStmt(expr)
+                br = self.expect_break()
+                self.set_repr(node_from, noderepr.SimpleStmtRepr(yield_tok, br))
+                return node_from  # return here, we've gotted the type
+            else:
+                expr = self.parse_expression()
+                node = YieldStmt(expr)
         br = self.expect_break()
-        node = YieldStmt(expr)
         self.set_repr(node, noderepr.SimpleStmtRepr(yield_tok, br))
         return node
 
+    def parse_yield_from_expr(self) -> YieldFromExpr:
+        y_tok = self.expect("yield")
+        expr = None # type: Node
+        node = YieldFromExpr(expr)
+        if self.current_str() == "from":
+            f_tok = self.expect("from")
+            tok = self.parse_expression()  # Here comes when yield from is assigned to a variable
+            node = YieldFromExpr(tok)
+        else:
+            # TODO
+            # Here comes the yield expression (ex:  x = yield 3 )
+            # tok = self.parse_expression()
+            # node = YieldExpr(tok)  # Doesn't exist now
+            pass
+        return node
+
     def parse_del_stmt(self) -> DelStmt:
         del_tok = self.expect('del')
         expr = self.parse_expression()
@@ -1054,6 +1082,8 @@ def parse_expression(self, prec: int = 0) -> Node:
                 expr = self.parse_unicode_literal()
             elif isinstance(self.current(), FloatLit):
                 expr = self.parse_float_expr()
+            elif isinstance(t, Keyword) and s == "yield":
+                expr = self.parse_yield_from_expr() # The expression yield from and yield to assign
             else:
                 # Invalid expression.
                 self.parse_error()