Merge pull request #3540 from WarriorOfWire/async_syntax

__await__ magic method and async/await

Author: tannewt

locale/circuitpython.pot

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2020-10-06 13:26-0400\n"
+"POT-Creation-Date: 2020-10-10 23:49-0700\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <[email protected]>\n"
@@ -246,6 +246,10 @@ msgstr ""
 msgid "'return' outside function"
 msgstr ""
 
+#: py/compile.c
+msgid "'yield from' inside async function"
+msgstr ""
+
 #: py/compile.c
 msgid "'yield' outside function"
 msgstr ""
@@ -289,6 +293,7 @@ msgid "All I2C peripherals are in use"
 msgstr ""
 
 #: ports/atmel-samd/common-hal/canio/Listener.c
+#: ports/stm/common-hal/canio/Listener.c
 msgid "All RX FIFOs in use"
 msgstr ""
 
@@ -896,6 +901,7 @@ msgid "File exists"
 msgstr ""
 
 #: ports/atmel-samd/common-hal/canio/Listener.c
+#: ports/stm/common-hal/canio/Listener.c
 msgid "Filters too complex"
 msgstr ""
 
@@ -932,7 +938,8 @@ msgid "Group full"
 msgstr ""
 
 #: ports/mimxrt10xx/common-hal/busio/SPI.c ports/stm/common-hal/busio/I2C.c
-#: ports/stm/common-hal/busio/SPI.c ports/stm/common-hal/sdioio/SDCard.c
+#: ports/stm/common-hal/busio/SPI.c ports/stm/common-hal/canio/CAN.c
+#: ports/stm/common-hal/sdioio/SDCard.c
 msgid "Hardware busy, try alternative pins"
 msgstr ""
 
@@ -1002,7 +1009,8 @@ msgid "Invalid %q pin"
 msgstr ""
 
 #: ports/stm/common-hal/busio/I2C.c ports/stm/common-hal/busio/SPI.c
-#: ports/stm/common-hal/busio/UART.c ports/stm/common-hal/sdioio/SDCard.c
+#: ports/stm/common-hal/busio/UART.c ports/stm/common-hal/canio/CAN.c
+#: ports/stm/common-hal/sdioio/SDCard.c
 msgid "Invalid %q pin selection"
 msgstr ""
 
@@ -3426,6 +3434,10 @@ msgstr ""
 msgid "type object '%q' has no attribute '%q'"
 msgstr ""
 
+#: py/objgenerator.c
+msgid "type object 'generator' has no attribute '__await__'"
+msgstr ""
+
 #: py/objtype.c
 msgid "type takes 1 or 3 arguments"
 msgstr ""

ports/atmel-samd/boards/circuitbrains_basic_m0/mpconfigboard.mk

@@ -16,6 +16,7 @@ CIRCUITPY_COUNTIO = 0
 CIRCUITPY_FREQUENCYIO = 0
 CIRCUITPY_I2CPERIPHERAL = 0
 CIRCUITPY_VECTORIO = 0
+MICROPY_PY_ASYNC_AWAIT = 0
 
 SUPEROPT_GC = 0
 

ports/atmel-samd/boards/circuitplayground_express/mpconfigboard.mk

@@ -15,6 +15,7 @@ LONGINT_IMPL = MPZ
 CIRCUITPY_DISPLAYIO = 0
 CIRCUITPY_FREQUENCYIO = 0
 CIRCUITPY_I2CPERIPHERAL = 0
+MICROPY_PY_ASYNC_AWAIT = 0
 
 SUPEROPT_GC = 0
 CFLAGS_INLINE_LIMIT = 55

ports/atmel-samd/boards/feather_m0_rfm69/mpconfigboard.mk

@@ -22,6 +22,7 @@ CIRCUITPY_USB_MIDI = 0
 CIRCUITPY_USB_HID = 0
 CIRCUITPY_TOUCHIO = 0
 CFLAGS_INLINE_LIMIT = 35
+
 # Make more room.
 SUPEROPT_GC = 0
 

ports/atmel-samd/boards/feather_m0_rfm9x/mpconfigboard.mk

@@ -22,6 +22,7 @@ CIRCUITPY_SAMD = 0
 CIRCUITPY_USB_MIDI = 0
 CIRCUITPY_USB_HID = 0
 CIRCUITPY_TOUCHIO = 0
+
 CFLAGS_INLINE_LIMIT = 35
 # Make more room.
 SUPEROPT_GC = 0

ports/atmel-samd/boards/trinket_m0_haxpress/mpconfigboard.mk

@@ -17,6 +17,7 @@ CIRCUITPY_COUNTIO = 0
 CIRCUITPY_RTC = 0
 CIRCUITPY_FREQUENCYIO = 0
 CIRCUITPY_I2CPERIPHERAL = 0
+MICROPY_PY_ASYNC_AWAIT = 0
 
 SUPEROPT_GC = 0
 

ports/nrf/boards/pca10100/mpconfigboard.mk

@@ -24,6 +24,7 @@ CIRCUITPY_RTC = 1
 CIRCUITPY_SDCARDIO = 0
 CIRCUITPY_TOUCHIO = 0
 CIRCUITPY_ULAB = 0
+MICROPY_PY_ASYNC_AWAIT = 0
 
 SUPEROPT_GC = 0
 

ports/nrf/boards/sparkfun_nrf52840_mini/mpconfigboard.mk

@@ -6,3 +6,5 @@ USB_MANUFACTURER = "SparkFun Electronics"
 MCU_CHIP = nrf52840
 
 INTERNAL_FLASH_FILESYSTEM = 1
+
+MICROPY_PY_ASYNC_AWAIT = 0

py/circuitpy_mpconfig.h

@@ -79,7 +79,6 @@
 
 #define MICROPY_PY_ARRAY                 (1)
 #define MICROPY_PY_ARRAY_SLICE_ASSIGN    (1)
-#define MICROPY_PY_ASYNC_AWAIT           (0)
 #define MICROPY_PY_ATTRTUPLE             (1)
 
 #define MICROPY_PY_BUILTINS_BYTEARRAY    (1)

py/circuitpy_mpconfig.mk

@@ -32,6 +32,9 @@
 CIRCUITPY_FULL_BUILD ?= 1
 CFLAGS += -DCIRCUITPY_FULL_BUILD=$(CIRCUITPY_FULL_BUILD)
 
+# async/await language keyword support
+MICROPY_PY_ASYNC_AWAIT ?= $(CIRCUITPY_FULL_BUILD)
+CFLAGS += -DMICROPY_PY_ASYNC_AWAIT=$(MICROPY_PY_ASYNC_AWAIT)
 
 CIRCUITPY_AESIO ?= 0
 CFLAGS += -DCIRCUITPY_AESIO=$(CIRCUITPY_AESIO)

py/compile.c

@@ -853,7 +853,7 @@ STATIC void compile_decorated(compiler_t *comp, mp_parse_node_struct_t *pns) {
         mp_parse_node_struct_t *pns0 = (mp_parse_node_struct_t*)pns_body->nodes[0];
         body_name = compile_funcdef_helper(comp, pns0, emit_options);
         scope_t *fscope = (scope_t*)pns0->nodes[4];
-        fscope->scope_flags |= MP_SCOPE_FLAG_GENERATOR;
+        fscope->scope_flags |= MP_SCOPE_FLAG_GENERATOR | MP_SCOPE_FLAG_ASYNC;
     #endif
     } else {
         assert(MP_PARSE_NODE_STRUCT_KIND(pns_body) == PN_classdef); // should be
@@ -2632,6 +2632,12 @@ STATIC void compile_yield_expr(compiler_t *comp, mp_parse_node_struct_t *pns) {
         EMIT_ARG(yield, MP_EMIT_YIELD_VALUE);
     } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_yield_arg_from)) {
         pns = (mp_parse_node_struct_t*)pns->nodes[0];
+#if MICROPY_PY_ASYNC_AWAIT
+        if(comp->scope_cur->scope_flags & MP_SCOPE_FLAG_ASYNC) {
+            compile_syntax_error(comp, (mp_parse_node_t)pns, translate("'yield from' inside async function"));
+            return;
+        }
+#endif
         compile_node(comp, pns->nodes[0]);
         compile_yield_from(comp);
     } else {
@@ -2648,7 +2654,14 @@ STATIC void compile_atom_expr_await(compiler_t *comp, mp_parse_node_struct_t *pn
     }
     compile_require_async_context(comp, pns);
     compile_atom_expr_normal(comp, pns);
-    compile_yield_from(comp);
+
+    // If it's an awaitable thing, need to reach for the __await__ method for the coroutine.
+    // async def functions' __await__ return themselves, which are able to receive a send(),
+    // while other types with custom __await__ implementations return async generators.
+    EMIT_ARG(load_method, MP_QSTR___await__, false);
+    EMIT_ARG(call_method, 0, 0, 0);
+    EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE);
+    EMIT_ARG(yield, MP_EMIT_YIELD_FROM);
 }
 #endif
 

py/objgenerator.c

@@ -225,6 +225,21 @@ STATIC mp_obj_t gen_instance_send(mp_obj_t self_in, mp_obj_t send_value) {
 
 STATIC MP_DEFINE_CONST_FUN_OBJ_2(gen_instance_send_obj, gen_instance_send);
 
+#if MICROPY_PY_ASYNC_AWAIT
+STATIC mp_obj_t gen_instance_await(mp_obj_t self_in) {
+    mp_obj_gen_instance_t *self = MP_OBJ_TO_PTR(self_in);
+    if ( !self->coroutine_generator ) {
+        // Pretend like a generator does not have this coroutine behavior.
+        // Pay no attention to the dir() behind the curtain
+        nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_AttributeError,
+            translate("type object 'generator' has no attribute '__await__'")));
+    }
+    // You can directly call send on a coroutine generator or you can __await__ then send on the return of that.
+    return self;
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_1(gen_instance_await_obj, gen_instance_await);
+#endif
+
 STATIC mp_obj_t gen_instance_close(mp_obj_t self_in);
 STATIC mp_obj_t gen_instance_throw(size_t n_args, const mp_obj_t *args) {
     mp_obj_t exc = (n_args == 2) ? args[1] : args[2];
@@ -280,6 +295,9 @@ STATIC const mp_rom_map_elem_t gen_instance_locals_dict_table[] = {
     #if MICROPY_PY_GENERATOR_PEND_THROW
     { MP_ROM_QSTR(MP_QSTR_pend_throw), MP_ROM_PTR(&gen_instance_pend_throw_obj) },
     #endif
+    #if MICROPY_PY_ASYNC_AWAIT
+    { MP_ROM_QSTR(MP_QSTR___await__), MP_ROM_PTR(&gen_instance_await_obj) },
+    #endif
 };
 
 STATIC MP_DEFINE_CONST_DICT(gen_instance_locals_dict, gen_instance_locals_dict_table);

tests/basics/async_await2.py

@@ -1,22 +1,22 @@
 # test await expression
 
-import sys
-if sys.implementation.name in ('micropython', 'circuitpython'):
-    # uPy allows normal generators to be awaitables
-    coroutine = lambda f: f
-else:
-    import types
-    coroutine = types.coroutine
+# uPy allows normal generators to be awaitables.
+# CircuitPython does not.
+# In CircuitPython you need to have an __await__ method on an awaitable like in CPython;
+#  and like in CPython, generators do not have __await__.
 
-@coroutine
-def wait(value):
-    print('wait value:', value)
-    msg = yield 'message from wait(%u)' % value
-    print('wait got back:', msg)
-    return 10
+class Awaitable:
+    def __init__(self, value):
+        self.value = value
+
+    def __await__(self):
+        print('wait value:', self.value)
+        msg = yield 'message from wait(%u)' % self.value
+        print('wait got back:', msg)
+        return 10
 
 async def f():
-    x = await wait(1)**2
+    x = await Awaitable(1)**2
     print('x =', x)
 
 coro = f()

tests/basics/async_for2.py

@@ -1,19 +1,19 @@
 # test waiting within "async for" __anext__ function
 
-import sys
-if sys.implementation.name in ('micropython', 'circuitpython'):
-    # uPy allows normal generators to be awaitables
-    coroutine = lambda f: f
-else:
-    import types
-    coroutine = types.coroutine
-
-@coroutine
-def f(x):
-    print('f start:', x)
-    yield x + 1
-    yield x + 2
-    return x + 3
+# uPy allows normal generators to be awaitables.
+# CircuitPython does not.
+# In CircuitPython you need to have an __await__ method on an awaitable like in CPython;
+#  and like in CPython, generators do not have __await__.
+
+class Awaitable:
+    def __init__(self, x):
+        self.x = x
+
+    def __await__(self):
+        print('f start:', self.x)
+        yield self.x + 1
+        yield self.x + 2
+        return self.x + 3
 
 class ARange:
     def __init__(self, high):
@@ -27,7 +27,7 @@ def __aiter__(self):
 
     async def __anext__(self):
         print('anext')
-        print('f returned:', await f(20))
+        print('f returned:', await Awaitable(20))
         if self.cur < self.high:
             val = self.cur
             self.cur += 1

tests/basics/async_with2.py

@@ -1,32 +1,32 @@
 # test waiting within async with enter/exit functions
 
-import sys
-if sys.implementation.name in ('micropython', 'circuitpython'):
-    # uPy allows normal generators to be awaitables
-    coroutine = lambda f: f
-else:
-    import types
-    coroutine = types.coroutine
+# uPy allows normal generators to be awaitables.
+# CircuitPython does not.
+# In CircuitPython you need to have an __await__ method on an awaitable like in CPython;
+#  and like in CPython, generators do not have __await__.
 
-@coroutine
-def f(x):
-    print('f start:', x)
-    yield x + 1
-    yield x + 2
-    return x + 3
+class Awaitable:
+    def __init__(self, x):
+        self.x = x
+
+    def __await__(self):
+        print('f start:', self.x)
+        yield self.x + 1
+        yield self.x + 2
+        return self.x + 3
 
 class AContext:
     async def __aenter__(self):
         print('enter')
-        print('f returned:', await f(10))
+        print('f returned:', await Awaitable(10))
     async def __aexit__(self, exc_type, exc, tb):
         print('exit', exc_type, exc)
-        print('f returned:', await f(20))
+        print('f returned:', await Awaitable(20))
 
 async def coro():
     async with AContext():
         print('body start')
-        print('body f returned:', await f(30))
+        print('body f returned:', await Awaitable(30))
         print('body end')
 
 o = coro()