From 63c911b6c9a418e5d4d86b02c0befc515c3442c5 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 11 May 2018 12:05:05 +0300 Subject: [PATCH 1/7] bpo-33346: Allow async comprehensions inside implicit asyns comprehensions. --- Lib/test/test_coroutines.py | 57 +++++++++++++++++++ .../2018-05-11-12-44-03.bpo-33346.ZgBkvB.rst | 3 + Python/compile.c | 7 ++- Python/symtable.c | 9 ++- 4 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2018-05-11-12-44-03.bpo-33346.ZgBkvB.rst diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 091b6626dcc8be..70db326af7214f 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -27,6 +27,12 @@ def __await__(self): yield self.value +async def asynciter(iterable): + """Convert an iterable to an asynchronous iterator.""" + for x in iterable: + yield x + + def run_async(coro): assert coro.__class__ in {types.GeneratorType, types.CoroutineType} @@ -124,6 +130,11 @@ def bar(): for c in b] """, + """async def foo(): + def bar(): + [[async for i in b] for b in els] + """, + """async def foo(): def bar(): [i for i in els @@ -2002,6 +2013,52 @@ async def f(): run_async(f()), ([], {1: 1, 2: 2, 3: 3})) + def test_nested_comp(self): + async def run_list_inside_list(): + return [[i + j async for i in asynciter([1, 2])] for j in [10, 20]] + self.assertEqual( + run_async(run_list_inside_list()), + ([], [[11, 12], [21, 22]])) + + async def run_set_inside_list(): + return [{i + j async for i in asynciter([1, 2])} for j in [10, 20]] + self.assertEqual( + run_async(run_set_inside_list()), + ([], [{11, 12}, {21, 22}])) + + async def run_list_inside_set(): + return {sum([i async for i in asynciter(range(j))]) for j in [3, 5]} + self.assertEqual( + run_async(run_list_inside_set()), + ([], {3, 10})) + + async def run_dict_inside_dict(): + return {j: {i: i + j async for i in asynciter([1, 2])} for j in [10, 20]} + self.assertEqual( + run_async(run_dict_inside_dict()), + ([], {10: {1: 11, 2: 12}, 20: {1: 21, 2: 22}})) + + async def run_list_inside_gen(): + gen = ([i + j async for i in asynciter([1, 2])] for j in [10, 20]) + return [x async for x in gen] + self.assertEqual( + run_async(run_list_inside_gen()), + ([], [[11, 12], [21, 22]])) + + async def run_gen_inside_list(): + gens = [(i async for i in asynciter(range(j))) for j in [3, 5]] + return [x for g in gens async for x in g] + self.assertEqual( + run_async(run_gen_inside_list()), + ([], [0, 1, 2, 0, 1, 2, 3, 4])) + + async def run_gen_inside_gen(): + gens = ((i async for i in asynciter(range(j))) for j in [3, 5]) + return [x for g in gens async for x in g] + self.assertEqual( + run_async(run_gen_inside_gen()), + ([], [0, 1, 2, 0, 1, 2, 3, 4])) + def test_copy(self): async def func(): pass coro = func() diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-05-11-12-44-03.bpo-33346.ZgBkvB.rst b/Misc/NEWS.d/next/Core and Builtins/2018-05-11-12-44-03.bpo-33346.ZgBkvB.rst new file mode 100644 index 00000000000000..9c91a8c0bf9ee4 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-05-11-12-44-03.bpo-33346.ZgBkvB.rst @@ -0,0 +1,3 @@ +Asynchronous comprehensions are now allowed inside comprehensions in +asynchronous functions. Outer comprehensions implicitly become +asynchronous. diff --git a/Python/compile.c b/Python/compile.c index 3528670ef67503..ae66ba3abd4001 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -4075,7 +4075,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, PyCodeObject *co = NULL; comprehension_ty outermost; PyObject *qualname = NULL; - int is_async_function = c->u->u_ste->ste_coroutine; + int scope_type = c->u->u_scope_type; int is_async_generator = 0; outermost = (comprehension_ty) asdl_seq_GET(generators, 0); @@ -4088,7 +4088,10 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, is_async_generator = c->u->u_ste->ste_coroutine; - if (is_async_generator && !is_async_function && type != COMP_GENEXP) { + if (is_async_generator && type != COMP_GENEXP && + scope_type != COMPILER_SCOPE_ASYNC_FUNCTION && + scope_type != COMPILER_SCOPE_COMPREHENSION) + { if (e->lineno > c->u->u_lineno) { c->u->u_lineno = e->lineno; c->u->u_lineno_set = 0; diff --git a/Python/symtable.c b/Python/symtable.c index 3e8c6f5dae30c9..4600ab64572a13 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1741,7 +1741,14 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, return 0; } st->st_cur->ste_generator = is_generator; - return symtable_exit_block(st, (void *)e); + int is_async = st->st_cur->ste_coroutine && !is_generator; + if (!symtable_exit_block(st, (void *)e)) { + return 0; + } + if (is_async) { + st->st_cur->ste_coroutine = 1; + } + return 1; } static int From fa40bd78c7846597739fad1a8e0febf099afb9c7 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 22 Sep 2018 14:04:29 +0300 Subject: [PATCH 2/7] Update documentation. --- Doc/reference/expressions.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index f4b16182829d49..2293b92b4b41a7 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -204,9 +204,9 @@ A comprehension in an :keyword:`async def` function may consist of either a :keyword:`for` or :keyword:`async for` clause following the leading expression, may contain additional :keyword:`for` or :keyword:`async for` clauses, and may also use :keyword:`await` expressions. -If a comprehension contains either :keyword:`async for` clauses -or :keyword:`await` expressions it is called an -:dfn:`asynchronous comprehension`. An asynchronous comprehension may +If a comprehension contains either :keyword:`async for` clauses or +:keyword:`await` expressions or other asynchronous comprehensions it is called +an :dfn:`asynchronous comprehension`. An asynchronous comprehension may suspend the execution of the coroutine function in which it appears. See also :pep:`530`. @@ -216,6 +216,11 @@ See also :pep:`530`. .. versionchanged:: 3.8 ``yield`` and ``yield from`` prohibited in the implicitly nested scope. +.. versionchanged:: 3.8 + Asynchronous comprehensions are now allowed inside comprehensions in + asynchronous functions. Outer comprehensions implicitly become + asynchronous. + .. _lists: From 70ace5b3accea9a9aefc01089400dc197f362ee6 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 7 Mar 2019 08:36:32 +0200 Subject: [PATCH 3/7] Add more tests. --- Lib/test/test_coroutines.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index af68edabb82289..701dcbc0aea5d7 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -209,6 +209,13 @@ def bar(): [i for i in els if await i] """, + """def bar(): + [[i async for i in a] for a in elts] + """, + + """[[i async for i in a] for a in elts] + """, + """async def foo(): await """, @@ -2058,6 +2065,14 @@ async def run_gen_inside_gen(): run_async(run_gen_inside_gen()), ([], [0, 1, 2, 0, 1, 2, 3, 4])) + async def run_list_inside_list_inside_list(): + return [[[i + j + k async for i in asynciter([1, 2])] + for j in [10, 20]] + for k in [100, 200]] + self.assertEqual( + run_async(run_list_inside_list_inside_list()), + ([], [[[111, 112], [121, 122]], [[211, 212], [221, 222]]])) + def test_copy(self): async def func(): pass coro = func() From d26b481e842ae06b479281d72d7579e688e30915 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 29 Jun 2020 18:09:23 +0300 Subject: [PATCH 4/7] Update version. --- Doc/reference/expressions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index d71c0598091fed..c0313e84e5c83e 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -226,7 +226,7 @@ See also :pep:`530`. .. versionchanged:: 3.8 ``yield`` and ``yield from`` prohibited in the implicitly nested scope. -.. versionchanged:: 3.8 +.. versionchanged:: 3.10 Asynchronous comprehensions are now allowed inside comprehensions in asynchronous functions. Outer comprehensions implicitly become asynchronous. From ccd55a4a89df92db816f0ad7ee17e4c7b976f645 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 29 Jun 2020 18:14:28 +0300 Subject: [PATCH 5/7] Add a What's New entry. --- Doc/whatsnew/3.10.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 0c4ff026bd201e..af724eb02d6f2b 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -70,6 +70,10 @@ Summary -- Release highlights New Features ============ +* Asynchronous comprehensions are now allowed inside comprehensions in + asynchronous functions. Outer comprehensions implicitly become + asynchronous. (Contributed by Serhiy Storchaka in :issue:`33346`.) + * The :class:`int` type has a new method :meth:`int.bit_count`, returning the number of ones in the binary expansion of a given integer, also known as the population count. (Contributed by Niklas Fiekas in :issue:`29882`.) From e21ae1adda49e28e6dfba27888f7df0150628f46 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 29 Jun 2020 21:36:27 +0300 Subject: [PATCH 6/7] Add support for PyCF_ALLOW_TOP_LEVEL_AWAIT. --- Python/compile.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Python/compile.c b/Python/compile.c index 1e9963e88ce8fc..47193f7d4859d9 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -4589,8 +4589,9 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, PyObject *qualname = NULL; int scope_type = c->u->u_scope_type; int is_async_generator = 0; + int is_top_level_await = IS_TOP_LEVEL_AWAIT(c); - if (IS_TOP_LEVEL_AWAIT(c)) { + if (is_top_level_await) { c->u->u_ste->ste_coroutine = 1; } @@ -4605,7 +4606,8 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, if (is_async_generator && type != COMP_GENEXP && scope_type != COMPILER_SCOPE_ASYNC_FUNCTION && - scope_type != COMPILER_SCOPE_COMPREHENSION) + scope_type != COMPILER_SCOPE_COMPREHENSION && + !is_top_level_await) { compiler_error(c, "asynchronous comprehension outside of " "an asynchronous function"); From 08f657d1ba31b6113840b6e5053aff0cd2202689 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 13 Jul 2021 21:01:44 +0100 Subject: [PATCH 7/7] Update docs --- Doc/reference/expressions.rst | 2 +- Doc/whatsnew/3.11.rst | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 5a6a4e696bff28..aaa21349076ba2 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -230,7 +230,7 @@ See also :pep:`530`. .. versionchanged:: 3.8 ``yield`` and ``yield from`` prohibited in the implicitly nested scope. -.. versionchanged:: 3.10 +.. versionchanged:: 3.11 Asynchronous comprehensions are now allowed inside comprehensions in asynchronous functions. Outer comprehensions implicitly become asynchronous. diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index bfadda1a881f2e..929817831ea02f 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -70,7 +70,9 @@ Summary -- Release highlights New Features ============ - +* Asynchronous comprehensions are now allowed inside comprehensions in + asynchronous functions. Outer comprehensions implicitly become + asynchronous. (Contributed by Serhiy Storchaka in :issue:`33346`.) Other Language Changes ======================