gh-107015: Remove async_hacks from the tokenizer (#107018)

Author: pablogsal

Doc/library/ast.rst

@@ -2146,7 +2146,7 @@ and classes for traversing abstract syntax trees:
    Currently ``major`` must equal to ``3``.  For example, setting
    ``feature_version=(3, 4)`` will allow the use of ``async`` and
    ``await`` as variable names.  The lowest supported version is
-   ``(3, 4)``; the highest is ``sys.version_info[0:2]``.
+   ``(3, 7)``; the highest is ``sys.version_info[0:2]``.
 
    If source contains a null character ('\0'), :exc:`ValueError` is raised.
 
@@ -2169,6 +2169,9 @@ and classes for traversing abstract syntax trees:
    .. versionchanged:: 3.8
       Added ``type_comments``, ``mode='func_type'`` and ``feature_version``.
 
+   .. versionchanged:: 3.13
+      The minimum supported version for feature_version is now (3,7)
+
 
 .. function:: unparse(ast_obj)
 

Doc/library/token-list.inc

@@ -80,17 +80,21 @@ the :mod:`tokenize` module.
 
 
 .. versionchanged:: 3.5
-   Added :data:`AWAIT` and :data:`ASYNC` tokens.
+   Added :data:`!AWAIT` and :data:`!ASYNC` tokens.
 
 .. versionchanged:: 3.7
    Added :data:`COMMENT`, :data:`NL` and :data:`ENCODING` tokens.
 
 .. versionchanged:: 3.7
-   Removed :data:`AWAIT` and :data:`ASYNC` tokens. "async" and "await" are
+   Removed :data:`!AWAIT` and :data:`!ASYNC` tokens. "async" and "await" are
    now tokenized as :data:`NAME` tokens.
 
 .. versionchanged:: 3.8
    Added :data:`TYPE_COMMENT`, :data:`TYPE_IGNORE`, :data:`COLONEQUAL`.
-   Added :data:`AWAIT` and :data:`ASYNC` tokens back (they're needed
+   Added :data:`!AWAIT` and :data:`!ASYNC` tokens back (they're needed
    to support parsing older Python versions for :func:`ast.parse` with
    ``feature_version`` set to 6 or lower).
+
+.. versionchanged:: 3.13
+   Removed :data:`!AWAIT` and :data:`!ASYNC` tokens again.
+

Doc/library/token.rst

@@ -56,8 +56,6 @@ COLONEQUAL              ':='
 EXCLAMATION             '!'
 
 OP
-AWAIT
-ASYNC
 TYPE_IGNORE
 TYPE_COMMENT
 SOFT_KEYWORD

Grammar/Tokens

@@ -127,11 +127,11 @@ simple_stmt[stmt_ty] (memo):
     | &'nonlocal' nonlocal_stmt
 
 compound_stmt[stmt_ty]:
-    | &('def' | '@' | ASYNC) function_def
+    | &('def' | '@' | 'async') function_def
     | &'if' if_stmt
     | &('class' | '@') class_def
-    | &('with' | ASYNC) with_stmt
-    | &('for' | ASYNC) for_stmt
+    | &('with' | 'async') with_stmt
+    | &('for' | 'async') for_stmt
     | &'try' try_stmt
     | &'while' while_stmt
     | match_stmt
@@ -272,7 +272,7 @@ function_def_raw[stmt_ty]:
         _PyAST_FunctionDef(n->v.Name.id,
                         (params) ? params : CHECK(arguments_ty, _PyPegen_empty_arguments(p)),
                         b, NULL, a, NEW_TYPE_COMMENT(p, tc), t, EXTRA) }
-    | ASYNC 'def' n=NAME t=[type_params] &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block {
+    | 'async' 'def' n=NAME t=[type_params] &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block {
         CHECK_VERSION(
             stmt_ty,
             5,
@@ -385,7 +385,7 @@ for_stmt[stmt_ty]:
     | invalid_for_stmt
     | 'for' t=star_targets 'in' ~ ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] {
         _PyAST_For(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA) }
-    | ASYNC 'for' t=star_targets 'in' ~ ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] {
+    | 'async' 'for' t=star_targets 'in' ~ ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] {
         CHECK_VERSION(stmt_ty, 5, "Async for loops are", _PyAST_AsyncFor(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA)) }
     | invalid_for_target
 
@@ -398,9 +398,9 @@ with_stmt[stmt_ty]:
         CHECK_VERSION(stmt_ty, 9, "Parenthesized context managers are", _PyAST_With(a, b, NULL, EXTRA)) }
     | 'with' a[asdl_withitem_seq*]=','.with_item+ ':' tc=[TYPE_COMMENT] b=block {
         _PyAST_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) }
-    | ASYNC 'with' '(' a[asdl_withitem_seq*]=','.with_item+ ','? ')' ':' b=block {
+    | 'async' 'with' '(' a[asdl_withitem_seq*]=','.with_item+ ','? ')' ':' b=block {
        CHECK_VERSION(stmt_ty, 5, "Async with statements are", _PyAST_AsyncWith(a, b, NULL, EXTRA)) }
-    | ASYNC 'with' a[asdl_withitem_seq*]=','.with_item+ ':' tc=[TYPE_COMMENT] b=block {
+    | 'async' 'with' a[asdl_withitem_seq*]=','.with_item+ ':' tc=[TYPE_COMMENT] b=block {
        CHECK_VERSION(stmt_ty, 5, "Async with statements are", _PyAST_AsyncWith(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA)) }
     | invalid_with_stmt
 
@@ -814,7 +814,7 @@ power[expr_ty]:
 # Primary elements are things like "obj.something.something", "obj[something]", "obj(something)", "obj" ...
 
 await_primary[expr_ty] (memo):
-    | AWAIT a=primary { CHECK_VERSION(expr_ty, 5, "Await expressions are", _PyAST_Await(a, EXTRA)) }
+    | 'await' a=primary { CHECK_VERSION(expr_ty, 5, "Await expressions are", _PyAST_Await(a, EXTRA)) }
     | primary
 
 primary[expr_ty]:
@@ -966,7 +966,7 @@ for_if_clauses[asdl_comprehension_seq*]:
     | a[asdl_comprehension_seq*]=for_if_clause+ { a }
 
 for_if_clause[comprehension_ty]:
-    | ASYNC 'for' a=star_targets 'in' ~ b=disjunction c[asdl_expr_seq*]=('if' z=disjunction { z })* {
+    | 'async' 'for' a=star_targets 'in' ~ b=disjunction c[asdl_expr_seq*]=('if' z=disjunction { z })* {
         CHECK_VERSION(comprehension_ty, 6, "Async comprehensions are", _PyAST_comprehension(a, b, c, 1, p->arena)) }
     | 'for' a=star_targets 'in' ~ b=disjunction c[asdl_expr_seq*]=('if' z=disjunction { z })* {
         _PyAST_comprehension(a, b, c, 0, p->arena) }
@@ -1284,7 +1284,7 @@ invalid_with_item:
         RAISE_SYNTAX_ERROR_INVALID_TARGET(STAR_TARGETS, a) }
 
 invalid_for_target:
-    | ASYNC? 'for' a=star_expressions {
+    | 'async'? 'for' a=star_expressions {
         RAISE_SYNTAX_ERROR_INVALID_TARGET(FOR_TARGETS, a) }
 
 invalid_group:
@@ -1301,12 +1301,12 @@ invalid_import_from_targets:
         RAISE_SYNTAX_ERROR("trailing comma not allowed without surrounding parentheses") }
 
 invalid_with_stmt:
-    | [ASYNC] 'with' ','.(expression ['as' star_target])+ NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
-    | [ASYNC] 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
+    | ['async'] 'with' ','.(expression ['as' star_target])+ NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
+    | ['async'] 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
 invalid_with_stmt_indent:
-    | [ASYNC] a='with' ','.(expression ['as' star_target])+ ':' NEWLINE !INDENT {
+    | ['async'] a='with' ','.(expression ['as' star_target])+ ':' NEWLINE !INDENT {
         RAISE_INDENTATION_ERROR("expected an indented block after 'with' statement on line %d", a->lineno) }
-    | [ASYNC] a='with' '(' ','.(expressions ['as' star_target])+ ','? ')' ':' NEWLINE !INDENT {
+    | ['async'] a='with' '(' ','.(expressions ['as' star_target])+ ','? ')' ':' NEWLINE !INDENT {
         RAISE_INDENTATION_ERROR("expected an indented block after 'with' statement on line %d", a->lineno) }
 
 invalid_try_stmt:
@@ -1367,11 +1367,11 @@ invalid_while_stmt:
     | a='while' named_expression ':' NEWLINE !INDENT {
         RAISE_INDENTATION_ERROR("expected an indented block after 'while' statement on line %d", a->lineno) }
 invalid_for_stmt:
-    | [ASYNC] 'for' star_targets 'in' star_expressions NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
-    | [ASYNC] a='for' star_targets 'in' star_expressions ':' NEWLINE !INDENT {
+    | ['async'] 'for' star_targets 'in' star_expressions NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
+    | ['async'] a='for' star_targets 'in' star_expressions ':' NEWLINE !INDENT {
         RAISE_INDENTATION_ERROR("expected an indented block after 'for' statement on line %d", a->lineno) }
 invalid_def_raw:
-    | [ASYNC] a='def' NAME '(' [params] ')' ['->' expression] ':' NEWLINE !INDENT {
+    | ['async'] a='def' NAME '(' [params] ')' ['->' expression] ':' NEWLINE !INDENT {
         RAISE_INDENTATION_ERROR("expected an indented block after function definition on line %d", a->lineno) }
 invalid_class_def_raw:
     | 'class' NAME ['(' [arguments] ')'] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }

Grammar/python.gram

@@ -69,18 +69,16 @@ extern "C" {
 #define COLONEQUAL      53
 #define EXCLAMATION     54
 #define OP              55
-#define AWAIT           56
-#define ASYNC           57
-#define TYPE_IGNORE     58
-#define TYPE_COMMENT    59
-#define SOFT_KEYWORD    60
-#define FSTRING_START   61
-#define FSTRING_MIDDLE  62
-#define FSTRING_END     63
-#define COMMENT         64
-#define NL              65
-#define ERRORTOKEN      66
-#define N_TOKENS        68
+#define TYPE_IGNORE     56
+#define TYPE_COMMENT    57
+#define SOFT_KEYWORD    58
+#define FSTRING_START   59
+#define FSTRING_MIDDLE  60
+#define FSTRING_END     61
+#define COMMENT         62
+#define NL              63
+#define ERRORTOKEN      64
+#define N_TOKENS        66
 #define NT_OFFSET       256
 
 /* Special definitions for cooperation with parser */

Include/internal/pycore_token.h

@@ -404,7 +404,7 @@ def test_ternary_operator(self) -> None:
             a='[' b=NAME c=for_if_clauses d=']' { _PyAST_ListComp(b, c, EXTRA) }
         )
         for_if_clauses[asdl_comprehension_seq*]: (
-            a[asdl_comprehension_seq*]=(y=[ASYNC] 'for' a=NAME 'in' b=NAME c[asdl_expr_seq*]=('if' z=NAME { z })*
+            a[asdl_comprehension_seq*]=(y=['async'] 'for' a=NAME 'in' b=NAME c[asdl_expr_seq*]=('if' z=NAME { z })*
                 { _PyAST_comprehension(_PyAST_Name(((expr_ty) a)->v.Name.id, Store, EXTRA), b, c, (y == NULL) ? 0 : 1, p->arena) })+ { a }
         )
         """