bpo-43244: Fix test_peg_generator for PyAST_Validate() (GH-24912)

test_peg_generator now defines _Py_TEST_PEGEN macro when building C
code to not call PyAST_Validate() in Parser/pegen.c. Moreover, it
defines Py_BUILD_CORE_MODULE macro to get access to the internal
C API.

Remove "global_ast_state" from Python-ast.c when it's built by
test_peg_generator: always get the AST state from the current interpreter.

Author: vstinner

Parser/asdl_c.py

@@ -1373,17 +1373,13 @@ def generate_ast_fini(module_state, f):
     f.write(textwrap.dedent("""
             void _PyAST_Fini(PyInterpreterState *interp)
             {
-            #ifdef Py_BUILD_CORE
                 struct ast_state *state = &interp->ast;
-            #else
-                struct ast_state *state = &global_ast_state;
-            #endif
 
     """))
     for s in module_state:
         f.write("    Py_CLEAR(state->" + s + ');\n')
     f.write(textwrap.dedent("""
-            #if defined(Py_BUILD_CORE) && !defined(NDEBUG)
+            #if !defined(NDEBUG)
                 state->initialized = -1;
             #else
                 state->initialized = 0;
@@ -1428,24 +1424,15 @@ def generate_module_def(mod, f, internal_h):
     generate_ast_state(module_state, internal_h)
 
     print(textwrap.dedent(f"""
-        #ifdef Py_BUILD_CORE
-        #  include "pycore_ast_state.h"     // struct ast_state
-        #  include "pycore_interp.h"        // _PyInterpreterState.ast
-        #  include "pycore_pystate.h"       // _PyInterpreterState_GET()
-        #else
-    """).strip(), file=f)
-
-    generate_ast_state(module_state, f)
-
-    print(textwrap.dedent(f"""
-        #endif   // Py_BUILD_CORE
+        #include "pycore_ast_state.h"       // struct ast_state
+        #include "pycore_interp.h"          // _PyInterpreterState.ast
+        #include "pycore_pystate.h"         // _PyInterpreterState_GET()
     """).rstrip(), file=f)
 
     f.write("""
 // Forward declaration
 static int init_types(struct ast_state *state);
 
-#ifdef Py_BUILD_CORE
 static struct ast_state*
 get_ast_state(void)
 {
@@ -1456,19 +1443,6 @@ def generate_module_def(mod, f, internal_h):
     }
     return state;
 }
-#else
-static struct ast_state global_ast_state;
-
-static struct ast_state*
-get_ast_state(void)
-{
-    struct ast_state *state = &global_ast_state;
-    if (!init_types(state)) {
-        return NULL;
-    }
-    return state;
-}
-#endif   // Py_BUILD_CORE
 """)
 
     # f-string for {mod.name}

Parser/pegen.c

@@ -1265,7 +1265,8 @@ _PyPegen_run_parser(Parser *p)
         return RAISE_SYNTAX_ERROR("multiple statements found while compiling a single statement");
     }
 
-#if defined(Py_DEBUG) && defined(Py_BUILD_CORE)
+    // test_peg_generator defines _Py_TEST_PEGEN to not call PyAST_Validate()
+#if defined(Py_DEBUG) && !defined(_Py_TEST_PEGEN)
     if (p->start_rule == Py_single_input ||
         p->start_rule == Py_file_input ||
         p->start_rule == Py_eval_input)

Python/Python-ast.c

@@ -56,6 +56,9 @@ def compile_c_extension(
     source_file_path = pathlib.Path(generated_source_path)
     extension_name = source_file_path.stem
     extra_compile_args = get_extra_flags("CFLAGS", "PY_CFLAGS_NODIST")
+    extra_compile_args.append("-DPy_BUILD_CORE_MODULE")
+    # Define _Py_TEST_PEGEN to not call PyAST_Validate() in Parser/pegen.c
+    extra_compile_args.append('-D_Py_TEST_PEGEN')
     extra_link_args = get_extra_flags("LDFLAGS", "PY_LDFLAGS_NODIST")
     if keep_asserts:
         extra_compile_args.append("-UNDEBUG")