From 9bbc05e2dc4b251f1d1f0e7ee322edb9b044b62c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 16 Sep 2020 12:37:25 +0200 Subject: [PATCH 1/7] bpo-41796: Make _ast module state per interpreter The ast module internal state is now per interpreter. * Rename "astmodulestate" to "struct ast_state" * Add pycore_ast.h internal header: the ast_state structure is now declared in pycore_ast.h. * Add PyInterpreterState.ast (struct ast_state) * Remove get_ast_state() * Rename get_global_ast_state() to get_ast_state() * PyAST_obj2mod() now handles get_ast_state() failures --- Include/Python-ast.h | 2 +- Include/internal/pycore_ast.h | 234 ++++++++++ Include/internal/pycore_interp.h | 11 +- Makefile.pre.in | 2 + .../2020-10-29-12-49-08.bpo-41796.tkGdHq.rst | 2 + Parser/asdl_c.py | 169 ++++--- Python/Python-ast.c | 436 +++++------------- 7 files changed, 450 insertions(+), 406 deletions(-) create mode 100644 Include/internal/pycore_ast.h create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-10-29-12-49-08.bpo-41796.tkGdHq.rst diff --git a/Include/Python-ast.h b/Include/Python-ast.h index e14bab566fb5a2..fc9f65c97a2296 100644 --- a/Include/Python-ast.h +++ b/Include/Python-ast.h @@ -1,4 +1,4 @@ -/* File automatically generated by Parser/asdl_c.py. */ +// File automatically generated by Parser/asdl_c.py. #ifndef Py_PYTHON_AST_H #define Py_PYTHON_AST_H diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h new file mode 100644 index 00000000000000..058fbc0fcdcbb3 --- /dev/null +++ b/Include/internal/pycore_ast.h @@ -0,0 +1,234 @@ +// File automatically generated by Parser/asdl_c.py. + +#ifndef Py_INTERNAL_AST_H +#define Py_INTERNAL_AST_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +struct ast_state { + int initialized; + PyObject *AST_type; + PyObject *Add_singleton; + PyObject *Add_type; + PyObject *And_singleton; + PyObject *And_type; + PyObject *AnnAssign_type; + PyObject *Assert_type; + PyObject *Assign_type; + PyObject *AsyncFor_type; + PyObject *AsyncFunctionDef_type; + PyObject *AsyncWith_type; + PyObject *Attribute_type; + PyObject *AugAssign_type; + PyObject *Await_type; + PyObject *BinOp_type; + PyObject *BitAnd_singleton; + PyObject *BitAnd_type; + PyObject *BitOr_singleton; + PyObject *BitOr_type; + PyObject *BitXor_singleton; + PyObject *BitXor_type; + PyObject *BoolOp_type; + PyObject *Break_type; + PyObject *Call_type; + PyObject *ClassDef_type; + PyObject *Compare_type; + PyObject *Constant_type; + PyObject *Continue_type; + PyObject *Del_singleton; + PyObject *Del_type; + PyObject *Delete_type; + PyObject *DictComp_type; + PyObject *Dict_type; + PyObject *Div_singleton; + PyObject *Div_type; + PyObject *Eq_singleton; + PyObject *Eq_type; + PyObject *ExceptHandler_type; + PyObject *Expr_type; + PyObject *Expression_type; + PyObject *FloorDiv_singleton; + PyObject *FloorDiv_type; + PyObject *For_type; + PyObject *FormattedValue_type; + PyObject *FunctionDef_type; + PyObject *FunctionType_type; + PyObject *GeneratorExp_type; + PyObject *Global_type; + PyObject *GtE_singleton; + PyObject *GtE_type; + PyObject *Gt_singleton; + PyObject *Gt_type; + PyObject *IfExp_type; + PyObject *If_type; + PyObject *ImportFrom_type; + PyObject *Import_type; + PyObject *In_singleton; + PyObject *In_type; + PyObject *Interactive_type; + PyObject *Invert_singleton; + PyObject *Invert_type; + PyObject *IsNot_singleton; + PyObject *IsNot_type; + PyObject *Is_singleton; + PyObject *Is_type; + PyObject *JoinedStr_type; + PyObject *LShift_singleton; + PyObject *LShift_type; + PyObject *Lambda_type; + PyObject *ListComp_type; + PyObject *List_type; + PyObject *Load_singleton; + PyObject *Load_type; + PyObject *LtE_singleton; + PyObject *LtE_type; + PyObject *Lt_singleton; + PyObject *Lt_type; + PyObject *MatMult_singleton; + PyObject *MatMult_type; + PyObject *Mod_singleton; + PyObject *Mod_type; + PyObject *Module_type; + PyObject *Mult_singleton; + PyObject *Mult_type; + PyObject *Name_type; + PyObject *NamedExpr_type; + PyObject *Nonlocal_type; + PyObject *NotEq_singleton; + PyObject *NotEq_type; + PyObject *NotIn_singleton; + PyObject *NotIn_type; + PyObject *Not_singleton; + PyObject *Not_type; + PyObject *Or_singleton; + PyObject *Or_type; + PyObject *Pass_type; + PyObject *Pow_singleton; + PyObject *Pow_type; + PyObject *RShift_singleton; + PyObject *RShift_type; + PyObject *Raise_type; + PyObject *Return_type; + PyObject *SetComp_type; + PyObject *Set_type; + PyObject *Slice_type; + PyObject *Starred_type; + PyObject *Store_singleton; + PyObject *Store_type; + PyObject *Sub_singleton; + PyObject *Sub_type; + PyObject *Subscript_type; + PyObject *Try_type; + PyObject *Tuple_type; + PyObject *TypeIgnore_type; + PyObject *UAdd_singleton; + PyObject *UAdd_type; + PyObject *USub_singleton; + PyObject *USub_type; + PyObject *UnaryOp_type; + PyObject *While_type; + PyObject *With_type; + PyObject *YieldFrom_type; + PyObject *Yield_type; + PyObject *__dict__; + PyObject *__doc__; + PyObject *__module__; + PyObject *_attributes; + PyObject *_fields; + PyObject *alias_type; + PyObject *annotation; + PyObject *arg; + PyObject *arg_type; + PyObject *args; + PyObject *argtypes; + PyObject *arguments_type; + PyObject *asname; + PyObject *ast; + PyObject *attr; + PyObject *bases; + PyObject *body; + PyObject *boolop_type; + PyObject *cause; + PyObject *cmpop_type; + PyObject *col_offset; + PyObject *comparators; + PyObject *comprehension_type; + PyObject *context_expr; + PyObject *conversion; + PyObject *ctx; + PyObject *decorator_list; + PyObject *defaults; + PyObject *elt; + PyObject *elts; + PyObject *end_col_offset; + PyObject *end_lineno; + PyObject *exc; + PyObject *excepthandler_type; + PyObject *expr_context_type; + PyObject *expr_type; + PyObject *finalbody; + PyObject *format_spec; + PyObject *func; + PyObject *generators; + PyObject *handlers; + PyObject *id; + PyObject *ifs; + PyObject *is_async; + PyObject *items; + PyObject *iter; + PyObject *key; + PyObject *keys; + PyObject *keyword_type; + PyObject *keywords; + PyObject *kind; + PyObject *kw_defaults; + PyObject *kwarg; + PyObject *kwonlyargs; + PyObject *left; + PyObject *level; + PyObject *lineno; + PyObject *lower; + PyObject *mod_type; + PyObject *module; + PyObject *msg; + PyObject *name; + PyObject *names; + PyObject *op; + PyObject *operand; + PyObject *operator_type; + PyObject *ops; + PyObject *optional_vars; + PyObject *orelse; + PyObject *posonlyargs; + PyObject *returns; + PyObject *right; + PyObject *simple; + PyObject *slice; + PyObject *step; + PyObject *stmt_type; + PyObject *tag; + PyObject *target; + PyObject *targets; + PyObject *test; + PyObject *type; + PyObject *type_comment; + PyObject *type_ignore_type; + PyObject *type_ignores; + PyObject *unaryop_type; + PyObject *upper; + PyObject *value; + PyObject *values; + PyObject *vararg; + PyObject *withitem_type; +}; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_AST_H */ + diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index eee369a44bfc72..b3582f46953cce 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -8,10 +8,11 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_atomic.h" /* _Py_atomic_address */ -#include "pycore_gil.h" /* struct _gil_runtime_state */ -#include "pycore_gc.h" /* struct _gc_runtime_state */ -#include "pycore_warnings.h" /* struct _warnings_runtime_state */ +#include "pycore_atomic.h" // _Py_atomic_address +#include "pycore_ast.h" // struct ast_state +#include "pycore_gil.h" // struct _gil_runtime_state +#include "pycore_gc.h" // struct _gc_runtime_state +#include "pycore_warnings.h" // struct _warnings_runtime_state struct _pending_calls { PyThread_type_lock lock; @@ -258,6 +259,8 @@ struct _is { struct _Py_async_gen_state async_gen; struct _Py_context_state context; struct _Py_exc_state exc_state; + + struct ast_state ast; }; /* Used by _PyImport_Cleanup() */ diff --git a/Makefile.pre.in b/Makefile.pre.in index 5b6c0b9b62d334..a8912cd418470a 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -840,9 +840,11 @@ regen-ast: $(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \ $(srcdir)/Parser/Python.asdl \ -H $(srcdir)/Include/Python-ast.h.new \ + -I $(srcdir)/Include/internal/pycore_ast.h.new \ -C $(srcdir)/Python/Python-ast.c.new $(UPDATE_FILE) $(srcdir)/Include/Python-ast.h $(srcdir)/Include/Python-ast.h.new + $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_ast.h $(srcdir)/Include/internal/pycore_ast.h.new $(UPDATE_FILE) $(srcdir)/Python/Python-ast.c $(srcdir)/Python/Python-ast.c.new .PHONY: regen-opcode diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-10-29-12-49-08.bpo-41796.tkGdHq.rst b/Misc/NEWS.d/next/Core and Builtins/2020-10-29-12-49-08.bpo-41796.tkGdHq.rst new file mode 100644 index 00000000000000..b3ac08edd84fcd --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-10-29-12-49-08.bpo-41796.tkGdHq.rst @@ -0,0 +1,2 @@ +The :mod:`ast` module internal state is now per interpreter. Patch by Victor +Stinner. diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 481261cd85359a..757f6f11048e76 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -3,6 +3,7 @@ import os import sys +import textwrap from argparse import ArgumentParser from pathlib import Path @@ -11,7 +12,7 @@ TABSIZE = 4 MAX_COL = 80 -AUTOGEN_MESSAGE = "/* File automatically generated by {}. */\n\n" +AUTOGEN_MESSAGE = "// File automatically generated by {}.\n\n" def get_c_type(name): """Return a string for the C name of the type. @@ -414,7 +415,7 @@ def visitField(self, sum): class Obj2ModPrototypeVisitor(PickleVisitor): def visitProduct(self, prod, name): - code = "static int obj2ast_%s(astmodulestate *state, PyObject* obj, %s* out, PyArena* arena);" + code = "static int obj2ast_%s(struct ast_state *state, PyObject* obj, %s* out, PyArena* arena);" self.emit(code % (name, get_c_type(name)), 0) visitSum = visitProduct @@ -424,7 +425,7 @@ class Obj2ModVisitor(PickleVisitor): def funcHeader(self, name): ctype = get_c_type(name) self.emit("int", 0) - self.emit("obj2ast_%s(astmodulestate *state, PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0) + self.emit("obj2ast_%s(struct ast_state *state, PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0) self.emit("{", 0) self.emit("int isinstance;", 1) self.emit("", 0) @@ -506,7 +507,7 @@ def visitSum(self, sum, name): def visitProduct(self, prod, name): ctype = get_c_type(name) self.emit("int", 0) - self.emit("obj2ast_%s(astmodulestate *state, PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0) + self.emit("obj2ast_%s(struct ast_state *state, PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0) self.emit("{", 0) self.emit("PyObject* tmp = NULL;", 1) for f in prod.fields: @@ -640,7 +641,7 @@ class PyTypesDeclareVisitor(PickleVisitor): def visitProduct(self, prod, name): self.emit_type("%s_type" % name) - self.emit("static PyObject* ast2obj_%s(astmodulestate *state, void*);" % name, 0) + self.emit("static PyObject* ast2obj_%s(struct ast_state *state, void*);" % name, 0) if prod.attributes: for a in prod.attributes: self.emit_identifier(a.name) @@ -670,7 +671,7 @@ def visitSum(self, sum, name): ptype = get_c_type(name) for t in sum.types: self.emit_singleton("%s_singleton" % t.name) - self.emit("static PyObject* ast2obj_%s(astmodulestate *state, %s);" % (name, ptype), 0) + self.emit("static PyObject* ast2obj_%s(struct ast_state *state, %s);" % (name, ptype), 0) for t in sum.types: self.visitConstructor(t, name) @@ -725,7 +726,7 @@ def visitModule(self, mod): static int ast_type_init(PyObject *self, PyObject *args, PyObject *kw) { - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); if (state == NULL) { return -1; } @@ -801,7 +802,7 @@ def visitModule(self, mod): static PyObject * ast_type_reduce(PyObject *self, PyObject *unused) { - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); if (state == NULL) { return NULL; } @@ -856,7 +857,7 @@ def visitModule(self, mod): }; static PyObject * -make_type(astmodulestate *state, const char *type, PyObject* base, +make_type(struct ast_state *state, const char *type, PyObject* base, const char* const* fields, int num_fields, const char *doc) { PyObject *fnames, *result; @@ -882,7 +883,7 @@ def visitModule(self, mod): } static int -add_attributes(astmodulestate *state, PyObject *type, const char * const *attrs, int num_fields) +add_attributes(struct ast_state *state, PyObject *type, const char * const *attrs, int num_fields) { int i, result; PyObject *s, *l = PyTuple_New(num_fields); @@ -903,7 +904,7 @@ def visitModule(self, mod): /* Conversion AST -> Python */ -static PyObject* ast2obj_list(astmodulestate *state, asdl_seq *seq, PyObject* (*func)(astmodulestate *state, void*)) +static PyObject* ast2obj_list(struct ast_state *state, asdl_seq *seq, PyObject* (*func)(struct ast_state *state, void*)) { Py_ssize_t i, n = asdl_seq_LEN(seq); PyObject *result = PyList_New(n); @@ -921,7 +922,7 @@ def visitModule(self, mod): return result; } -static PyObject* ast2obj_object(astmodulestate *Py_UNUSED(state), void *o) +static PyObject* ast2obj_object(struct ast_state *Py_UNUSED(state), void *o) { if (!o) o = Py_None; @@ -932,14 +933,14 @@ def visitModule(self, mod): #define ast2obj_identifier ast2obj_object #define ast2obj_string ast2obj_object -static PyObject* ast2obj_int(astmodulestate *Py_UNUSED(state), long b) +static PyObject* ast2obj_int(struct ast_state *Py_UNUSED(state), long b) { return PyLong_FromLong(b); } /* Conversion Python -> AST */ -static int obj2ast_object(astmodulestate *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_object(struct ast_state *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) { if (obj == Py_None) obj = NULL; @@ -954,7 +955,7 @@ def visitModule(self, mod): return 0; } -static int obj2ast_constant(astmodulestate *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_constant(struct ast_state *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) { if (PyArena_AddPyObject(arena, obj) < 0) { *out = NULL; @@ -965,7 +966,7 @@ def visitModule(self, mod): return 0; } -static int obj2ast_identifier(astmodulestate *state, PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_identifier(struct ast_state *state, PyObject* obj, PyObject** out, PyArena* arena) { if (!PyUnicode_CheckExact(obj) && obj != Py_None) { PyErr_SetString(PyExc_TypeError, "AST identifier must be of type str"); @@ -974,7 +975,7 @@ def visitModule(self, mod): return obj2ast_object(state, obj, out, arena); } -static int obj2ast_string(astmodulestate *state, PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_string(struct ast_state *state, PyObject* obj, PyObject** out, PyArena* arena) { if (!PyUnicode_CheckExact(obj) && !PyBytes_CheckExact(obj)) { PyErr_SetString(PyExc_TypeError, "AST string must be of type str"); @@ -983,7 +984,7 @@ def visitModule(self, mod): return obj2ast_object(state, obj, out, arena); } -static int obj2ast_int(astmodulestate* Py_UNUSED(state), PyObject* obj, int* out, PyArena* arena) +static int obj2ast_int(struct ast_state* Py_UNUSED(state), PyObject* obj, int* out, PyArena* arena) { int i; if (!PyLong_Check(obj)) { @@ -998,7 +999,7 @@ def visitModule(self, mod): return 0; } -static int add_ast_fields(astmodulestate *state) +static int add_ast_fields(struct ast_state *state) { PyObject *empty_tuple; empty_tuple = PyTuple_New(0); @@ -1014,7 +1015,7 @@ def visitModule(self, mod): """, 0, reflow=False) - self.emit("static int init_types(astmodulestate *state)",0) + self.emit("static int init_types(struct ast_state *state)",0) self.emit("{", 0) self.emit("if (state->initialized) return 1;", 1) self.emit("if (init_identifiers(state) < 0) return 0;", 1) @@ -1093,12 +1094,7 @@ def visitModule(self, mod): self.emit("static int", 0) self.emit("astmodule_exec(PyObject *m)", 0) self.emit("{", 0) - self.emit('astmodulestate *state = get_ast_state(m);', 1) - self.emit("", 0) - - self.emit("if (!init_types(state)) {", 1) - self.emit("return -1;", 2) - self.emit("}", 1) + self.emit('struct ast_state *state = get_ast_state();', 1) self.emit('if (PyModule_AddObject(m, "AST", state->AST_type) < 0) {', 1) self.emit('return -1;', 2) self.emit('}', 1) @@ -1126,7 +1122,7 @@ def visitModule(self, mod): static struct PyModuleDef _astmodule = { PyModuleDef_HEAD_INIT, .m_name = "_ast", - // The _ast module uses a global state (global_ast_state). + // The _ast module uses a per-interpreter state (PyInterpreterState.ast) .m_size = 0, .m_slots = astmodule_slots, }; @@ -1169,7 +1165,7 @@ class ObjVisitor(PickleVisitor): def func_begin(self, name): ctype = get_c_type(name) self.emit("PyObject*", 0) - self.emit("ast2obj_%s(astmodulestate *state, void* _o)" % (name), 0) + self.emit("ast2obj_%s(struct ast_state *state, void* _o)" % (name), 0) self.emit("{", 0) self.emit("%s o = (%s)_o;" % (ctype, ctype), 1) self.emit("PyObject *result = NULL, *value = NULL;", 1) @@ -1206,7 +1202,7 @@ def visitSum(self, sum, name): self.func_end() def simpleSum(self, sum, name): - self.emit("PyObject* ast2obj_%s(astmodulestate *state, %s_ty o)" % (name, name), 0) + self.emit("PyObject* ast2obj_%s(struct ast_state *state, %s_ty o)" % (name, name), 0) self.emit("{", 0) self.emit("switch(o) {", 1) for t in sum.types: @@ -1280,7 +1276,7 @@ class PartingShots(StaticVisitor): CODE = """ PyObject* PyAST_mod2obj(mod_ty t) { - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); if (state == NULL) { return NULL; } @@ -1297,7 +1293,11 @@ class PartingShots(StaticVisitor): return NULL; } - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); + if (state == NULL) { + return NULL; + } + PyObject *req_type[3]; req_type[0] = state->Module_type; req_type[1] = state->Expression_type; @@ -1323,7 +1323,7 @@ class PartingShots(StaticVisitor): int PyAST_Check(PyObject* obj) { - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); if (state == NULL) { return -1; } @@ -1341,7 +1341,7 @@ def visit(self, object): v.emit("", 0) -def generate_module_def(f, mod): +def generate_module_def(mod, f, internal_h): # Gather all the data needed for ModuleSpec visitor_list = set() with open(os.devnull, "w") as devnull: @@ -1371,41 +1371,31 @@ def generate_module_def(f, mod): module_state.add(tp) state_strings = sorted(state_strings) module_state = sorted(module_state) - f.write('typedef struct {\n') - f.write(' int initialized;\n') + + internal_h.write('struct ast_state {\n') + internal_h.write(' int initialized;\n') for s in module_state: - f.write(' PyObject *' + s + ';\n') - f.write('} astmodulestate;\n\n') + internal_h.write(' PyObject *' + s + ';\n') + internal_h.write('};\n') + f.write(""" // Forward declaration -static int init_types(astmodulestate *state); +static int init_types(struct ast_state *state); -// bpo-41194, bpo-41261, bpo-41631: The _ast module uses a global state. -static astmodulestate global_ast_state = {0}; - -static astmodulestate* -get_global_ast_state(void) +static struct ast_state* +get_ast_state(void) { - astmodulestate* state = &global_ast_state; + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct ast_state *state = &interp->ast; if (!init_types(state)) { return NULL; } return state; } -static astmodulestate* -get_ast_state(PyObject* Py_UNUSED(module)) -{ - astmodulestate* state = get_global_ast_state(); - // get_ast_state() must only be called after _ast module is imported, - // and astmodule_exec() calls init_types() - assert(state != NULL); - return state; -} - void _PyAST_Fini(PyThreadState *tstate) { - astmodulestate* state = &global_ast_state; + struct ast_state* state = &tstate->interp->ast; """) for s in module_state: f.write(" Py_CLEAR(state->" + s + ');\n') @@ -1414,7 +1404,7 @@ def generate_module_def(f, mod): } """) - f.write('static int init_identifiers(astmodulestate *state)\n') + f.write('static int init_identifiers(struct ast_state *state)\n') f.write('{\n') for identifier in state_strings: f.write(' if ((state->' + identifier) @@ -1423,7 +1413,7 @@ def generate_module_def(f, mod): f.write(' return 1;\n') f.write('};\n\n') -def write_header(f, mod): +def write_header(mod, f): f.write('#ifndef Py_PYTHON_AST_H\n') f.write('#define Py_PYTHON_AST_H\n') f.write('#ifdef __cplusplus\n') @@ -1452,15 +1442,42 @@ def write_header(f, mod): f.write('#endif\n') f.write('#endif /* !Py_PYTHON_AST_H */\n') -def write_source(f, mod): + +def write_internal_h_header(mod, f): + print(textwrap.dedent(""" + #ifndef Py_INTERNAL_AST_H + #define Py_INTERNAL_AST_H + #ifdef __cplusplus + extern "C" { + #endif + + #ifndef Py_BUILD_CORE + # error "this header requires Py_BUILD_CORE define" + #endif + """).lstrip(), file=f) + + +def write_internal_h_footer(mod, f): + print(textwrap.dedent(""" + #ifdef __cplusplus + } + #endif + #endif /* !Py_INTERNAL_AST_H */ + """), file=f) + + +def write_source(mod, f, internal_h_file): f.write('#include \n') f.write('\n') f.write('#include "Python.h"\n') + f.write('#include "pycore_ast.h" // struct ast_state\n') + f.write('#include "pycore_interp.h" // _PyInterpreterState.ast\n') + f.write('#include "pycore_pystate.h" // _PyInterpreterState_GET()\n') f.write('#include "%s-ast.h"\n' % mod.name) f.write('#include "structmember.h" // PyMemberDef\n') f.write('\n') - generate_module_def(f, mod) + generate_module_def(mod, f, internal_h_file) v = ChainOfVisitors( SequenceConstructorVisitor(f), @@ -1475,27 +1492,37 @@ def write_source(f, mod): ) v.visit(mod) -def main(input_file, c_file, h_file, dump_module=False): +def main(input_filename, c_filename, h_filename, internal_h_filename, dump_module=False): auto_gen_msg = AUTOGEN_MESSAGE.format("/".join(Path(__file__).parts[-2:])) - mod = asdl.parse(input_file) + mod = asdl.parse(input_filename) if dump_module: print('Parsed Module:') print(mod) if not asdl.check(mod): sys.exit(1) - for file, writer in (c_file, write_source), (h_file, write_header): - if file is not None: - with file.open("w") as f: - f.write(auto_gen_msg) - writer(f, mod) - print(file, "regenerated.") + + with (c_filename.open("w") as c_file, + h_filename.open("w") as h_file, + internal_h_filename.open("w") as internal_h_file): + c_file.write(auto_gen_msg) + h_file.write(auto_gen_msg) + internal_h_file.write(auto_gen_msg) + + write_internal_h_header(mod, internal_h_file) + write_source(mod, c_file, internal_h_file) + write_header(mod, h_file) + write_internal_h_footer(mod, internal_h_file) + + print(f"{c_filename}, {h_filename}, {internal_h_filename} regenerated.") if __name__ == "__main__": parser = ArgumentParser() parser.add_argument("input_file", type=Path) - parser.add_argument("-C", "--c-file", type=Path, default=None) - parser.add_argument("-H", "--h-file", type=Path, default=None) + parser.add_argument("-C", "--c-file", type=Path, required=True) + parser.add_argument("-H", "--h-file", type=Path, required=True) + parser.add_argument("-I", "--internal-h-file", type=Path, required=True) parser.add_argument("-d", "--dump-module", action="store_true") - options = parser.parse_args() - main(**vars(options)) + args = parser.parse_args() + main(args.input_file, args.c_file, args.h_file, + args.internal_h_file, args.dump_module) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 13657a67275667..33869cbf638cd0 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -1,258 +1,32 @@ -/* File automatically generated by Parser/asdl_c.py. */ +// File automatically generated by Parser/asdl_c.py. #include #include "Python.h" +#include "pycore_ast.h" // struct ast_state +#include "pycore_interp.h" // _PyInterpreterState.ast +#include "pycore_pystate.h" // _PyInterpreterState_GET() #include "Python-ast.h" #include "structmember.h" // PyMemberDef -typedef struct { - int initialized; - PyObject *AST_type; - PyObject *Add_singleton; - PyObject *Add_type; - PyObject *And_singleton; - PyObject *And_type; - PyObject *AnnAssign_type; - PyObject *Assert_type; - PyObject *Assign_type; - PyObject *AsyncFor_type; - PyObject *AsyncFunctionDef_type; - PyObject *AsyncWith_type; - PyObject *Attribute_type; - PyObject *AugAssign_type; - PyObject *Await_type; - PyObject *BinOp_type; - PyObject *BitAnd_singleton; - PyObject *BitAnd_type; - PyObject *BitOr_singleton; - PyObject *BitOr_type; - PyObject *BitXor_singleton; - PyObject *BitXor_type; - PyObject *BoolOp_type; - PyObject *Break_type; - PyObject *Call_type; - PyObject *ClassDef_type; - PyObject *Compare_type; - PyObject *Constant_type; - PyObject *Continue_type; - PyObject *Del_singleton; - PyObject *Del_type; - PyObject *Delete_type; - PyObject *DictComp_type; - PyObject *Dict_type; - PyObject *Div_singleton; - PyObject *Div_type; - PyObject *Eq_singleton; - PyObject *Eq_type; - PyObject *ExceptHandler_type; - PyObject *Expr_type; - PyObject *Expression_type; - PyObject *FloorDiv_singleton; - PyObject *FloorDiv_type; - PyObject *For_type; - PyObject *FormattedValue_type; - PyObject *FunctionDef_type; - PyObject *FunctionType_type; - PyObject *GeneratorExp_type; - PyObject *Global_type; - PyObject *GtE_singleton; - PyObject *GtE_type; - PyObject *Gt_singleton; - PyObject *Gt_type; - PyObject *IfExp_type; - PyObject *If_type; - PyObject *ImportFrom_type; - PyObject *Import_type; - PyObject *In_singleton; - PyObject *In_type; - PyObject *Interactive_type; - PyObject *Invert_singleton; - PyObject *Invert_type; - PyObject *IsNot_singleton; - PyObject *IsNot_type; - PyObject *Is_singleton; - PyObject *Is_type; - PyObject *JoinedStr_type; - PyObject *LShift_singleton; - PyObject *LShift_type; - PyObject *Lambda_type; - PyObject *ListComp_type; - PyObject *List_type; - PyObject *Load_singleton; - PyObject *Load_type; - PyObject *LtE_singleton; - PyObject *LtE_type; - PyObject *Lt_singleton; - PyObject *Lt_type; - PyObject *MatMult_singleton; - PyObject *MatMult_type; - PyObject *Mod_singleton; - PyObject *Mod_type; - PyObject *Module_type; - PyObject *Mult_singleton; - PyObject *Mult_type; - PyObject *Name_type; - PyObject *NamedExpr_type; - PyObject *Nonlocal_type; - PyObject *NotEq_singleton; - PyObject *NotEq_type; - PyObject *NotIn_singleton; - PyObject *NotIn_type; - PyObject *Not_singleton; - PyObject *Not_type; - PyObject *Or_singleton; - PyObject *Or_type; - PyObject *Pass_type; - PyObject *Pow_singleton; - PyObject *Pow_type; - PyObject *RShift_singleton; - PyObject *RShift_type; - PyObject *Raise_type; - PyObject *Return_type; - PyObject *SetComp_type; - PyObject *Set_type; - PyObject *Slice_type; - PyObject *Starred_type; - PyObject *Store_singleton; - PyObject *Store_type; - PyObject *Sub_singleton; - PyObject *Sub_type; - PyObject *Subscript_type; - PyObject *Try_type; - PyObject *Tuple_type; - PyObject *TypeIgnore_type; - PyObject *UAdd_singleton; - PyObject *UAdd_type; - PyObject *USub_singleton; - PyObject *USub_type; - PyObject *UnaryOp_type; - PyObject *While_type; - PyObject *With_type; - PyObject *YieldFrom_type; - PyObject *Yield_type; - PyObject *__dict__; - PyObject *__doc__; - PyObject *__module__; - PyObject *_attributes; - PyObject *_fields; - PyObject *alias_type; - PyObject *annotation; - PyObject *arg; - PyObject *arg_type; - PyObject *args; - PyObject *argtypes; - PyObject *arguments_type; - PyObject *asname; - PyObject *ast; - PyObject *attr; - PyObject *bases; - PyObject *body; - PyObject *boolop_type; - PyObject *cause; - PyObject *cmpop_type; - PyObject *col_offset; - PyObject *comparators; - PyObject *comprehension_type; - PyObject *context_expr; - PyObject *conversion; - PyObject *ctx; - PyObject *decorator_list; - PyObject *defaults; - PyObject *elt; - PyObject *elts; - PyObject *end_col_offset; - PyObject *end_lineno; - PyObject *exc; - PyObject *excepthandler_type; - PyObject *expr_context_type; - PyObject *expr_type; - PyObject *finalbody; - PyObject *format_spec; - PyObject *func; - PyObject *generators; - PyObject *handlers; - PyObject *id; - PyObject *ifs; - PyObject *is_async; - PyObject *items; - PyObject *iter; - PyObject *key; - PyObject *keys; - PyObject *keyword_type; - PyObject *keywords; - PyObject *kind; - PyObject *kw_defaults; - PyObject *kwarg; - PyObject *kwonlyargs; - PyObject *left; - PyObject *level; - PyObject *lineno; - PyObject *lower; - PyObject *mod_type; - PyObject *module; - PyObject *msg; - PyObject *name; - PyObject *names; - PyObject *op; - PyObject *operand; - PyObject *operator_type; - PyObject *ops; - PyObject *optional_vars; - PyObject *orelse; - PyObject *posonlyargs; - PyObject *returns; - PyObject *right; - PyObject *simple; - PyObject *slice; - PyObject *step; - PyObject *stmt_type; - PyObject *tag; - PyObject *target; - PyObject *targets; - PyObject *test; - PyObject *type; - PyObject *type_comment; - PyObject *type_ignore_type; - PyObject *type_ignores; - PyObject *unaryop_type; - PyObject *upper; - PyObject *value; - PyObject *values; - PyObject *vararg; - PyObject *withitem_type; -} astmodulestate; - // Forward declaration -static int init_types(astmodulestate *state); - -// bpo-41194, bpo-41261, bpo-41631: The _ast module uses a global state. -static astmodulestate global_ast_state = {0}; +static int init_types(struct ast_state *state); -static astmodulestate* -get_global_ast_state(void) +static struct ast_state* +get_ast_state(void) { - astmodulestate* state = &global_ast_state; + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct ast_state *state = &interp->ast; if (!init_types(state)) { return NULL; } return state; } -static astmodulestate* -get_ast_state(PyObject* Py_UNUSED(module)) -{ - astmodulestate* state = get_global_ast_state(); - // get_ast_state() must only be called after _ast module is imported, - // and astmodule_exec() calls init_types() - assert(state != NULL); - return state; -} - void _PyAST_Fini(PyThreadState *tstate) { - astmodulestate* state = &global_ast_state; + struct ast_state* state = &tstate->interp->ast; Py_CLEAR(state->AST_type); Py_CLEAR(state->Add_singleton); Py_CLEAR(state->Add_type); @@ -470,7 +244,7 @@ void _PyAST_Fini(PyThreadState *tstate) state->initialized = 0; } -static int init_identifiers(astmodulestate *state) +static int init_identifiers(struct ast_state *state) { if ((state->__dict__ = PyUnicode_InternFromString("__dict__")) == NULL) return 0; if ((state->__doc__ = PyUnicode_InternFromString("__doc__")) == NULL) return 0; @@ -561,7 +335,7 @@ GENERATE_ASDL_SEQ_CONSTRUCTOR(alias, alias_ty) GENERATE_ASDL_SEQ_CONSTRUCTOR(withitem, withitem_ty) GENERATE_ASDL_SEQ_CONSTRUCTOR(type_ignore, type_ignore_ty) -static PyObject* ast2obj_mod(astmodulestate *state, void*); +static PyObject* ast2obj_mod(struct ast_state *state, void*); static const char * const Module_fields[]={ "body", "type_ignores", @@ -582,7 +356,7 @@ static const char * const stmt_attributes[] = { "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_stmt(astmodulestate *state, void*); +static PyObject* ast2obj_stmt(struct ast_state *state, void*); static const char * const FunctionDef_fields[]={ "name", "args", @@ -699,7 +473,7 @@ static const char * const expr_attributes[] = { "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_expr(astmodulestate *state, void*); +static PyObject* ast2obj_expr(struct ast_state *state, void*); static const char * const BoolOp_fields[]={ "op", "values", @@ -812,12 +586,12 @@ static const char * const Slice_fields[]={ "upper", "step", }; -static PyObject* ast2obj_expr_context(astmodulestate *state, expr_context_ty); -static PyObject* ast2obj_boolop(astmodulestate *state, boolop_ty); -static PyObject* ast2obj_operator(astmodulestate *state, operator_ty); -static PyObject* ast2obj_unaryop(astmodulestate *state, unaryop_ty); -static PyObject* ast2obj_cmpop(astmodulestate *state, cmpop_ty); -static PyObject* ast2obj_comprehension(astmodulestate *state, void*); +static PyObject* ast2obj_expr_context(struct ast_state *state, expr_context_ty); +static PyObject* ast2obj_boolop(struct ast_state *state, boolop_ty); +static PyObject* ast2obj_operator(struct ast_state *state, operator_ty); +static PyObject* ast2obj_unaryop(struct ast_state *state, unaryop_ty); +static PyObject* ast2obj_cmpop(struct ast_state *state, cmpop_ty); +static PyObject* ast2obj_comprehension(struct ast_state *state, void*); static const char * const comprehension_fields[]={ "target", "iter", @@ -830,13 +604,13 @@ static const char * const excepthandler_attributes[] = { "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_excepthandler(astmodulestate *state, void*); +static PyObject* ast2obj_excepthandler(struct ast_state *state, void*); static const char * const ExceptHandler_fields[]={ "type", "name", "body", }; -static PyObject* ast2obj_arguments(astmodulestate *state, void*); +static PyObject* ast2obj_arguments(struct ast_state *state, void*); static const char * const arguments_fields[]={ "posonlyargs", "args", @@ -846,7 +620,7 @@ static const char * const arguments_fields[]={ "kwarg", "defaults", }; -static PyObject* ast2obj_arg(astmodulestate *state, void*); +static PyObject* ast2obj_arg(struct ast_state *state, void*); static const char * const arg_attributes[] = { "lineno", "col_offset", @@ -858,7 +632,7 @@ static const char * const arg_fields[]={ "annotation", "type_comment", }; -static PyObject* ast2obj_keyword(astmodulestate *state, void*); +static PyObject* ast2obj_keyword(struct ast_state *state, void*); static const char * const keyword_attributes[] = { "lineno", "col_offset", @@ -869,17 +643,17 @@ static const char * const keyword_fields[]={ "arg", "value", }; -static PyObject* ast2obj_alias(astmodulestate *state, void*); +static PyObject* ast2obj_alias(struct ast_state *state, void*); static const char * const alias_fields[]={ "name", "asname", }; -static PyObject* ast2obj_withitem(astmodulestate *state, void*); +static PyObject* ast2obj_withitem(struct ast_state *state, void*); static const char * const withitem_fields[]={ "context_expr", "optional_vars", }; -static PyObject* ast2obj_type_ignore(astmodulestate *state, void*); +static PyObject* ast2obj_type_ignore(struct ast_state *state, void*); static const char * const TypeIgnore_fields[]={ "lineno", "tag", @@ -923,7 +697,7 @@ ast_clear(AST_object *self) static int ast_type_init(PyObject *self, PyObject *args, PyObject *kw) { - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); if (state == NULL) { return -1; } @@ -999,7 +773,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) static PyObject * ast_type_reduce(PyObject *self, PyObject *unused) { - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); if (state == NULL) { return NULL; } @@ -1054,7 +828,7 @@ static PyType_Spec AST_type_spec = { }; static PyObject * -make_type(astmodulestate *state, const char *type, PyObject* base, +make_type(struct ast_state *state, const char *type, PyObject* base, const char* const* fields, int num_fields, const char *doc) { PyObject *fnames, *result; @@ -1080,7 +854,7 @@ make_type(astmodulestate *state, const char *type, PyObject* base, } static int -add_attributes(astmodulestate *state, PyObject *type, const char * const *attrs, int num_fields) +add_attributes(struct ast_state *state, PyObject *type, const char * const *attrs, int num_fields) { int i, result; PyObject *s, *l = PyTuple_New(num_fields); @@ -1101,7 +875,7 @@ add_attributes(astmodulestate *state, PyObject *type, const char * const *attrs, /* Conversion AST -> Python */ -static PyObject* ast2obj_list(astmodulestate *state, asdl_seq *seq, PyObject* (*func)(astmodulestate *state, void*)) +static PyObject* ast2obj_list(struct ast_state *state, asdl_seq *seq, PyObject* (*func)(struct ast_state *state, void*)) { Py_ssize_t i, n = asdl_seq_LEN(seq); PyObject *result = PyList_New(n); @@ -1119,7 +893,7 @@ static PyObject* ast2obj_list(astmodulestate *state, asdl_seq *seq, PyObject* (* return result; } -static PyObject* ast2obj_object(astmodulestate *Py_UNUSED(state), void *o) +static PyObject* ast2obj_object(struct ast_state *Py_UNUSED(state), void *o) { if (!o) o = Py_None; @@ -1130,14 +904,14 @@ static PyObject* ast2obj_object(astmodulestate *Py_UNUSED(state), void *o) #define ast2obj_identifier ast2obj_object #define ast2obj_string ast2obj_object -static PyObject* ast2obj_int(astmodulestate *Py_UNUSED(state), long b) +static PyObject* ast2obj_int(struct ast_state *Py_UNUSED(state), long b) { return PyLong_FromLong(b); } /* Conversion Python -> AST */ -static int obj2ast_object(astmodulestate *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_object(struct ast_state *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) { if (obj == Py_None) obj = NULL; @@ -1152,7 +926,7 @@ static int obj2ast_object(astmodulestate *Py_UNUSED(state), PyObject* obj, PyObj return 0; } -static int obj2ast_constant(astmodulestate *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_constant(struct ast_state *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) { if (PyArena_AddPyObject(arena, obj) < 0) { *out = NULL; @@ -1163,7 +937,7 @@ static int obj2ast_constant(astmodulestate *Py_UNUSED(state), PyObject* obj, PyO return 0; } -static int obj2ast_identifier(astmodulestate *state, PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_identifier(struct ast_state *state, PyObject* obj, PyObject** out, PyArena* arena) { if (!PyUnicode_CheckExact(obj) && obj != Py_None) { PyErr_SetString(PyExc_TypeError, "AST identifier must be of type str"); @@ -1172,7 +946,7 @@ static int obj2ast_identifier(astmodulestate *state, PyObject* obj, PyObject** o return obj2ast_object(state, obj, out, arena); } -static int obj2ast_string(astmodulestate *state, PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_string(struct ast_state *state, PyObject* obj, PyObject** out, PyArena* arena) { if (!PyUnicode_CheckExact(obj) && !PyBytes_CheckExact(obj)) { PyErr_SetString(PyExc_TypeError, "AST string must be of type str"); @@ -1181,7 +955,7 @@ static int obj2ast_string(astmodulestate *state, PyObject* obj, PyObject** out, return obj2ast_object(state, obj, out, arena); } -static int obj2ast_int(astmodulestate* Py_UNUSED(state), PyObject* obj, int* out, PyArena* arena) +static int obj2ast_int(struct ast_state* Py_UNUSED(state), PyObject* obj, int* out, PyArena* arena) { int i; if (!PyLong_Check(obj)) { @@ -1196,7 +970,7 @@ static int obj2ast_int(astmodulestate* Py_UNUSED(state), PyObject* obj, int* out return 0; } -static int add_ast_fields(astmodulestate *state) +static int add_ast_fields(struct ast_state *state) { PyObject *empty_tuple; empty_tuple = PyTuple_New(0); @@ -1211,7 +985,7 @@ static int add_ast_fields(astmodulestate *state) } -static int init_types(astmodulestate *state) +static int init_types(struct ast_state *state) { if (state->initialized) return 1; if (init_identifiers(state) < 0) return 0; @@ -1890,37 +1664,37 @@ static int init_types(astmodulestate *state) return 1; } -static int obj2ast_mod(astmodulestate *state, PyObject* obj, mod_ty* out, +static int obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena); -static int obj2ast_stmt(astmodulestate *state, PyObject* obj, stmt_ty* out, +static int obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* arena); -static int obj2ast_expr(astmodulestate *state, PyObject* obj, expr_ty* out, +static int obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* arena); -static int obj2ast_expr_context(astmodulestate *state, PyObject* obj, +static int obj2ast_expr_context(struct ast_state *state, PyObject* obj, expr_context_ty* out, PyArena* arena); -static int obj2ast_boolop(astmodulestate *state, PyObject* obj, boolop_ty* out, - PyArena* arena); -static int obj2ast_operator(astmodulestate *state, PyObject* obj, operator_ty* - out, PyArena* arena); -static int obj2ast_unaryop(astmodulestate *state, PyObject* obj, unaryop_ty* +static int obj2ast_boolop(struct ast_state *state, PyObject* obj, boolop_ty* + out, PyArena* arena); +static int obj2ast_operator(struct ast_state *state, PyObject* obj, + operator_ty* out, PyArena* arena); +static int obj2ast_unaryop(struct ast_state *state, PyObject* obj, unaryop_ty* out, PyArena* arena); -static int obj2ast_cmpop(astmodulestate *state, PyObject* obj, cmpop_ty* out, +static int obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* arena); -static int obj2ast_comprehension(astmodulestate *state, PyObject* obj, +static int obj2ast_comprehension(struct ast_state *state, PyObject* obj, comprehension_ty* out, PyArena* arena); -static int obj2ast_excepthandler(astmodulestate *state, PyObject* obj, +static int obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* out, PyArena* arena); -static int obj2ast_arguments(astmodulestate *state, PyObject* obj, +static int obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, PyArena* arena); -static int obj2ast_arg(astmodulestate *state, PyObject* obj, arg_ty* out, +static int obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena); -static int obj2ast_keyword(astmodulestate *state, PyObject* obj, keyword_ty* +static int obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, PyArena* arena); -static int obj2ast_alias(astmodulestate *state, PyObject* obj, alias_ty* out, +static int obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* arena); -static int obj2ast_withitem(astmodulestate *state, PyObject* obj, withitem_ty* - out, PyArena* arena); -static int obj2ast_type_ignore(astmodulestate *state, PyObject* obj, +static int obj2ast_withitem(struct ast_state *state, PyObject* obj, + withitem_ty* out, PyArena* arena); +static int obj2ast_type_ignore(struct ast_state *state, PyObject* obj, type_ignore_ty* out, PyArena* arena); mod_ty @@ -3391,7 +3165,7 @@ TypeIgnore(int lineno, string tag, PyArena *arena) PyObject* -ast2obj_mod(astmodulestate *state, void* _o) +ast2obj_mod(struct ast_state *state, void* _o) { mod_ty o = (mod_ty)_o; PyObject *result = NULL, *value = NULL; @@ -3462,7 +3236,7 @@ ast2obj_mod(astmodulestate *state, void* _o) } PyObject* -ast2obj_stmt(astmodulestate *state, void* _o) +ast2obj_stmt(struct ast_state *state, void* _o) { stmt_ty o = (stmt_ty)_o; PyObject *result = NULL, *value = NULL; @@ -3977,7 +3751,7 @@ ast2obj_stmt(astmodulestate *state, void* _o) } PyObject* -ast2obj_expr(astmodulestate *state, void* _o) +ast2obj_expr(struct ast_state *state, void* _o) { expr_ty o = (expr_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4453,7 +4227,7 @@ ast2obj_expr(astmodulestate *state, void* _o) return NULL; } -PyObject* ast2obj_expr_context(astmodulestate *state, expr_context_ty o) +PyObject* ast2obj_expr_context(struct ast_state *state, expr_context_ty o) { switch(o) { case Load: @@ -4468,7 +4242,7 @@ PyObject* ast2obj_expr_context(astmodulestate *state, expr_context_ty o) } Py_UNREACHABLE(); } -PyObject* ast2obj_boolop(astmodulestate *state, boolop_ty o) +PyObject* ast2obj_boolop(struct ast_state *state, boolop_ty o) { switch(o) { case And: @@ -4480,7 +4254,7 @@ PyObject* ast2obj_boolop(astmodulestate *state, boolop_ty o) } Py_UNREACHABLE(); } -PyObject* ast2obj_operator(astmodulestate *state, operator_ty o) +PyObject* ast2obj_operator(struct ast_state *state, operator_ty o) { switch(o) { case Add: @@ -4525,7 +4299,7 @@ PyObject* ast2obj_operator(astmodulestate *state, operator_ty o) } Py_UNREACHABLE(); } -PyObject* ast2obj_unaryop(astmodulestate *state, unaryop_ty o) +PyObject* ast2obj_unaryop(struct ast_state *state, unaryop_ty o) { switch(o) { case Invert: @@ -4543,7 +4317,7 @@ PyObject* ast2obj_unaryop(astmodulestate *state, unaryop_ty o) } Py_UNREACHABLE(); } -PyObject* ast2obj_cmpop(astmodulestate *state, cmpop_ty o) +PyObject* ast2obj_cmpop(struct ast_state *state, cmpop_ty o) { switch(o) { case Eq: @@ -4580,7 +4354,7 @@ PyObject* ast2obj_cmpop(astmodulestate *state, cmpop_ty o) Py_UNREACHABLE(); } PyObject* -ast2obj_comprehension(astmodulestate *state, void* _o) +ast2obj_comprehension(struct ast_state *state, void* _o) { comprehension_ty o = (comprehension_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4619,7 +4393,7 @@ ast2obj_comprehension(astmodulestate *state, void* _o) } PyObject* -ast2obj_excepthandler(astmodulestate *state, void* _o) +ast2obj_excepthandler(struct ast_state *state, void* _o) { excepthandler_ty o = (excepthandler_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4678,7 +4452,7 @@ ast2obj_excepthandler(astmodulestate *state, void* _o) } PyObject* -ast2obj_arguments(astmodulestate *state, void* _o) +ast2obj_arguments(struct ast_state *state, void* _o) { arguments_ty o = (arguments_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4732,7 +4506,7 @@ ast2obj_arguments(astmodulestate *state, void* _o) } PyObject* -ast2obj_arg(astmodulestate *state, void* _o) +ast2obj_arg(struct ast_state *state, void* _o) { arg_ty o = (arg_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4786,7 +4560,7 @@ ast2obj_arg(astmodulestate *state, void* _o) } PyObject* -ast2obj_keyword(astmodulestate *state, void* _o) +ast2obj_keyword(struct ast_state *state, void* _o) { keyword_ty o = (keyword_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4835,7 +4609,7 @@ ast2obj_keyword(astmodulestate *state, void* _o) } PyObject* -ast2obj_alias(astmodulestate *state, void* _o) +ast2obj_alias(struct ast_state *state, void* _o) { alias_ty o = (alias_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4864,7 +4638,7 @@ ast2obj_alias(astmodulestate *state, void* _o) } PyObject* -ast2obj_withitem(astmodulestate *state, void* _o) +ast2obj_withitem(struct ast_state *state, void* _o) { withitem_ty o = (withitem_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4893,7 +4667,7 @@ ast2obj_withitem(astmodulestate *state, void* _o) } PyObject* -ast2obj_type_ignore(astmodulestate *state, void* _o) +ast2obj_type_ignore(struct ast_state *state, void* _o) { type_ignore_ty o = (type_ignore_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4927,7 +4701,7 @@ ast2obj_type_ignore(astmodulestate *state, void* _o) int -obj2ast_mod(astmodulestate *state, PyObject* obj, mod_ty* out, PyArena* arena) +obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) { int isinstance; @@ -5154,7 +4928,8 @@ obj2ast_mod(astmodulestate *state, PyObject* obj, mod_ty* out, PyArena* arena) } int -obj2ast_stmt(astmodulestate *state, PyObject* obj, stmt_ty* out, PyArena* arena) +obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* + arena) { int isinstance; @@ -7052,7 +6827,8 @@ obj2ast_stmt(astmodulestate *state, PyObject* obj, stmt_ty* out, PyArena* arena) } int -obj2ast_expr(astmodulestate *state, PyObject* obj, expr_ty* out, PyArena* arena) +obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* + arena) { int isinstance; @@ -8562,7 +8338,7 @@ obj2ast_expr(astmodulestate *state, PyObject* obj, expr_ty* out, PyArena* arena) } int -obj2ast_expr_context(astmodulestate *state, PyObject* obj, expr_context_ty* +obj2ast_expr_context(struct ast_state *state, PyObject* obj, expr_context_ty* out, PyArena* arena) { int isinstance; @@ -8597,7 +8373,7 @@ obj2ast_expr_context(astmodulestate *state, PyObject* obj, expr_context_ty* } int -obj2ast_boolop(astmodulestate *state, PyObject* obj, boolop_ty* out, PyArena* +obj2ast_boolop(struct ast_state *state, PyObject* obj, boolop_ty* out, PyArena* arena) { int isinstance; @@ -8624,7 +8400,7 @@ obj2ast_boolop(astmodulestate *state, PyObject* obj, boolop_ty* out, PyArena* } int -obj2ast_operator(astmodulestate *state, PyObject* obj, operator_ty* out, +obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, PyArena* arena) { int isinstance; @@ -8739,8 +8515,8 @@ obj2ast_operator(astmodulestate *state, PyObject* obj, operator_ty* out, } int -obj2ast_unaryop(astmodulestate *state, PyObject* obj, unaryop_ty* out, PyArena* - arena) +obj2ast_unaryop(struct ast_state *state, PyObject* obj, unaryop_ty* out, + PyArena* arena) { int isinstance; @@ -8782,7 +8558,7 @@ obj2ast_unaryop(astmodulestate *state, PyObject* obj, unaryop_ty* out, PyArena* } int -obj2ast_cmpop(astmodulestate *state, PyObject* obj, cmpop_ty* out, PyArena* +obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* arena) { int isinstance; @@ -8873,7 +8649,7 @@ obj2ast_cmpop(astmodulestate *state, PyObject* obj, cmpop_ty* out, PyArena* } int -obj2ast_comprehension(astmodulestate *state, PyObject* obj, comprehension_ty* +obj2ast_comprehension(struct ast_state *state, PyObject* obj, comprehension_ty* out, PyArena* arena) { PyObject* tmp = NULL; @@ -8962,7 +8738,7 @@ obj2ast_comprehension(astmodulestate *state, PyObject* obj, comprehension_ty* } int -obj2ast_excepthandler(astmodulestate *state, PyObject* obj, excepthandler_ty* +obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* out, PyArena* arena) { int isinstance; @@ -9112,7 +8888,7 @@ obj2ast_excepthandler(astmodulestate *state, PyObject* obj, excepthandler_ty* } int -obj2ast_arguments(astmodulestate *state, PyObject* obj, arguments_ty* out, +obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, PyArena* arena) { PyObject* tmp = NULL; @@ -9324,7 +9100,7 @@ obj2ast_arguments(astmodulestate *state, PyObject* obj, arguments_ty* out, } int -obj2ast_arg(astmodulestate *state, PyObject* obj, arg_ty* out, PyArena* arena) +obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) { PyObject* tmp = NULL; identifier arg; @@ -9435,8 +9211,8 @@ obj2ast_arg(astmodulestate *state, PyObject* obj, arg_ty* out, PyArena* arena) } int -obj2ast_keyword(astmodulestate *state, PyObject* obj, keyword_ty* out, PyArena* - arena) +obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, + PyArena* arena) { PyObject* tmp = NULL; identifier arg; @@ -9533,7 +9309,7 @@ obj2ast_keyword(astmodulestate *state, PyObject* obj, keyword_ty* out, PyArena* } int -obj2ast_alias(astmodulestate *state, PyObject* obj, alias_ty* out, PyArena* +obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* arena) { PyObject* tmp = NULL; @@ -9574,7 +9350,7 @@ obj2ast_alias(astmodulestate *state, PyObject* obj, alias_ty* out, PyArena* } int -obj2ast_withitem(astmodulestate *state, PyObject* obj, withitem_ty* out, +obj2ast_withitem(struct ast_state *state, PyObject* obj, withitem_ty* out, PyArena* arena) { PyObject* tmp = NULL; @@ -9615,8 +9391,8 @@ obj2ast_withitem(astmodulestate *state, PyObject* obj, withitem_ty* out, } int -obj2ast_type_ignore(astmodulestate *state, PyObject* obj, type_ignore_ty* out, - PyArena* arena) +obj2ast_type_ignore(struct ast_state *state, PyObject* obj, type_ignore_ty* + out, PyArena* arena) { int isinstance; @@ -9677,11 +9453,7 @@ obj2ast_type_ignore(astmodulestate *state, PyObject* obj, type_ignore_ty* out, static int astmodule_exec(PyObject *m) { - astmodulestate *state = get_ast_state(m); - - if (!init_types(state)) { - return -1; - } + struct ast_state *state = get_ast_state(); if (PyModule_AddObject(m, "AST", state->AST_type) < 0) { return -1; } @@ -10132,7 +9904,7 @@ static PyModuleDef_Slot astmodule_slots[] = { static struct PyModuleDef _astmodule = { PyModuleDef_HEAD_INIT, .m_name = "_ast", - // The _ast module uses a global state (global_ast_state). + // The _ast module uses a per-interpreter state (PyInterpreterState.ast) .m_size = 0, .m_slots = astmodule_slots, }; @@ -10146,7 +9918,7 @@ PyInit__ast(void) PyObject* PyAST_mod2obj(mod_ty t) { - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); if (state == NULL) { return NULL; } @@ -10163,7 +9935,11 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) return NULL; } - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); + if (state == NULL) { + return NULL; + } + PyObject *req_type[3]; req_type[0] = state->Module_type; req_type[1] = state->Expression_type; @@ -10189,7 +9965,7 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) int PyAST_Check(PyObject* obj) { - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); if (state == NULL) { return -1; } From 6ff45d6245641650f828caf7fa1e0a8ef13bb44a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 29 Oct 2020 13:16:32 +0100 Subject: [PATCH 2/7] Fix test_peg_generator --- Parser/asdl_c.py | 64 +++++++++--- Python/Python-ast.c | 250 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 295 insertions(+), 19 deletions(-) diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 757f6f11048e76..a110a7dcd1d4c2 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1341,6 +1341,13 @@ def visit(self, object): v.emit("", 0) +def generate_ast_state(module_state, f): + f.write('struct ast_state {\n') + f.write(' int initialized;\n') + for s in module_state: + f.write(' PyObject *' + s + ';\n') + f.write('};') + def generate_module_def(mod, f, internal_h): # Gather all the data needed for ModuleSpec visitor_list = set() @@ -1372,16 +1379,27 @@ def generate_module_def(mod, f, internal_h): state_strings = sorted(state_strings) module_state = sorted(module_state) - internal_h.write('struct ast_state {\n') - internal_h.write(' int initialized;\n') - for s in module_state: - internal_h.write(' PyObject *' + s + ';\n') - internal_h.write('};\n') + generate_ast_state(module_state, internal_h) + + print(textwrap.dedent(f""" + #ifdef Py_BUILD_CORE + # include "pycore_ast.h" // struct ast_state + # include "pycore_interp.h" // _PyInterpreterState.ast + # include "pycore_pystate.h" // _PyInterpreterState_GET() + #else + """).strip(), file=f) + + generate_ast_state(module_state, f) + + print(textwrap.dedent(f""" + #endif + """).rstrip(), file=f) f.write(""" // Forward declaration static int init_types(struct ast_state *state); +#ifdef Py_BUILD_CORE static struct ast_state* get_ast_state(void) { @@ -1392,10 +1410,29 @@ def generate_module_def(mod, f, internal_h): } return state; } +#else +static struct ast_state global_ast_state; + +static struct ast_state* +get_ast_state(void) +{ + struct ast_state *state = &global_ast_state; + if (!init_types(state)) { + return NULL; + } + return state; +} +#endif + void _PyAST_Fini(PyThreadState *tstate) { +#ifdef Py_BUILD_CORE struct ast_state* state = &tstate->interp->ast; +#else + struct ast_state *state = &global_ast_state; +#endif + """) for s in module_state: f.write(" Py_CLEAR(state->" + s + ');\n') @@ -1459,6 +1496,7 @@ def write_internal_h_header(mod, f): def write_internal_h_footer(mod, f): print(textwrap.dedent(""" + #ifdef __cplusplus } #endif @@ -1467,15 +1505,13 @@ def write_internal_h_footer(mod, f): def write_source(mod, f, internal_h_file): - f.write('#include \n') - f.write('\n') - f.write('#include "Python.h"\n') - f.write('#include "pycore_ast.h" // struct ast_state\n') - f.write('#include "pycore_interp.h" // _PyInterpreterState.ast\n') - f.write('#include "pycore_pystate.h" // _PyInterpreterState_GET()\n') - f.write('#include "%s-ast.h"\n' % mod.name) - f.write('#include "structmember.h" // PyMemberDef\n') - f.write('\n') + print(textwrap.dedent(f""" + #include + + #include "Python.h" + #include "{mod.name}-ast.h" + #include "structmember.h" + """), file=f) generate_module_def(mod, f, internal_h_file) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 33869cbf638cd0..277fdc51e243d3 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -1,18 +1,239 @@ // File automatically generated by Parser/asdl_c.py. + #include #include "Python.h" -#include "pycore_ast.h" // struct ast_state -#include "pycore_interp.h" // _PyInterpreterState.ast -#include "pycore_pystate.h" // _PyInterpreterState_GET() #include "Python-ast.h" -#include "structmember.h" // PyMemberDef - +#include "structmember.h" + +#ifdef Py_BUILD_CORE +# include "pycore_ast.h" // struct ast_state +# include "pycore_interp.h" // _PyInterpreterState.ast +# include "pycore_pystate.h" // _PyInterpreterState_GET() +#else +struct ast_state { + int initialized; + PyObject *AST_type; + PyObject *Add_singleton; + PyObject *Add_type; + PyObject *And_singleton; + PyObject *And_type; + PyObject *AnnAssign_type; + PyObject *Assert_type; + PyObject *Assign_type; + PyObject *AsyncFor_type; + PyObject *AsyncFunctionDef_type; + PyObject *AsyncWith_type; + PyObject *Attribute_type; + PyObject *AugAssign_type; + PyObject *Await_type; + PyObject *BinOp_type; + PyObject *BitAnd_singleton; + PyObject *BitAnd_type; + PyObject *BitOr_singleton; + PyObject *BitOr_type; + PyObject *BitXor_singleton; + PyObject *BitXor_type; + PyObject *BoolOp_type; + PyObject *Break_type; + PyObject *Call_type; + PyObject *ClassDef_type; + PyObject *Compare_type; + PyObject *Constant_type; + PyObject *Continue_type; + PyObject *Del_singleton; + PyObject *Del_type; + PyObject *Delete_type; + PyObject *DictComp_type; + PyObject *Dict_type; + PyObject *Div_singleton; + PyObject *Div_type; + PyObject *Eq_singleton; + PyObject *Eq_type; + PyObject *ExceptHandler_type; + PyObject *Expr_type; + PyObject *Expression_type; + PyObject *FloorDiv_singleton; + PyObject *FloorDiv_type; + PyObject *For_type; + PyObject *FormattedValue_type; + PyObject *FunctionDef_type; + PyObject *FunctionType_type; + PyObject *GeneratorExp_type; + PyObject *Global_type; + PyObject *GtE_singleton; + PyObject *GtE_type; + PyObject *Gt_singleton; + PyObject *Gt_type; + PyObject *IfExp_type; + PyObject *If_type; + PyObject *ImportFrom_type; + PyObject *Import_type; + PyObject *In_singleton; + PyObject *In_type; + PyObject *Interactive_type; + PyObject *Invert_singleton; + PyObject *Invert_type; + PyObject *IsNot_singleton; + PyObject *IsNot_type; + PyObject *Is_singleton; + PyObject *Is_type; + PyObject *JoinedStr_type; + PyObject *LShift_singleton; + PyObject *LShift_type; + PyObject *Lambda_type; + PyObject *ListComp_type; + PyObject *List_type; + PyObject *Load_singleton; + PyObject *Load_type; + PyObject *LtE_singleton; + PyObject *LtE_type; + PyObject *Lt_singleton; + PyObject *Lt_type; + PyObject *MatMult_singleton; + PyObject *MatMult_type; + PyObject *Mod_singleton; + PyObject *Mod_type; + PyObject *Module_type; + PyObject *Mult_singleton; + PyObject *Mult_type; + PyObject *Name_type; + PyObject *NamedExpr_type; + PyObject *Nonlocal_type; + PyObject *NotEq_singleton; + PyObject *NotEq_type; + PyObject *NotIn_singleton; + PyObject *NotIn_type; + PyObject *Not_singleton; + PyObject *Not_type; + PyObject *Or_singleton; + PyObject *Or_type; + PyObject *Pass_type; + PyObject *Pow_singleton; + PyObject *Pow_type; + PyObject *RShift_singleton; + PyObject *RShift_type; + PyObject *Raise_type; + PyObject *Return_type; + PyObject *SetComp_type; + PyObject *Set_type; + PyObject *Slice_type; + PyObject *Starred_type; + PyObject *Store_singleton; + PyObject *Store_type; + PyObject *Sub_singleton; + PyObject *Sub_type; + PyObject *Subscript_type; + PyObject *Try_type; + PyObject *Tuple_type; + PyObject *TypeIgnore_type; + PyObject *UAdd_singleton; + PyObject *UAdd_type; + PyObject *USub_singleton; + PyObject *USub_type; + PyObject *UnaryOp_type; + PyObject *While_type; + PyObject *With_type; + PyObject *YieldFrom_type; + PyObject *Yield_type; + PyObject *__dict__; + PyObject *__doc__; + PyObject *__module__; + PyObject *_attributes; + PyObject *_fields; + PyObject *alias_type; + PyObject *annotation; + PyObject *arg; + PyObject *arg_type; + PyObject *args; + PyObject *argtypes; + PyObject *arguments_type; + PyObject *asname; + PyObject *ast; + PyObject *attr; + PyObject *bases; + PyObject *body; + PyObject *boolop_type; + PyObject *cause; + PyObject *cmpop_type; + PyObject *col_offset; + PyObject *comparators; + PyObject *comprehension_type; + PyObject *context_expr; + PyObject *conversion; + PyObject *ctx; + PyObject *decorator_list; + PyObject *defaults; + PyObject *elt; + PyObject *elts; + PyObject *end_col_offset; + PyObject *end_lineno; + PyObject *exc; + PyObject *excepthandler_type; + PyObject *expr_context_type; + PyObject *expr_type; + PyObject *finalbody; + PyObject *format_spec; + PyObject *func; + PyObject *generators; + PyObject *handlers; + PyObject *id; + PyObject *ifs; + PyObject *is_async; + PyObject *items; + PyObject *iter; + PyObject *key; + PyObject *keys; + PyObject *keyword_type; + PyObject *keywords; + PyObject *kind; + PyObject *kw_defaults; + PyObject *kwarg; + PyObject *kwonlyargs; + PyObject *left; + PyObject *level; + PyObject *lineno; + PyObject *lower; + PyObject *mod_type; + PyObject *module; + PyObject *msg; + PyObject *name; + PyObject *names; + PyObject *op; + PyObject *operand; + PyObject *operator_type; + PyObject *ops; + PyObject *optional_vars; + PyObject *orelse; + PyObject *posonlyargs; + PyObject *returns; + PyObject *right; + PyObject *simple; + PyObject *slice; + PyObject *step; + PyObject *stmt_type; + PyObject *tag; + PyObject *target; + PyObject *targets; + PyObject *test; + PyObject *type; + PyObject *type_comment; + PyObject *type_ignore_type; + PyObject *type_ignores; + PyObject *unaryop_type; + PyObject *upper; + PyObject *value; + PyObject *values; + PyObject *vararg; + PyObject *withitem_type; +}; +#endif // Forward declaration static int init_types(struct ast_state *state); +#ifdef Py_BUILD_CORE static struct ast_state* get_ast_state(void) { @@ -23,10 +244,29 @@ get_ast_state(void) } return state; } +#else +static struct ast_state global_ast_state; + +static struct ast_state* +get_ast_state(void) +{ + struct ast_state *state = &global_ast_state; + if (!init_types(state)) { + return NULL; + } + return state; +} +#endif + void _PyAST_Fini(PyThreadState *tstate) { +#ifdef Py_BUILD_CORE struct ast_state* state = &tstate->interp->ast; +#else + struct ast_state *state = &global_ast_state; +#endif + Py_CLEAR(state->AST_type); Py_CLEAR(state->Add_singleton); Py_CLEAR(state->Add_type); From 5a4283df59758013958b0286a810ef5c693cb452 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 29 Oct 2020 13:48:13 +0100 Subject: [PATCH 3/7] Make asdl_c.py compatible with Python 3.8 and older --- Parser/asdl_c.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index a110a7dcd1d4c2..5d786e294ccb14 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1537,9 +1537,9 @@ def main(input_filename, c_filename, h_filename, internal_h_filename, dump_modul if not asdl.check(mod): sys.exit(1) - with (c_filename.open("w") as c_file, - h_filename.open("w") as h_file, - internal_h_filename.open("w") as internal_h_file): + with c_filename.open("w") as c_file, \ + h_filename.open("w") as h_file, \ + internal_h_filename.open("w") as internal_h_file: c_file.write(auto_gen_msg) h_file.write(auto_gen_msg) internal_h_file.write(auto_gen_msg) From 65c2ecb2dcf7d5680b0ae7da62a2ee657b309256 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 29 Oct 2020 21:30:58 +0100 Subject: [PATCH 4/7] Update PCbuild/regen.vcxproj --- PCbuild/regen.vcxproj | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/PCbuild/regen.vcxproj b/PCbuild/regen.vcxproj index 90d6dc68d5404f..4636eab276461b 100644 --- a/PCbuild/regen.vcxproj +++ b/PCbuild/regen.vcxproj @@ -137,6 +137,8 @@ + + @@ -161,15 +163,18 @@ - - + + - + + + + @@ -204,6 +209,7 @@ + From 48bce9767f905e11f25e1d91368959226870a23d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 29 Oct 2020 21:37:31 +0100 Subject: [PATCH 5/7] Fix the Yield macro on Windows --- Parser/asdl_c.py | 47 +++++++++++++++++++++++++++++---------------- Python/Python-ast.c | 6 ++++-- 2 files changed, 34 insertions(+), 19 deletions(-) diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 5d786e294ccb14..12d3a377b89be7 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1348,6 +1348,27 @@ def generate_ast_state(module_state, f): f.write(' PyObject *' + s + ';\n') f.write('};') + +def generate_ast_fini(module_state, f): + f.write(""" +void _PyAST_Fini(PyThreadState *tstate) +{ +#ifdef Py_BUILD_CORE + struct ast_state* state = &tstate->interp->ast; +#else + struct ast_state *state = &global_ast_state; +#endif + +""") + for s in module_state: + f.write(" Py_CLEAR(state->" + s + ');\n') + f.write(""" + state->initialized = 0; +} + +""") + + def generate_module_def(mod, f, internal_h): # Gather all the data needed for ModuleSpec visitor_list = set() @@ -1423,24 +1444,18 @@ def generate_module_def(mod, f, internal_h): return state; } #endif - - -void _PyAST_Fini(PyThreadState *tstate) -{ -#ifdef Py_BUILD_CORE - struct ast_state* state = &tstate->interp->ast; -#else - struct ast_state *state = &global_ast_state; -#endif - """) - for s in module_state: - f.write(" Py_CLEAR(state->" + s + ');\n') - f.write(""" - state->initialized = 0; -} + # f-string for {mod.name} + f.write(f""" +// Include {mod.name}-ast.h after pycore_interp.h to avoid conflicts +// with the Yield macro redefined by +#include "{mod.name}-ast.h" +#include "structmember.h" """) + + generate_ast_fini(module_state, f) + f.write('static int init_identifiers(struct ast_state *state)\n') f.write('{\n') for identifier in state_strings: @@ -1509,8 +1524,6 @@ def write_source(mod, f, internal_h_file): #include #include "Python.h" - #include "{mod.name}-ast.h" - #include "structmember.h" """), file=f) generate_module_def(mod, f, internal_h_file) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 277fdc51e243d3..60c251fa91237f 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -4,8 +4,6 @@ #include #include "Python.h" -#include "Python-ast.h" -#include "structmember.h" #ifdef Py_BUILD_CORE # include "pycore_ast.h" // struct ast_state @@ -258,6 +256,10 @@ get_ast_state(void) } #endif +// Include Python-ast.h after pycore_interp.h to avoid conflicts +// with the Yield macro redefined by +#include "Python-ast.h" +#include "structmember.h" void _PyAST_Fini(PyThreadState *tstate) { From b513e348c351b6d5b7f0fee64ca9d52626158794 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 29 Oct 2020 21:47:57 +0100 Subject: [PATCH 6/7] Add comments to #endif --- Parser/asdl_c.py | 4 ++-- Python/Python-ast.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 12d3a377b89be7..fc020cee941d27 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1413,7 +1413,7 @@ def generate_module_def(mod, f, internal_h): generate_ast_state(module_state, f) print(textwrap.dedent(f""" - #endif + #endif // Py_BUILD_CORE """).rstrip(), file=f) f.write(""" @@ -1443,7 +1443,7 @@ def generate_module_def(mod, f, internal_h): } return state; } -#endif +#endif // Py_BUILD_CORE """) # f-string for {mod.name} diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 60c251fa91237f..ece55e051be004 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -226,7 +226,7 @@ struct ast_state { PyObject *vararg; PyObject *withitem_type; }; -#endif +#endif // Py_BUILD_CORE // Forward declaration static int init_types(struct ast_state *state); @@ -254,7 +254,7 @@ get_ast_state(void) } return state; } -#endif +#endif // Py_BUILD_CORE // Include Python-ast.h after pycore_interp.h to avoid conflicts // with the Yield macro redefined by From ef38d8423a75d65215b1ea3c4e47b27492c07c4a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 29 Oct 2020 23:18:18 +0100 Subject: [PATCH 7/7] Address Batuhan's review --- PCbuild/regen.vcxproj | 2 +- Parser/asdl_c.py | 5 ++++- Python/Python-ast.c | 5 ++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/PCbuild/regen.vcxproj b/PCbuild/regen.vcxproj index 4636eab276461b..936f5fd24646a5 100644 --- a/PCbuild/regen.vcxproj +++ b/PCbuild/regen.vcxproj @@ -163,7 +163,7 @@ - + diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index fc020cee941d27..9a833e841de41e 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1095,6 +1095,9 @@ def visitModule(self, mod): self.emit("astmodule_exec(PyObject *m)", 0) self.emit("{", 0) self.emit('struct ast_state *state = get_ast_state();', 1) + self.emit('if (state == NULL) {', 1) + self.emit('return -1;', 2) + self.emit('}', 1) self.emit('if (PyModule_AddObject(m, "AST", state->AST_type) < 0) {', 1) self.emit('return -1;', 2) self.emit('}', 1) @@ -1354,7 +1357,7 @@ def generate_ast_fini(module_state, f): void _PyAST_Fini(PyThreadState *tstate) { #ifdef Py_BUILD_CORE - struct ast_state* state = &tstate->interp->ast; + struct ast_state *state = &tstate->interp->ast; #else struct ast_state *state = &global_ast_state; #endif diff --git a/Python/Python-ast.c b/Python/Python-ast.c index ece55e051be004..f04addbe2011dd 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -264,7 +264,7 @@ get_ast_state(void) void _PyAST_Fini(PyThreadState *tstate) { #ifdef Py_BUILD_CORE - struct ast_state* state = &tstate->interp->ast; + struct ast_state *state = &tstate->interp->ast; #else struct ast_state *state = &global_ast_state; #endif @@ -9696,6 +9696,9 @@ static int astmodule_exec(PyObject *m) { struct ast_state *state = get_ast_state(); + if (state == NULL) { + return -1; + } if (PyModule_AddObject(m, "AST", state->AST_type) < 0) { return -1; }