From 2ff5d7ca7e6d9ee13cda08467e29a00c5812ae80 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 6 Oct 2016 23:27:39 -0600 Subject: [PATCH 01/18] Add _PyImport_GetModuleDict(). --- Include/import.h | 3 +++ Python/import.c | 22 ++++++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/Include/import.h b/Include/import.h index 7e83985b3a43c6..00bf43454cc292 100644 --- a/Include/import.h +++ b/Include/import.h @@ -38,6 +38,9 @@ PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleObject( ); #endif PyAPI_FUNC(PyObject *) PyImport_GetModuleDict(void); +#ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject *) _PyImport_GetModuleDict(PyThreadState *tstate); +#endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 PyAPI_FUNC(PyObject *) PyImport_AddModuleObject( PyObject *name diff --git a/Python/import.c b/Python/import.c index 2aacf659351bb6..242e2547a15555 100644 --- a/Python/import.c +++ b/Python/import.c @@ -288,12 +288,30 @@ _PyImport_Fini(void) /* Helper for sys */ PyObject * -PyImport_GetModuleDict(void) +_PyImport_GetModuleDict(PyThreadState *tstate) { - PyInterpreterState *interp = PyThreadState_GET()->interp; + PyInterpreterState *interp = tstate->interp; if (interp->modules == NULL) Py_FatalError("PyImport_GetModuleDict: no module dictionary!"); return interp->modules; + + /* We aren't ready to do this yet. + if (tstate->interp->sysdict == NULL) + Py_FatalError("_PyImport_GetModuleDict: no sys module!"); + + _Py_IDENTIFIER(modules); + PyObject *modules = _PyDict_GetItemId(tstate->interp->sysdict, + &PyId_modules); + if (modules == NULL) + Py_FatalError("lost sys.modules"); + return modules; + */ +} + +PyObject * +PyImport_GetModuleDict(void) +{ + return _PyImport_GetModuleDict(PyThreadState_GET()); } From 79a84612247ddbf11b0c91fc03cd2c29a704e657 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 10 Oct 2016 12:11:52 -0600 Subject: [PATCH 02/18] Import the sys module before any other imports. --- Python/pylifecycle.c | 51 ++++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 0700569ac5affa..152b39d2321d4c 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -675,9 +675,20 @@ void _Py_InitializeCore(const _PyCoreConfig *config) if (!_PyFloat_Init()) Py_FatalError("Py_InitializeCore: can't init float"); - interp->modules = PyDict_New(); - if (interp->modules == NULL) + PyObject *modules = PyDict_New(); + if (modules == NULL) Py_FatalError("Py_InitializeCore: can't make modules dictionary"); + interp->modules = modules; + + sysmod = _PySys_BeginInit(); + if (sysmod == NULL) + Py_FatalError("Py_InitializeCore: can't initialize sys"); + interp->sysdict = PyModule_GetDict(sysmod); + if (interp->sysdict == NULL) + Py_FatalError("Py_InitializeCore: can't initialize sys dict"); + Py_INCREF(interp->sysdict); + PyDict_SetItemString(interp->sysdict, "modules", modules); + _PyImport_FixupBuiltin(sysmod, "sys"); /* Init Unicode implementation; relies on the codec registry */ if (_PyUnicode_Init() < 0) @@ -698,17 +709,6 @@ void _Py_InitializeCore(const _PyCoreConfig *config) /* initialize builtin exceptions */ _PyExc_Init(bimod); - sysmod = _PySys_BeginInit(); - if (sysmod == NULL) - Py_FatalError("Py_InitializeCore: can't initialize sys"); - interp->sysdict = PyModule_GetDict(sysmod); - if (interp->sysdict == NULL) - Py_FatalError("Py_InitializeCore: can't initialize sys dict"); - Py_INCREF(interp->sysdict); - _PyImport_FixupBuiltin(sysmod, "sys"); - PyDict_SetItemString(interp->sysdict, "modules", - interp->modules); - /* Set up a preliminary stderr printer until we have enough infrastructure for the io module in place. */ pstderr = PyFile_NewStdPrinter(fileno(stderr)); @@ -1203,7 +1203,21 @@ Py_NewInterpreter(void) /* XXX The following is lax in error checking */ - interp->modules = PyDict_New(); + PyObject *modules = PyDict_New(); + if (modules == NULL) + Py_FatalError("Py_NewInterpreter: can't make modules dictionary"); + interp->modules = modules; + + sysmod = _PyImport_FindBuiltin("sys"); + if (sysmod != NULL) { + interp->sysdict = PyModule_GetDict(sysmod); + if (interp->sysdict == NULL) + goto handle_error; + Py_INCREF(interp->sysdict); + PyDict_SetItemString(interp->sysdict, "modules", modules); + PySys_SetPath(Py_GetPath()); + _PySys_EndInit(interp->sysdict); + } bimod = _PyImport_FindBuiltin("builtins"); if (bimod != NULL) { @@ -1216,18 +1230,9 @@ Py_NewInterpreter(void) /* initialize builtin exceptions */ _PyExc_Init(bimod); - sysmod = _PyImport_FindBuiltin("sys"); if (bimod != NULL && sysmod != NULL) { PyObject *pstderr; - interp->sysdict = PyModule_GetDict(sysmod); - if (interp->sysdict == NULL) - goto handle_error; - Py_INCREF(interp->sysdict); - _PySys_EndInit(interp->sysdict); - PySys_SetPath(Py_GetPath()); - PyDict_SetItemString(interp->sysdict, "modules", - interp->modules); /* Set up a preliminary stderr printer until we have enough infrastructure for the io module in place. */ pstderr = PyFile_NewStdPrinter(fileno(stderr)); From 6f96d3a673f5a4d291c5d70c1654506a16a20e5d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 10 Oct 2016 13:01:59 -0600 Subject: [PATCH 03/18] Pass the modules dict to _PyImport_FindBuiltin(). --- Include/import.h | 8 +++++++- Python/import.c | 24 +++++++++++++++++++----- Python/pylifecycle.c | 4 ++-- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/Include/import.h b/Include/import.h index 00bf43454cc292..2501ec96453e0b 100644 --- a/Include/import.h +++ b/Include/import.h @@ -46,6 +46,9 @@ PyAPI_FUNC(PyObject *) PyImport_AddModuleObject( PyObject *name ); #endif +#ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject *) _PyImport_AddModuleObject(PyObject *, PyObject *); +#endif PyAPI_FUNC(PyObject *) PyImport_AddModule( const char *name /* UTF-8 encoded string */ ); @@ -95,9 +98,12 @@ PyAPI_FUNC(int) _PyImport_ReleaseLock(void); PyAPI_FUNC(void) _PyImport_ReInitLock(void); PyAPI_FUNC(PyObject *) _PyImport_FindBuiltin( - const char *name /* UTF-8 encoded string */ + const char *name, /* UTF-8 encoded string */ + PyObject *modules ); PyAPI_FUNC(PyObject *) _PyImport_FindExtensionObject(PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyImport_FindExtensionObjectEx(PyObject *, PyObject *, + PyObject *); PyAPI_FUNC(int) _PyImport_FixupBuiltin( PyObject *mod, const char *name /* UTF-8 encoded string */ diff --git a/Python/import.c b/Python/import.c index 242e2547a15555..994690ac2eca67 100644 --- a/Python/import.c +++ b/Python/import.c @@ -594,6 +594,14 @@ _PyImport_FixupBuiltin(PyObject *mod, const char *name) PyObject * _PyImport_FindExtensionObject(PyObject *name, PyObject *filename) +{ + PyObject *modules = PyImport_GetModuleDict(); + return _PyImport_FindExtensionObjectEx(name, filename, modules); +} + +PyObject * +_PyImport_FindExtensionObjectEx(PyObject *name, PyObject *filename, + PyObject *modules) { PyObject *mod, *mdict, *key; PyModuleDef* def; @@ -610,7 +618,7 @@ _PyImport_FindExtensionObject(PyObject *name, PyObject *filename) /* Module does not support repeated initialization */ if (def->m_base.m_copy == NULL) return NULL; - mod = PyImport_AddModuleObject(name); + mod = _PyImport_AddModuleObject(name, modules); if (mod == NULL) return NULL; mdict = PyModule_GetDict(mod); @@ -625,14 +633,14 @@ _PyImport_FindExtensionObject(PyObject *name, PyObject *filename) mod = def->m_base.m_init(); if (mod == NULL) return NULL; - if (PyDict_SetItem(PyImport_GetModuleDict(), name, mod) == -1) { + if (PyDict_SetItem(modules, name, mod) == -1) { Py_DECREF(mod); return NULL; } Py_DECREF(mod); } if (_PyState_AddModule(mod, def) < 0) { - PyDict_DelItem(PyImport_GetModuleDict(), name); + PyDict_DelItem(modules, name); Py_DECREF(mod); return NULL; } @@ -644,13 +652,13 @@ _PyImport_FindExtensionObject(PyObject *name, PyObject *filename) } PyObject * -_PyImport_FindBuiltin(const char *name) +_PyImport_FindBuiltin(const char *name, PyObject *modules) { PyObject *res, *nameobj; nameobj = PyUnicode_InternFromString(name); if (nameobj == NULL) return NULL; - res = _PyImport_FindExtensionObject(nameobj, nameobj); + res = _PyImport_FindExtensionObjectEx(nameobj, nameobj, modules); Py_DECREF(nameobj); return res; } @@ -665,6 +673,12 @@ PyObject * PyImport_AddModuleObject(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); + return _PyImport_AddModuleObject(name, modules); +} + +PyObject * +_PyImport_AddModuleObject(PyObject *name, PyObject *modules) +{ PyObject *m; if ((m = PyDict_GetItemWithError(modules, name)) != NULL && diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 152b39d2321d4c..bad6c269c69145 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1208,7 +1208,7 @@ Py_NewInterpreter(void) Py_FatalError("Py_NewInterpreter: can't make modules dictionary"); interp->modules = modules; - sysmod = _PyImport_FindBuiltin("sys"); + sysmod = _PyImport_FindBuiltin("sys", modules); if (sysmod != NULL) { interp->sysdict = PyModule_GetDict(sysmod); if (interp->sysdict == NULL) @@ -1219,7 +1219,7 @@ Py_NewInterpreter(void) _PySys_EndInit(interp->sysdict); } - bimod = _PyImport_FindBuiltin("builtins"); + bimod = _PyImport_FindBuiltin("builtins", modules); if (bimod != NULL) { interp->builtins = PyModule_GetDict(bimod); if (interp->builtins == NULL) From ad8c082fa07bca377a5ea646bdaab17837bd5bb4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 7 Oct 2016 00:02:53 -0600 Subject: [PATCH 04/18] Replace usage of interp->modules with PyImport_GetModuleDict(). --- Objects/typeobject.c | 4 ++-- Python/import.c | 8 +++++--- Python/pylifecycle.c | 5 ++--- Python/sysmodule.c | 5 +++-- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index dc4d2edc5cf622..9ebbb21ef8d37d 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3902,7 +3902,6 @@ import_copyreg(void) { PyObject *copyreg_str; PyObject *copyreg_module; - PyInterpreterState *interp = PyThreadState_GET()->interp; _Py_IDENTIFIER(copyreg); copyreg_str = _PyUnicode_FromId(&PyId_copyreg); @@ -3914,7 +3913,8 @@ import_copyreg(void) by storing a reference to the cached module in a static variable, but this broke when multiple embedded interpreters were in use (see issue #17408 and #19088). */ - copyreg_module = PyDict_GetItemWithError(interp->modules, copyreg_str); + PyObject *modules = PyImport_GetModuleDict(); + copyreg_module = PyDict_GetItemWithError(modules, copyreg_str); if (copyreg_module != NULL) { Py_INCREF(copyreg_module); return copyreg_module; diff --git a/Python/import.c b/Python/import.c index 994690ac2eca67..4f4f76df733338 100644 --- a/Python/import.c +++ b/Python/import.c @@ -341,7 +341,7 @@ PyImport_Cleanup(void) Py_ssize_t pos; PyObject *key, *value, *dict; PyInterpreterState *interp = PyThreadState_GET()->interp; - PyObject *modules = interp->modules; + PyObject *modules = PyImport_GetModuleDict(); PyObject *weaklist = NULL; const char * const *p; @@ -1543,7 +1543,8 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, Py_INCREF(abs_name); } - mod = PyDict_GetItem(interp->modules, abs_name); + PyObject *modules = PyImport_GetModuleDict(); + mod = PyDict_GetItem(modules, abs_name); if (mod != NULL && mod != Py_None) { _Py_IDENTIFIER(__spec__); _Py_IDENTIFIER(_initializing); @@ -1630,7 +1631,8 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, goto error; } - final_mod = PyDict_GetItem(interp->modules, to_return); + PyObject *modules = PyImport_GetModuleDict(); + final_mod = PyDict_GetItem(modules, to_return); Py_DECREF(to_return); if (final_mod == NULL) { PyErr_Format(PyExc_KeyError, diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index bad6c269c69145..789d82f5eb747e 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1908,9 +1908,8 @@ wait_for_thread_shutdown(void) { _Py_IDENTIFIER(_shutdown); PyObject *result; - PyThreadState *tstate = PyThreadState_GET(); - PyObject *threading = PyMapping_GetItemString(tstate->interp->modules, - "threading"); + PyObject *modules = PyImport_GetModuleDict(); + PyObject *threading = PyMapping_GetItemString(modules, "threading"); if (threading == NULL) { /* threading not imported */ PyErr_Clear(); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index dd127a1f7ce003..e0bed71bc35036 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -160,8 +160,9 @@ static PyObject * sys_displayhook(PyObject *self, PyObject *o) { PyObject *outf; - PyInterpreterState *interp = PyThreadState_GET()->interp; - PyObject *modules = interp->modules; + PyObject *modules = PyImport_GetModuleDict(); + if (modules == NULL) + return NULL; PyObject *builtins; static PyObject *newline = NULL; int err; From 12f55d30be0a66760cb0806e06093e08f43c60da Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 10 Oct 2016 11:23:44 -0600 Subject: [PATCH 05/18] Add _PyImport_EnsureInitialized(). --- Include/import.h | 1 + Python/import.c | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/Include/import.h b/Include/import.h index 2501ec96453e0b..e71888ed91a19c 100644 --- a/Include/import.h +++ b/Include/import.h @@ -40,6 +40,7 @@ PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleObject( PyAPI_FUNC(PyObject *) PyImport_GetModuleDict(void); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _PyImport_GetModuleDict(PyThreadState *tstate); +PyAPI_FUNC(void) _PyImport_EnsureInitialized(PyInterpreterState *interp); #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 PyAPI_FUNC(PyObject *) PyImport_AddModuleObject( diff --git a/Python/import.c b/Python/import.c index 4f4f76df733338..8e29085ab1f585 100644 --- a/Python/import.c +++ b/Python/import.c @@ -314,6 +314,21 @@ PyImport_GetModuleDict(void) return _PyImport_GetModuleDict(PyThreadState_GET()); } +/* In some corner cases it is important to be sure that the import + machinery has been initialized (or not cleaned up yet). For + example, see issue #4236 and PyModule_Create2(). */ + +void +_PyImport_EnsureInitialized(PyInterpreterState *interp) +{ + if (interp->modules == NULL) + goto notinitialized; + return; + +notinitialized: + Py_FatalError("Python import machinery not initialized"); +} + /* List of names to clear in sys */ static const char * const sys_deletes[] = { From eb2d6f980179a61b7b667b3dafc13a591e3efef1 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 10 Oct 2016 11:30:02 -0600 Subject: [PATCH 06/18] Add _PyModule_Create2(). --- Include/modsupport.h | 4 ++++ Objects/moduleobject.c | 11 ++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/Include/modsupport.h b/Include/modsupport.h index 8c7cf39d9a3ea9..3ba15b7853d248 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -191,6 +191,10 @@ PyAPI_FUNC(int) PyModule_ExecDef(PyObject *module, PyModuleDef *def); PyAPI_FUNC(PyObject *) PyModule_Create2(struct PyModuleDef*, int apiver); +#ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject *) _PyModule_Create2(struct PyModuleDef*, + int apiver); +#endif #ifdef Py_LIMITED_API #define PyModule_Create(module) \ diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 5fab06d31df036..17ee15782b23fa 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -161,12 +161,17 @@ _add_methods_to_object(PyObject *module, PyObject *name, PyMethodDef *functions) PyObject * PyModule_Create2(struct PyModuleDef* module, int module_api_version) +{ + _PyImport_EnsureInitialized(PyThreadState_GET()->interp); + return _PyModule_Create2(module, module_api_version); +} + +PyObject * +_PyModule_Create2(struct PyModuleDef* module, int module_api_version) { const char* name; PyModuleObject *m; - PyInterpreterState *interp = PyThreadState_Get()->interp; - if (interp->modules == NULL) - Py_FatalError("Python import machinery not initialized"); + if (!PyModuleDef_Init(module)) return NULL; name = module->m_name; From 6a04c1c2f68fe17b6c33a9274c26148d14106dab Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 10 Oct 2016 11:31:51 -0600 Subject: [PATCH 07/18] Use _PyModule_Create2() for key builtin modules. --- Python/bltinmodule.c | 2 +- Python/sysmodule.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 5e1f1d3854fc67..df84cf1c0d3c16 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2685,7 +2685,7 @@ _PyBuiltin_Init(void) PyType_Ready(&PyZip_Type) < 0) return NULL; - mod = PyModule_Create(&builtinsmodule); + mod = _PyModule_Create2(&builtinsmodule, PYTHON_API_VERSION); if (mod == NULL) return NULL; dict = PyModule_GetDict(mod); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index e0bed71bc35036..4827cf2413945d 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1940,7 +1940,7 @@ _PySys_BeginInit(void) PyObject *m, *sysdict, *version_info; int res; - m = PyModule_Create(&sysmodule); + m = _PyModule_Create2(&sysmodule, PYTHON_API_VERSION); if (m == NULL) return NULL; sysdict = PyModule_GetDict(m); From 26bdb7e65ca9eb481af4801c4ca7880360495e25 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 10 Oct 2016 11:23:44 -0600 Subject: [PATCH 08/18] Check sys.modules in _PyImport_EnsureInitialized(). --- Python/import.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Python/import.c b/Python/import.c index 8e29085ab1f585..5cff558e0384e5 100644 --- a/Python/import.c +++ b/Python/import.c @@ -323,6 +323,14 @@ _PyImport_EnsureInitialized(PyInterpreterState *interp) { if (interp->modules == NULL) goto notinitialized; + /* We aren't ready to do this yet. + if (interp->sysdict == NULL) + goto notinitialized; + _Py_IDENTIFIER(modules); + PyObject *modules = _PyDict_GetItemId(interp->sysdict, &PyId_modules); + if (modules == NULL) + goto notinitialized; + */ return; notinitialized: From eac74033ee09643f6f1369fffe2eb30d148d0892 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 10 Oct 2016 13:55:55 -0600 Subject: [PATCH 09/18] Pass the modules dict to _PyImport_FixupBuiltin(). --- Include/import.h | 5 ++++- Python/import.c | 15 +++++++++++---- Python/pylifecycle.c | 4 ++-- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/Include/import.h b/Include/import.h index e71888ed91a19c..a50b7fdd87abac 100644 --- a/Include/import.h +++ b/Include/import.h @@ -107,9 +107,12 @@ PyAPI_FUNC(PyObject *) _PyImport_FindExtensionObjectEx(PyObject *, PyObject *, PyObject *); PyAPI_FUNC(int) _PyImport_FixupBuiltin( PyObject *mod, - const char *name /* UTF-8 encoded string */ + const char *name, /* UTF-8 encoded string */ + PyObject *modules ); PyAPI_FUNC(int) _PyImport_FixupExtensionObject(PyObject*, PyObject *, PyObject *); +PyAPI_FUNC(int) _PyImport_FixupExtensionObjectEx(PyObject*, PyObject *, + PyObject *, PyObject *); struct _inittab { const char *name; /* ASCII encoded string */ diff --git a/Python/import.c b/Python/import.c index 5cff558e0384e5..69aac3b6084aea 100644 --- a/Python/import.c +++ b/Python/import.c @@ -554,7 +554,15 @@ int _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, PyObject *filename) { - PyObject *modules, *dict, *key; + PyObject *modules = PyImport_GetModuleDict(); + return _PyImport_FixupExtensionObjectEx(mod, name, filename, modules); +} + +int +_PyImport_FixupExtensionObjectEx(PyObject *mod, PyObject *name, + PyObject *filename, PyObject *modules) +{ + PyObject *dict, *key; struct PyModuleDef *def; int res; if (extensions == NULL) { @@ -571,7 +579,6 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, PyErr_BadInternalCall(); return -1; } - modules = PyImport_GetModuleDict(); if (PyDict_SetItem(modules, name, mod) < 0) return -1; if (_PyState_AddModule(mod, def) < 0) { @@ -603,14 +610,14 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, } int -_PyImport_FixupBuiltin(PyObject *mod, const char *name) +_PyImport_FixupBuiltin(PyObject *mod, const char *name, PyObject *modules) { int res; PyObject *nameobj; nameobj = PyUnicode_InternFromString(name); if (nameobj == NULL) return -1; - res = _PyImport_FixupExtensionObject(mod, nameobj, nameobj); + res = _PyImport_FixupExtensionObjectEx(mod, nameobj, nameobj, modules); Py_DECREF(nameobj); return res; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 789d82f5eb747e..330fb634af6429 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -688,7 +688,7 @@ void _Py_InitializeCore(const _PyCoreConfig *config) Py_FatalError("Py_InitializeCore: can't initialize sys dict"); Py_INCREF(interp->sysdict); PyDict_SetItemString(interp->sysdict, "modules", modules); - _PyImport_FixupBuiltin(sysmod, "sys"); + _PyImport_FixupBuiltin(sysmod, "sys", modules); /* Init Unicode implementation; relies on the codec registry */ if (_PyUnicode_Init() < 0) @@ -700,7 +700,7 @@ void _Py_InitializeCore(const _PyCoreConfig *config) bimod = _PyBuiltin_Init(); if (bimod == NULL) Py_FatalError("Py_InitializeCore: can't initialize builtins modules"); - _PyImport_FixupBuiltin(bimod, "builtins"); + _PyImport_FixupBuiltin(bimod, "builtins", modules); interp->builtins = PyModule_GetDict(bimod); if (interp->builtins == NULL) Py_FatalError("Py_InitializeCore: can't initialize builtins dict"); From 1166c57254c0fdda066e6b5ddc6a88e9696d707f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 11 Oct 2016 17:00:32 -0600 Subject: [PATCH 10/18] Pass PyInterpreterState to _PyImport_GetModuleDict(). --- Include/import.h | 4 ++-- Python/import.c | 12 +++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/Include/import.h b/Include/import.h index a50b7fdd87abac..a8d74f2b913716 100644 --- a/Include/import.h +++ b/Include/import.h @@ -39,8 +39,8 @@ PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleObject( #endif PyAPI_FUNC(PyObject *) PyImport_GetModuleDict(void); #ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyImport_GetModuleDict(PyThreadState *tstate); -PyAPI_FUNC(void) _PyImport_EnsureInitialized(PyInterpreterState *interp); +PyAPI_FUNC(PyObject *) _PyImport_GetModuleDict(PyInterpreterState *); +PyAPI_FUNC(void) _PyImport_EnsureInitialized(PyInterpreterState *); #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 PyAPI_FUNC(PyObject *) PyImport_AddModuleObject( diff --git a/Python/import.c b/Python/import.c index 69aac3b6084aea..2016ce721fac0f 100644 --- a/Python/import.c +++ b/Python/import.c @@ -288,20 +288,18 @@ _PyImport_Fini(void) /* Helper for sys */ PyObject * -_PyImport_GetModuleDict(PyThreadState *tstate) +_PyImport_GetModuleDict(PyInterpreterState *interp) { - PyInterpreterState *interp = tstate->interp; if (interp->modules == NULL) Py_FatalError("PyImport_GetModuleDict: no module dictionary!"); return interp->modules; /* We aren't ready to do this yet. - if (tstate->interp->sysdict == NULL) - Py_FatalError("_PyImport_GetModuleDict: no sys module!"); + if (interp->sysdict == NULL) + Py_FatalError("PyImport_GetModuleDict: no sys module!"); _Py_IDENTIFIER(modules); - PyObject *modules = _PyDict_GetItemId(tstate->interp->sysdict, - &PyId_modules); + PyObject *modules = _PyDict_GetItemId(interp->sysdict, &PyId_modules); if (modules == NULL) Py_FatalError("lost sys.modules"); return modules; @@ -311,7 +309,7 @@ _PyImport_GetModuleDict(PyThreadState *tstate) PyObject * PyImport_GetModuleDict(void) { - return _PyImport_GetModuleDict(PyThreadState_GET()); + return _PyImport_GetModuleDict(PyThreadState_GET()->interp); } /* In some corner cases it is important to be sure that the import From da3ce95e65e39be547753f39ba262b043159c929 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 11 Oct 2016 17:04:30 -0600 Subject: [PATCH 11/18] Drop _PyImport_GetModuleDict(). --- Include/import.h | 1 - Python/import.c | 14 +++++--------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/Include/import.h b/Include/import.h index a8d74f2b913716..233697b89ebcb1 100644 --- a/Include/import.h +++ b/Include/import.h @@ -39,7 +39,6 @@ PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleObject( #endif PyAPI_FUNC(PyObject *) PyImport_GetModuleDict(void); #ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyImport_GetModuleDict(PyInterpreterState *); PyAPI_FUNC(void) _PyImport_EnsureInitialized(PyInterpreterState *); #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 diff --git a/Python/import.c b/Python/import.c index 2016ce721fac0f..6db277e46a28e8 100644 --- a/Python/import.c +++ b/Python/import.c @@ -288,30 +288,26 @@ _PyImport_Fini(void) /* Helper for sys */ PyObject * -_PyImport_GetModuleDict(PyInterpreterState *interp) +PyImport_GetModuleDict(void) { + PyInterpreterState *interp = PyThreadState_GET()->interp; if (interp->modules == NULL) Py_FatalError("PyImport_GetModuleDict: no module dictionary!"); return interp->modules; /* We aren't ready to do this yet. - if (interp->sysdict == NULL) + PyObject *sysdict = PyThreadState_GET()->interp->sysdict; + if (sysdict == NULL) Py_FatalError("PyImport_GetModuleDict: no sys module!"); _Py_IDENTIFIER(modules); - PyObject *modules = _PyDict_GetItemId(interp->sysdict, &PyId_modules); + PyObject *modules = _PyDict_GetItemId(sysdict, &PyId_modules); if (modules == NULL) Py_FatalError("lost sys.modules"); return modules; */ } -PyObject * -PyImport_GetModuleDict(void) -{ - return _PyImport_GetModuleDict(PyThreadState_GET()->interp); -} - /* In some corner cases it is important to be sure that the import machinery has been initialized (or not cleaned up yet). For example, see issue #4236 and PyModule_Create2(). */ From ef17b812f0a0d06f84bb7edef148c52f769a110b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 17 May 2017 16:55:14 -0700 Subject: [PATCH 12/18] _PyModule_Create2 -> _PyModule_CreateInitialized --- Include/modsupport.h | 4 ++-- Objects/moduleobject.c | 4 ++-- Python/bltinmodule.c | 2 +- Python/sysmodule.c | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Include/modsupport.h b/Include/modsupport.h index 3ba15b7853d248..73d86a94b957f6 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -192,8 +192,8 @@ PyAPI_FUNC(int) PyModule_ExecDef(PyObject *module, PyModuleDef *def); PyAPI_FUNC(PyObject *) PyModule_Create2(struct PyModuleDef*, int apiver); #ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyModule_Create2(struct PyModuleDef*, - int apiver); +PyAPI_FUNC(PyObject *) _PyModule_CreateInitialized(struct PyModuleDef*, + int apiver); #endif #ifdef Py_LIMITED_API diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 17ee15782b23fa..35916f32639ea8 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -163,11 +163,11 @@ PyObject * PyModule_Create2(struct PyModuleDef* module, int module_api_version) { _PyImport_EnsureInitialized(PyThreadState_GET()->interp); - return _PyModule_Create2(module, module_api_version); + return _PyModule_CreateInitialized(module, module_api_version); } PyObject * -_PyModule_Create2(struct PyModuleDef* module, int module_api_version) +_PyModule_CreateInitialized(struct PyModuleDef* module, int module_api_version) { const char* name; PyModuleObject *m; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index df84cf1c0d3c16..c363cfe8cea26b 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2685,7 +2685,7 @@ _PyBuiltin_Init(void) PyType_Ready(&PyZip_Type) < 0) return NULL; - mod = _PyModule_Create2(&builtinsmodule, PYTHON_API_VERSION); + mod = _PyModule_CreateInitialized(&builtinsmodule, PYTHON_API_VERSION); if (mod == NULL) return NULL; dict = PyModule_GetDict(mod); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 4827cf2413945d..ff2f9aef92dc58 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1940,7 +1940,7 @@ _PySys_BeginInit(void) PyObject *m, *sysdict, *version_info; int res; - m = _PyModule_Create2(&sysmodule, PYTHON_API_VERSION); + m = _PyModule_CreateInitialized(&sysmodule, PYTHON_API_VERSION); if (m == NULL) return NULL; sysdict = PyModule_GetDict(m); From a17d5ff9366aac6b5f2991e6b45968f6432595ac Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 17 May 2017 22:41:44 -0700 Subject: [PATCH 13/18] _PyImport_EnsureInitialized() -> _PyImport_IsInitialized() --- Include/import.h | 2 +- Objects/moduleobject.c | 3 ++- Python/import.c | 15 ++++++--------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/Include/import.h b/Include/import.h index 233697b89ebcb1..ec20794c115afb 100644 --- a/Include/import.h +++ b/Include/import.h @@ -39,7 +39,7 @@ PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleObject( #endif PyAPI_FUNC(PyObject *) PyImport_GetModuleDict(void); #ifndef Py_LIMITED_API -PyAPI_FUNC(void) _PyImport_EnsureInitialized(PyInterpreterState *); +PyAPI_FUNC(int) _PyImport_IsInitialized(PyInterpreterState *); #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 PyAPI_FUNC(PyObject *) PyImport_AddModuleObject( diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 35916f32639ea8..2be49fbda38908 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -162,7 +162,8 @@ _add_methods_to_object(PyObject *module, PyObject *name, PyMethodDef *functions) PyObject * PyModule_Create2(struct PyModuleDef* module, int module_api_version) { - _PyImport_EnsureInitialized(PyThreadState_GET()->interp); + if (!_PyImport_IsInitialized(PyThreadState_GET()->interp)) + Py_FatalError("Python import machinery not initialized"); return _PyModule_CreateInitialized(module, module_api_version); } diff --git a/Python/import.c b/Python/import.c index 6db277e46a28e8..79e54db6802f6f 100644 --- a/Python/import.c +++ b/Python/import.c @@ -312,23 +312,20 @@ PyImport_GetModuleDict(void) machinery has been initialized (or not cleaned up yet). For example, see issue #4236 and PyModule_Create2(). */ -void -_PyImport_EnsureInitialized(PyInterpreterState *interp) +int +_PyImport_IsInitialized(PyInterpreterState *interp) { if (interp->modules == NULL) - goto notinitialized; + return 0; /* We aren't ready to do this yet. if (interp->sysdict == NULL) - goto notinitialized; + return 0; _Py_IDENTIFIER(modules); PyObject *modules = _PyDict_GetItemId(interp->sysdict, &PyId_modules); if (modules == NULL) - goto notinitialized; + return 0; */ - return; - -notinitialized: - Py_FatalError("Python import machinery not initialized"); + return 1; } From acc270b2c2b126d8e55f9029a308f6753a33c4b9 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Sep 2017 10:46:53 -0700 Subject: [PATCH 14/18] Add a Misc/NEWS entry. --- .../Core and Builtins/2017-09-04-10-46-09.bpo-28411.IU9rQL.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-09-04-10-46-09.bpo-28411.IU9rQL.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-04-10-46-09.bpo-28411.IU9rQL.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-04-10-46-09.bpo-28411.IU9rQL.rst new file mode 100644 index 00000000000000..0e5e5c837fe8ea --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-09-04-10-46-09.bpo-28411.IU9rQL.rst @@ -0,0 +1,3 @@ +``PyInterpreterState`` has a "modules" field that is copied into +``sys.modules`` during interpreter startup. We restructure code +to better isolate sys.modules, to make it easier to sync them up. From bc6a0f14b725590fc54ac02bb03f2e9823298b3e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Sep 2017 11:39:41 -0700 Subject: [PATCH 15/18] Fix style (bracket placement). --- Python/import.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Python/import.c b/Python/import.c index 79e54db6802f6f..772603c3cb0feb 100644 --- a/Python/import.c +++ b/Python/import.c @@ -297,13 +297,15 @@ PyImport_GetModuleDict(void) /* We aren't ready to do this yet. PyObject *sysdict = PyThreadState_GET()->interp->sysdict; - if (sysdict == NULL) + if (sysdict == NULL) { Py_FatalError("PyImport_GetModuleDict: no sys module!"); + } _Py_IDENTIFIER(modules); PyObject *modules = _PyDict_GetItemId(sysdict, &PyId_modules); - if (modules == NULL) + if (modules == NULL) { Py_FatalError("lost sys.modules"); + } return modules; */ } @@ -328,7 +330,6 @@ _PyImport_IsInitialized(PyInterpreterState *interp) return 1; } - /* List of names to clear in sys */ static const char * const sys_deletes[] = { "path", "argv", "ps1", "ps2", From c7a25242925afc5e6d9808463766cd8dfda22601 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Sep 2017 13:19:52 -0700 Subject: [PATCH 16/18] Drop _PyImport_FixupExtensionObjectEx(). --- Include/import.h | 5 ++--- Python/import.c | 17 +++++++---------- Python/importdl.c | 3 ++- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/Include/import.h b/Include/import.h index ec20794c115afb..95c52b0bfba129 100644 --- a/Include/import.h +++ b/Include/import.h @@ -109,9 +109,8 @@ PyAPI_FUNC(int) _PyImport_FixupBuiltin( const char *name, /* UTF-8 encoded string */ PyObject *modules ); -PyAPI_FUNC(int) _PyImport_FixupExtensionObject(PyObject*, PyObject *, PyObject *); -PyAPI_FUNC(int) _PyImport_FixupExtensionObjectEx(PyObject*, PyObject *, - PyObject *, PyObject *); +PyAPI_FUNC(int) _PyImport_FixupExtensionObject(PyObject*, PyObject *, + PyObject *, PyObject *); struct _inittab { const char *name; /* ASCII encoded string */ diff --git a/Python/import.c b/Python/import.c index 772603c3cb0feb..084895cef66420 100644 --- a/Python/import.c +++ b/Python/import.c @@ -544,14 +544,6 @@ PyImport_GetMagicTag(void) int _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, - PyObject *filename) -{ - PyObject *modules = PyImport_GetModuleDict(); - return _PyImport_FixupExtensionObjectEx(mod, name, filename, modules); -} - -int -_PyImport_FixupExtensionObjectEx(PyObject *mod, PyObject *name, PyObject *filename, PyObject *modules) { PyObject *dict, *key; @@ -609,7 +601,7 @@ _PyImport_FixupBuiltin(PyObject *mod, const char *name, PyObject *modules) nameobj = PyUnicode_InternFromString(name); if (nameobj == NULL) return -1; - res = _PyImport_FixupExtensionObjectEx(mod, nameobj, nameobj, modules); + res = _PyImport_FixupExtensionObject(mod, nameobj, nameobj, modules); Py_DECREF(nameobj); return res; } @@ -1096,6 +1088,7 @@ _imp_create_builtin(PyObject *module, PyObject *spec) return NULL; } + PyObject *modules = NULL; for (p = PyImport_Inittab; p->name != NULL; p++) { PyModuleDef *def; if (_PyUnicode_EqualToASCIIString(name, p->name)) { @@ -1121,7 +1114,11 @@ _imp_create_builtin(PyObject *module, PyObject *spec) return NULL; } def->m_base.m_init = p->initfunc; - if (_PyImport_FixupExtensionObject(mod, name, name) < 0) { + if (modules == NULL) { + modules = PyImport_GetModuleDict(); + } + if (_PyImport_FixupExtensionObject(mod, name, name, + modules) < 0) { Py_DECREF(name); return NULL; } diff --git a/Python/importdl.c b/Python/importdl.c index d8656b943336ec..32fb7e1be212ca 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -215,7 +215,8 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) else Py_INCREF(path); - if (_PyImport_FixupExtensionObject(m, name_unicode, path) < 0) + PyObject *modules = PyImport_GetModuleDict(); + if (_PyImport_FixupExtensionObject(m, name_unicode, path, modules) < 0) goto error; Py_DECREF(name_unicode); From aceaacda94c6a40a3952088664c4cc381ad4c374 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 14 Sep 2017 15:24:38 +0000 Subject: [PATCH 17/18] Fix the NEWS entry. --- .../Core and Builtins/2017-09-04-10-46-09.bpo-28411.IU9rQL.rst | 3 --- .../Core and Builtins/2017-09-11-09-24-21.bpo-28411.12SpAm.rst | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-09-04-10-46-09.bpo-28411.IU9rQL.rst create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-09-11-09-24-21.bpo-28411.12SpAm.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-04-10-46-09.bpo-28411.IU9rQL.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-04-10-46-09.bpo-28411.IU9rQL.rst deleted file mode 100644 index 0e5e5c837fe8ea..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2017-09-04-10-46-09.bpo-28411.IU9rQL.rst +++ /dev/null @@ -1,3 +0,0 @@ -``PyInterpreterState`` has a "modules" field that is copied into -``sys.modules`` during interpreter startup. We restructure code -to better isolate sys.modules, to make it easier to sync them up. diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-11-09-24-21.bpo-28411.12SpAm.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-11-09-24-21.bpo-28411.12SpAm.rst new file mode 100644 index 00000000000000..47f9fa684e5a5a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-09-11-09-24-21.bpo-28411.12SpAm.rst @@ -0,0 +1,3 @@ +Change direct usage of PyInterpreterState.modules to PyImport_GetModuleDict(). +Also introduce more uniformity in other code that deals with sys.modules. +This helps reduce complications when working on sys.modules. From 5b87111ba5359c84e3d2a027e70212ac00ef073c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 14 Sep 2017 17:47:26 +0000 Subject: [PATCH 18/18] Drop commented-out code. --- Python/import.c | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/Python/import.c b/Python/import.c index 084895cef66420..7aa7a1bdf7988e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -294,20 +294,6 @@ PyImport_GetModuleDict(void) if (interp->modules == NULL) Py_FatalError("PyImport_GetModuleDict: no module dictionary!"); return interp->modules; - - /* We aren't ready to do this yet. - PyObject *sysdict = PyThreadState_GET()->interp->sysdict; - if (sysdict == NULL) { - Py_FatalError("PyImport_GetModuleDict: no sys module!"); - } - - _Py_IDENTIFIER(modules); - PyObject *modules = _PyDict_GetItemId(sysdict, &PyId_modules); - if (modules == NULL) { - Py_FatalError("lost sys.modules"); - } - return modules; - */ } /* In some corner cases it is important to be sure that the import @@ -319,14 +305,6 @@ _PyImport_IsInitialized(PyInterpreterState *interp) { if (interp->modules == NULL) return 0; - /* We aren't ready to do this yet. - if (interp->sysdict == NULL) - return 0; - _Py_IDENTIFIER(modules); - PyObject *modules = _PyDict_GetItemId(interp->sysdict, &PyId_modules); - if (modules == NULL) - return 0; - */ return 1; }