Pull in main

Author: erlend-aasland

Include/cpython/initconfig.h

@@ -248,6 +248,7 @@ typedef struct {
     int allow_exec;
     int allow_threads;
     int allow_daemon_threads;
+    int check_multi_interp_extensions;
 } _PyInterpreterConfig;
 
 #define _PyInterpreterConfig_INIT \
@@ -256,6 +257,7 @@ typedef struct {
         .allow_exec = 0, \
         .allow_threads = 1, \
         .allow_daemon_threads = 0, \
+        .check_multi_interp_extensions = 1, \
     }
 
 #define _PyInterpreterConfig_LEGACY_INIT \
@@ -264,6 +266,7 @@ typedef struct {
         .allow_exec = 1, \
         .allow_threads = 1, \
         .allow_daemon_threads = 1, \
+        .check_multi_interp_extensions = 0, \
     }
 
 /* --- Helper functions --------------------------------------- */

Include/cpython/pystate.h

@@ -11,6 +11,9 @@ is available in a given context.  For example, forking the process
 might not be allowed in the current interpreter (i.e. os.fork() would fail).
 */
 
+/* Set if import should check a module for subinterpreter support. */
+#define Py_RTFLAGS_MULTI_INTERP_EXTENSIONS (1UL << 8)
+
 /* Set if threads are allowed. */
 #define Py_RTFLAGS_THREADS (1UL << 10)
 

Include/internal/pycore_import.h

@@ -36,11 +36,113 @@ struct _import_runtime_state {
     const char * pkgcontext;
 };
 
+struct _import_state {
+    /* cached sys.modules dictionary */
+    PyObject *modules;
+    /* This is the list of module objects for all legacy (single-phase init)
+       extension modules ever loaded in this process (i.e. imported
+       in this interpreter or in any other).  Py_None stands in for
+       modules that haven't actually been imported in this interpreter.
+
+       A module's index (PyModuleDef.m_base.m_index) is used to look up
+       the corresponding module object for this interpreter, if any.
+       (See PyState_FindModule().)  When any extension module
+       is initialized during import, its moduledef gets initialized by
+       PyModuleDef_Init(), and the first time that happens for each
+       PyModuleDef, its index gets set to the current value of
+       a global counter (see _PyRuntimeState.imports.last_module_index).
+       The entry for that index in this interpreter remains unset until
+       the module is actually imported here.  (Py_None is used as
+       a placeholder.)  Note that multi-phase init modules always get
+       an index for which there will never be a module set.
+
+       This is initialized lazily in PyState_AddModule(), which is also
+       where modules get added. */
+    PyObject *modules_by_index;
+    /* importlib module._bootstrap */
+    PyObject *importlib;
+    /* override for config->use_frozen_modules (for tests)
+       (-1: "off", 1: "on", 0: no override) */
+    int override_frozen_modules;
+    int override_multi_interp_extensions_check;
+#ifdef HAVE_DLOPEN
+    int dlopenflags;
+#endif
+    PyObject *import_func;
+};
+
+#ifdef HAVE_DLOPEN
+#  include <dlfcn.h>
+#  if HAVE_DECL_RTLD_NOW
+#    define _Py_DLOPEN_FLAGS RTLD_NOW
+#  else
+#    define _Py_DLOPEN_FLAGS RTLD_LAZY
+#  endif
+#  define DLOPENFLAGS_INIT .dlopenflags = _Py_DLOPEN_FLAGS,
+#else
+#  define _Py_DLOPEN_FLAGS 0
+#  define DLOPENFLAGS_INIT
+#endif
+
+#define IMPORTS_INIT \
+    { \
+        .override_frozen_modules = 0, \
+        DLOPENFLAGS_INIT \
+    }
+
+extern void _PyImport_ClearCore(PyInterpreterState *interp);
+
+extern Py_ssize_t _PyImport_GetNextModuleIndex(void);
+extern const char * _PyImport_ResolveNameWithPackageContext(const char *name);
+extern const char * _PyImport_SwapPackageContext(const char *newcontext);
+
+extern int _PyImport_GetDLOpenFlags(PyInterpreterState *interp);
+extern void _PyImport_SetDLOpenFlags(PyInterpreterState *interp, int new_val);
+
+extern PyObject * _PyImport_InitModules(PyInterpreterState *interp);
+extern PyObject * _PyImport_GetModules(PyInterpreterState *interp);
+extern void _PyImport_ClearModules(PyInterpreterState *interp);
+
+extern void _PyImport_ClearModulesByIndex(PyInterpreterState *interp);
+
+extern int _PyImport_InitDefaultImportFunc(PyInterpreterState *interp);
+extern int _PyImport_IsDefaultImportFunc(
+        PyInterpreterState *interp,
+        PyObject *func);
+
+extern PyObject * _PyImport_GetImportlibLoader(
+        PyInterpreterState *interp,
+        const char *loader_name);
+extern PyObject * _PyImport_GetImportlibExternalLoader(
+        PyInterpreterState *interp,
+        const char *loader_name);
+extern PyObject * _PyImport_BlessMyLoader(
+        PyInterpreterState *interp,
+        PyObject *module_globals);
+extern PyObject * _PyImport_ImportlibModuleRepr(
+        PyInterpreterState *interp,
+        PyObject *module);
+
+
+extern PyStatus _PyImport_Init(void);
+extern void _PyImport_Fini(void);
+extern void _PyImport_Fini2(void);
+
+extern PyStatus _PyImport_InitCore(
+        PyThreadState *tstate,
+        PyObject *sysmod,
+        int importlib);
+extern PyStatus _PyImport_InitExternal(PyThreadState *tstate);
+extern void _PyImport_FiniCore(PyInterpreterState *interp);
+extern void _PyImport_FiniExternal(PyInterpreterState *interp);
+
 
 #ifdef HAVE_FORK
 extern PyStatus _PyImport_ReInitLock(void);
 #endif
-extern PyObject* _PyImport_BootstrapImp(PyThreadState *tstate);
+
+
+extern PyObject* _PyImport_GetBuiltinModuleNames(void);
 
 struct _module_alias {
     const char *name;                 /* ASCII encoded string */
@@ -52,6 +154,13 @@ PyAPI_DATA(const struct _frozen *) _PyImport_FrozenStdlib;
 PyAPI_DATA(const struct _frozen *) _PyImport_FrozenTest;
 extern const struct _module_alias * _PyImport_FrozenAliases;
 
+PyAPI_FUNC(int) _PyImport_CheckSubinterpIncompatibleExtensionAllowed(
+    const char *name);
+
+
+// for testing
+PyAPI_FUNC(int) _PyImport_ClearExtension(PyObject *name, PyObject *filename);
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_interp.h

@@ -21,6 +21,7 @@ extern "C" {
 #include "pycore_function.h"      // FUNC_MAX_WATCHERS
 #include "pycore_genobject.h"     // struct _Py_async_gen_state
 #include "pycore_gc.h"            // struct _gc_runtime_state
+#include "pycore_import.h"        // struct _import_state
 #include "pycore_list.h"          // struct _Py_list_state
 #include "pycore_global_objects.h"  // struct _Py_interp_static_objects
 #include "pycore_tuple.h"         // struct _Py_tuple_state
@@ -92,53 +93,24 @@ struct _is {
     struct _ceval_state ceval;
     struct _gc_runtime_state gc;
 
-    // sys.modules dictionary
-    PyObject *modules;
-    /* This is the list of module objects for all legacy (single-phase init)
-       extension modules ever loaded in this process (i.e. imported
-       in this interpreter or in any other).  Py_None stands in for
-       modules that haven't actually been imported in this interpreter.
-
-       A module's index (PyModuleDef.m_base.m_index) is used to look up
-       the corresponding module object for this interpreter, if any.
-       (See PyState_FindModule().)  When any extension module
-       is initialized during import, its moduledef gets initialized by
-       PyModuleDef_Init(), and the first time that happens for each
-       PyModuleDef, its index gets set to the current value of
-       a global counter (see _PyRuntimeState.imports.last_module_index).
-       The entry for that index in this interpreter remains unset until
-       the module is actually imported here.  (Py_None is used as
-       a placeholder.)  Note that multi-phase init modules always get
-       an index for which there will never be a module set.
-
-       This is initialized lazily in _PyState_AddModule(), which is also
-       where modules get added. */
-    PyObject *modules_by_index;
+    struct _import_state imports;
+
     // Dictionary of the sys module
     PyObject *sysdict;
     // Dictionary of the builtins module
     PyObject *builtins;
-    // importlib module
-    PyObject *importlib;
-    // override for config->use_frozen_modules (for tests)
-    // (-1: "off", 1: "on", 0: no override)
-    int override_frozen_modules;
 
     PyObject *codec_search_path;
     PyObject *codec_search_cache;
     PyObject *codec_error_registry;
     int codecs_initialized;
 
     PyConfig config;
-#ifdef HAVE_DLOPEN
-    int dlopenflags;
-#endif
     unsigned long feature_flags;
 
     PyObject *dict;  /* Stores per-interpreter state */
 
     PyObject *builtins_copy;
-    PyObject *import_func;
     // Initialized to _PyEval_EvalFrameDefault().
     _PyFrameEvalFunction eval_frame;
 
@@ -205,7 +177,6 @@ struct _is {
 
 /* other API */
 
-extern void _PyInterpreterState_ClearModules(PyInterpreterState *interp);
 extern void _PyInterpreterState_Clear(PyThreadState *tstate);
 
 

Include/internal/pycore_pylifecycle.h

@@ -30,7 +30,6 @@ PyAPI_FUNC(int) _Py_IsLocaleCoercionTarget(const char *ctype_loc);
 /* Various one-time initializers */
 
 extern void _Py_InitVersion(void);
-extern PyStatus _PyImport_Init(void);
 extern PyStatus _PyFaulthandler_Init(int enable);
 extern int _PyTraceMalloc_Init(int enable);
 extern PyObject * _PyBuiltin_Init(PyInterpreterState *interp);
@@ -45,7 +44,6 @@ extern int _PyBuiltins_AddExceptions(PyObject * bltinmod);
 extern PyStatus _Py_HashRandomization_Init(const PyConfig *);
 
 extern PyStatus _PyTime_Init(void);
-extern PyStatus _PyImportZip_Init(PyThreadState *tstate);
 extern PyStatus _PyGC_Init(PyInterpreterState *interp);
 extern PyStatus _PyAtExit_Init(PyInterpreterState *interp);
 extern int _Py_Deepfreeze_Init(void);
@@ -55,8 +53,6 @@ extern int _Py_Deepfreeze_Init(void);
 extern int _PySignal_Init(int install_signal_handlers);
 extern void _PySignal_Fini(void);
 
-extern void _PyImport_Fini(void);
-extern void _PyImport_Fini2(void);
 extern void _PyGC_Fini(PyInterpreterState *interp);
 extern void _Py_HashRandomization_Fini(void);
 extern void _PyFaulthandler_Fini(void);

Include/internal/pycore_pystate.h

@@ -152,12 +152,6 @@ extern void _PySignal_AfterFork(void);
 #endif
 
 
-PyAPI_FUNC(int) _PyState_AddModule(
-    PyThreadState *tstate,
-    PyObject* module,
-    PyModuleDef* def);
-
-
 PyAPI_FUNC(int) _PyOS_InterruptOccurred(PyThreadState *tstate);
 
 #define HEAD_LOCK(runtime) \

Include/internal/pycore_runtime_init.h

@@ -97,23 +97,10 @@ extern "C" {
         ._main_interpreter = _PyInterpreterState_INIT, \
     }
 
-#ifdef HAVE_DLOPEN
-#  include <dlfcn.h>
-#  if HAVE_DECL_RTLD_NOW
-#    define _Py_DLOPEN_FLAGS RTLD_NOW
-#  else
-#    define _Py_DLOPEN_FLAGS RTLD_LAZY
-#  endif
-#  define DLOPENFLAGS_INIT .dlopenflags = _Py_DLOPEN_FLAGS,
-#else
-#  define _Py_DLOPEN_FLAGS 0
-#  define DLOPENFLAGS_INIT
-#endif
-
 #define _PyInterpreterState_INIT \
     { \
         .id_refcount = -1, \
-        DLOPENFLAGS_INIT \
+        .imports = IMPORTS_INIT, \
         .ceval = { \
             .recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \
         }, \

Include/internal/pycore_sysmodule.h

@@ -20,6 +20,9 @@ extern void _PySys_ClearAuditHooks(PyThreadState *tstate);
 
 PyAPI_FUNC(int) _PySys_SetAttr(PyObject *, PyObject *);
 
+extern int _PySys_ClearAttrString(PyInterpreterState *interp,
+                                  const char *name, int verbose);
+
 #ifdef __cplusplus
 }
 #endif

Lib/hashlib.py

@@ -92,13 +92,13 @@ def __get_builtin_constructor(name):
             import _md5
             cache['MD5'] = cache['md5'] = _md5.md5
         elif name in {'SHA256', 'sha256', 'SHA224', 'sha224'}:
-            import _sha256
-            cache['SHA224'] = cache['sha224'] = _sha256.sha224
-            cache['SHA256'] = cache['sha256'] = _sha256.sha256
+            import _sha2
+            cache['SHA224'] = cache['sha224'] = _sha2.sha224
+            cache['SHA256'] = cache['sha256'] = _sha2.sha256
         elif name in {'SHA512', 'sha512', 'SHA384', 'sha384'}:
-            import _sha512
-            cache['SHA384'] = cache['sha384'] = _sha512.sha384
-            cache['SHA512'] = cache['sha512'] = _sha512.sha512
+            import _sha2
+            cache['SHA384'] = cache['sha384'] = _sha2.sha384
+            cache['SHA512'] = cache['sha512'] = _sha2.sha512
         elif name in {'blake2b', 'blake2s'}:
             import _blake2
             cache['blake2b'] = _blake2.blake2b

Lib/test/support/import_helper.py

@@ -105,6 +105,24 @@ def frozen_modules(enabled=True):
         _imp._override_frozen_modules_for_tests(0)
 
 
+@contextlib.contextmanager
+def multi_interp_extensions_check(enabled=True):
+    """Force legacy modules to be allowed in subinterpreters (or not).
+
+    ("legacy" == single-phase init)
+
+    This only applies to modules that haven't been imported yet.
+    It overrides the PyInterpreterConfig.check_multi_interp_extensions
+    setting (see support.run_in_subinterp_with_config() and
+    _xxsubinterpreters.create()).
+    """
+    old = _imp._override_multi_interp_extensions_check(1 if enabled else -1)
+    try:
+        yield
+    finally:
+        _imp._override_multi_interp_extensions_check(old)
+
+
 def import_fresh_module(name, fresh=(), blocked=(), *,
                         deprecated=False,
                         usefrozen=False,