Merge branch 'main' into isolate-io/winconsoleio

Author: erlend-aasland

Include/internal/pycore_ceval.h

@@ -100,7 +100,9 @@ extern int _PyEval_ThreadsInitialized(void);
 extern PyStatus _PyEval_InitGIL(PyThreadState *tstate, int own_gil);
 extern void _PyEval_FiniGIL(PyInterpreterState *interp);
 
+extern void _PyEval_AcquireLock(PyThreadState *tstate);
 extern void _PyEval_ReleaseLock(PyThreadState *tstate);
+extern PyThreadState * _PyThreadState_SwapNoGIL(PyThreadState *);
 
 extern void _PyEval_DeactivateOpCache(void);
 

Lib/test/test_import/__init__.py

@@ -1861,6 +1861,26 @@ def test_multi_init_extension_non_isolated_compat(self):
         with self.subTest(f'{modname}: not strict'):
             self.check_compatible_here(modname, filename, strict=False)
 
+    @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module")
+    def test_multi_init_extension_per_interpreter_gil_compat(self):
+        modname = '_test_shared_gil_only'
+        filename = _testmultiphase.__file__
+        loader = ExtensionFileLoader(modname, filename)
+        spec = importlib.util.spec_from_loader(modname, loader)
+        module = importlib.util.module_from_spec(spec)
+        loader.exec_module(module)
+        sys.modules[modname] = module
+
+        require_extension(module)
+        with self.subTest(f'{modname}: isolated, strict'):
+            self.check_incompatible_here(modname, filename, isolated=True)
+        with self.subTest(f'{modname}: not isolated, strict'):
+            self.check_compatible_here(modname, filename,
+                                       strict=True, isolated=False)
+        with self.subTest(f'{modname}: not isolated, not strict'):
+            self.check_compatible_here(modname, filename,
+                                       strict=False, isolated=False)
+
     def test_python_compat(self):
         module = 'threading'
         require_pure_python(module)

Lib/test/test_importlib/extension/test_loader.py

@@ -348,6 +348,8 @@ def test_bad_modules(self):
                 'exec_err',
                 'exec_raise',
                 'exec_unreported_exception',
+                'multiple_create_slots',
+                'multiple_multiple_interpreters_slots',
                 ]:
             with self.subTest(name_base):
                 name = self.name + '_' + name_base

Misc/NEWS.d/next/Core and Builtins/2023-05-05-13-18-56.gh-issue-99113.hT1ajK.rst

@@ -0,0 +1,11 @@
+Multi-phase init extension modules may now indicate that they support
+running in subinterpreters that have their own GIL.  This is done by using
+``Py_MOD_PER_INTERPRETER_GIL_SUPPORTED`` as the value for the
+``Py_mod_multiple_interpreters`` module def slot.  Otherwise the module, by
+default, cannot be imported in such subinterpreters.  (This does not affect
+the main interpreter or subinterpreters that do not have their own GIL.)  In
+addition to the isolation that multi-phase init already normally requires,
+support for per-interpreter GIL involves one additional constraint:
+thread-safety.  If the module has external (linked) dependencies and those
+libraries have any state that isn't thread-safe then the module must do the
+additional work to add thread-safety.  This should be an uncommon case.

Modules/_io/_iomodule.c

@@ -580,7 +580,6 @@ iomodule_traverse(PyObject *mod, visitproc visit, void *arg) {
     _PyIO_State *state = get_io_state(mod);
     if (!state->initialized)
         return 0;
-    Py_VISIT(state->locale_module);
     Py_VISIT(state->unsupported_operation);
 
     Py_VISIT(state->PyIncrementalNewlineDecoder_Type);
@@ -608,8 +607,6 @@ iomodule_clear(PyObject *mod) {
     _PyIO_State *state = get_io_state(mod);
     if (!state->initialized)
         return 0;
-    if (state->locale_module != NULL)
-        Py_CLEAR(state->locale_module);
     Py_CLEAR(state->unsupported_operation);
 
     Py_CLEAR(state->PyIncrementalNewlineDecoder_Type);

Modules/_io/_iomodule.h

@@ -143,8 +143,6 @@ extern PyModuleDef _PyIO_Module;
 
 typedef struct {
     int initialized;
-    PyObject *locale_module;
-
     PyObject *unsupported_operation;
 
     /* Types */

Modules/_testmultiphase.c

@@ -681,6 +681,27 @@ PyInit__testmultiphase_export_unreported_exception(void)
     return PyModuleDef_Init(&main_def);
 }
 
+static PyObject*
+createfunc_noop(PyObject *spec, PyModuleDef *def)
+{
+    return PyModule_New("spam");
+}
+
+static PyModuleDef_Slot slots_multiple_create_slots[] = {
+    {Py_mod_create, createfunc_noop},
+    {Py_mod_create, createfunc_noop},
+    {0, NULL},
+};
+
+static PyModuleDef def_multiple_create_slots = TEST_MODULE_DEF(
+    "_testmultiphase_multiple_create_slots", slots_multiple_create_slots, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_multiple_create_slots(void)
+{
+    return PyModuleDef_Init(&def_multiple_create_slots);
+}
+
 static PyObject*
 createfunc_null(PyObject *spec, PyModuleDef *def)
 {
@@ -892,7 +913,24 @@ PyInit__test_module_state_shared(void)
 }
 
 
-/* multiple interpreters supports */
+/* multiple interpreters support */
+
+static PyModuleDef_Slot slots_multiple_multiple_interpreters_slots[] = {
+    {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {0, NULL},
+};
+
+static PyModuleDef def_multiple_multiple_interpreters_slots = TEST_MODULE_DEF(
+    "_testmultiphase_multiple_multiple_interpreters_slots",
+    slots_multiple_multiple_interpreters_slots,
+    NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_multiple_multiple_interpreters_slots(void)
+{
+    return PyModuleDef_Init(&def_multiple_multiple_interpreters_slots);
+}
 
 static PyModuleDef_Slot non_isolated_slots[] = {
     {Py_mod_exec, execfunc},
@@ -909,3 +947,23 @@ PyInit__test_non_isolated(void)
 {
     return PyModuleDef_Init(&non_isolated_def);
 }
+
+
+static PyModuleDef_Slot shared_gil_only_slots[] = {
+    {Py_mod_exec, execfunc},
+    /* Note that Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED is the default.
+       We put it here explicitly to draw attention to the contrast
+       with Py_MOD_PER_INTERPRETER_GIL_SUPPORTED. */
+    {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED},
+    {0, NULL},
+};
+
+static PyModuleDef shared_gil_only_def = TEST_MODULE_DEF("_test_shared_gil_only",
+                                                         shared_gil_only_slots,
+                                                         testexport_methods);
+
+PyMODINIT_FUNC
+PyInit__test_shared_gil_only(void)
+{
+    return PyModuleDef_Init(&shared_gil_only_def);
+}

Objects/moduleobject.c

@@ -323,7 +323,13 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio
             goto error;
         }
     }
-    // XXX Do a similar check once we have PyInterpreterState.ceval.own_gil.
+    else if (multiple_interpreters != Py_MOD_PER_INTERPRETER_GIL_SUPPORTED
+             && interp->ceval.own_gil
+             && !_Py_IsMainInterpreter(interp)
+             && _PyImport_CheckSubinterpIncompatibleExtensionAllowed(name) < 0)
+    {
+        goto error;
+    }
 
     if (create) {
         m = create(spec, def);

Python/ceval_gil.c

@@ -499,42 +499,66 @@ PyEval_ThreadsInitialized(void)
     return _PyEval_ThreadsInitialized();
 }
 
+static inline int
+current_thread_holds_gil(struct _gil_runtime_state *gil, PyThreadState *tstate)
+{
+    if (((PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) != tstate) {
+        return 0;
+    }
+    return _Py_atomic_load_relaxed(&gil->locked);
+}
+
+static void
+init_shared_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil)
+{
+    assert(gil_created(gil));
+    interp->ceval.gil = gil;
+    interp->ceval.own_gil = 0;
+}
+
+static void
+init_own_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil)
+{
+    assert(!gil_created(gil));
+    create_gil(gil);
+    assert(gil_created(gil));
+    interp->ceval.gil = gil;
+    interp->ceval.own_gil = 1;
+}
+
 PyStatus
 _PyEval_InitGIL(PyThreadState *tstate, int own_gil)
 {
     assert(tstate->interp->ceval.gil == NULL);
+    int locked;
     if (!own_gil) {
         PyInterpreterState *main_interp = _PyInterpreterState_Main();
         assert(tstate->interp != main_interp);
         struct _gil_runtime_state *gil = main_interp->ceval.gil;
-        assert(gil_created(gil));
-        tstate->interp->ceval.gil = gil;
-        tstate->interp->ceval.own_gil = 0;
-        return _PyStatus_OK();
+        init_shared_gil(tstate->interp, gil);
+        locked = current_thread_holds_gil(gil, tstate);
     }
-
     /* XXX per-interpreter GIL */
-    struct _gil_runtime_state *gil = &tstate->interp->runtime->ceval.gil;
-    if (!_Py_IsMainInterpreter(tstate->interp)) {
+    else if (!_Py_IsMainInterpreter(tstate->interp)) {
         /* Currently, the GIL is shared by all interpreters,
            and only the main interpreter is responsible to create
            and destroy it. */
-        assert(gil_created(gil));
-        tstate->interp->ceval.gil = gil;
+        struct _gil_runtime_state *main_gil = _PyInterpreterState_Main()->ceval.gil;
+        init_shared_gil(tstate->interp, main_gil);
         // XXX For now we lie.
         tstate->interp->ceval.own_gil = 1;
-        return _PyStatus_OK();
+        locked = current_thread_holds_gil(main_gil, tstate);
+    }
+    else {
+        PyThread_init_thread();
+        // XXX per-interpreter GIL: switch to interp->_gil.
+        init_own_gil(tstate->interp, &tstate->interp->runtime->ceval.gil);
+        locked = 0;
+    }
+    if (!locked) {
+        take_gil(tstate);
     }
-    assert(own_gil);
-
-    assert(!gil_created(gil));
 
-    PyThread_init_thread();
-    create_gil(gil);
-    assert(gil_created(gil));
-    tstate->interp->ceval.gil = gil;
-    tstate->interp->ceval.own_gil = 1;
-    take_gil(tstate);
     return _PyStatus_OK();
 }
 
@@ -611,9 +635,17 @@ PyEval_ReleaseLock(void)
     drop_gil(ceval, tstate);
 }
 
+void
+_PyEval_AcquireLock(PyThreadState *tstate)
+{
+    _Py_EnsureTstateNotNULL(tstate);
+    take_gil(tstate);
+}
+
 void
 _PyEval_ReleaseLock(PyThreadState *tstate)
 {
+    _Py_EnsureTstateNotNULL(tstate);
     struct _ceval_state *ceval = &tstate->interp->ceval;
     drop_gil(ceval, tstate);
 }
@@ -625,7 +657,7 @@ PyEval_AcquireThread(PyThreadState *tstate)
 
     take_gil(tstate);
 
-    if (_PyThreadState_Swap(tstate->interp->runtime, tstate) != NULL) {
+    if (_PyThreadState_SwapNoGIL(tstate) != NULL) {
         Py_FatalError("non-NULL old thread state");
     }
 }
@@ -635,8 +667,7 @@ PyEval_ReleaseThread(PyThreadState *tstate)
 {
     assert(is_tstate_valid(tstate));
 
-    _PyRuntimeState *runtime = tstate->interp->runtime;
-    PyThreadState *new_tstate = _PyThreadState_Swap(runtime, NULL);
+    PyThreadState *new_tstate = _PyThreadState_SwapNoGIL(NULL);
     if (new_tstate != tstate) {
         Py_FatalError("wrong thread state");
     }
@@ -684,8 +715,7 @@ _PyEval_SignalAsyncExc(PyInterpreterState *interp)
 PyThreadState *
 PyEval_SaveThread(void)
 {
-    _PyRuntimeState *runtime = &_PyRuntime;
-    PyThreadState *tstate = _PyThreadState_Swap(runtime, NULL);
+    PyThreadState *tstate = _PyThreadState_SwapNoGIL(NULL);
     _Py_EnsureTstateNotNULL(tstate);
 
     struct _ceval_state *ceval = &tstate->interp->ceval;
@@ -701,7 +731,7 @@ PyEval_RestoreThread(PyThreadState *tstate)
 
     take_gil(tstate);
 
-    _PyThreadState_Swap(tstate->interp->runtime, tstate);
+    _PyThreadState_SwapNoGIL(tstate);
 }
 
 
@@ -1005,7 +1035,7 @@ _Py_HandlePending(PyThreadState *tstate)
     /* GIL drop request */
     if (_Py_atomic_load_relaxed_int32(&interp_ceval_state->gil_drop_request)) {
         /* Give another thread a chance */
-        if (_PyThreadState_Swap(runtime, NULL) != tstate) {
+        if (_PyThreadState_SwapNoGIL(NULL) != tstate) {
             Py_FatalError("tstate mix-up");
         }
         drop_gil(interp_ceval_state, tstate);
@@ -1014,7 +1044,7 @@ _Py_HandlePending(PyThreadState *tstate)
 
         take_gil(tstate);
 
-        if (_PyThreadState_Swap(runtime, tstate) != NULL) {
+        if (_PyThreadState_SwapNoGIL(tstate) != NULL) {
             Py_FatalError("orphan tstate");
         }
     }