bpo-44441: _PyImport_Fini2() resets PyImport_Inittab (GH-26874)

Py_RunMain() now resets PyImport_Inittab to its initial value at
exit. It must be possible to call PyImport_AppendInittab() or
PyImport_ExtendInittab() at each Python initialization.

Author: vstinner

Doc/c-api/import.rst

@@ -299,4 +299,8 @@ Importing Modules
    field; failure to provide the sentinel value can result in a memory fault.
    Returns ``0`` on success or ``-1`` if insufficient memory could be allocated to
    extend the internal table.  In the event of failure, no modules are added to the
-   internal table.  This should be called before :c:func:`Py_Initialize`.
+   internal table.  This must be called before :c:func:`Py_Initialize`.
+
+   If Python is initialized multiple times, :c:func:`PyImport_AppendInittab` or
+   :c:func:`PyImport_ExtendInittab` must be called before each Python
+   initialization.

Doc/c-api/init_config.rst

@@ -1193,7 +1193,10 @@ The caller is responsible to handle exceptions (error or exit) using
 
 If :c:func:`PyImport_FrozenModules`, :c:func:`PyImport_AppendInittab` or
 :c:func:`PyImport_ExtendInittab` are used, they must be set or called after
-Python preinitialization and before the Python initialization.
+Python preinitialization and before the Python initialization. If Python is
+initialized multiple times, :c:func:`PyImport_AppendInittab` or
+:c:func:`PyImport_ExtendInittab` must be called before each Python
+initialization.
 
 The current configuration (``PyConfig`` type) is stored in
 ``PyInterpreterState.config``.

Lib/test/test_embed.py

@@ -30,6 +30,7 @@
 # _PyCoreConfig_InitIsolatedConfig()
 API_ISOLATED = 3
 
+INIT_LOOPS = 16
 MAX_HASH_SEED = 4294967295
 
 
@@ -111,21 +112,21 @@ def run_repeated_init_and_subinterpreters(self):
         self.assertEqual(err, "")
 
         # The output from _testembed looks like this:
-        # --- Pass 0 ---
+        # --- Pass 1 ---
         # interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728
         # interp 1 <0x1d4f690>, thread state <0x1d35350>: id(modules) = 139650431165784
         # interp 2 <0x1d5a690>, thread state <0x1d99ed0>: id(modules) = 139650413140368
         # interp 3 <0x1d4f690>, thread state <0x1dc3340>: id(modules) = 139650412862200
         # interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728
-        # --- Pass 1 ---
+        # --- Pass 2 ---
         # ...
 
         interp_pat = (r"^interp (\d+) <(0x[\dA-F]+)>, "
                       r"thread state <(0x[\dA-F]+)>: "
                       r"id\(modules\) = ([\d]+)$")
         Interp = namedtuple("Interp", "id interp tstate modules")
 
-        numloops = 0
+        numloops = 1
         current_run = []
         for line in out.splitlines():
             if line == "--- Pass {} ---".format(numloops):
@@ -159,6 +160,8 @@ def run_repeated_init_and_subinterpreters(self):
 
 
 class EmbeddingTests(EmbeddingTestsMixin, unittest.TestCase):
+    maxDiff = 100 * 50
+
     def test_subinterps_main(self):
         for run in self.run_repeated_init_and_subinterpreters():
             main = run[0]
@@ -194,6 +197,14 @@ def test_subinterps_distinct_state(self):
                 self.assertNotEqual(sub.tstate, main.tstate)
                 self.assertNotEqual(sub.modules, main.modules)
 
+    def test_repeated_init_and_inittab(self):
+        out, err = self.run_embedded_interpreter("test_repeated_init_and_inittab")
+        self.assertEqual(err, "")
+
+        lines = [f"--- Pass {i} ---" for i in range(1, INIT_LOOPS+1)]
+        lines = "\n".join(lines) + "\n"
+        self.assertEqual(out, lines)
+
     def test_forced_io_encoding(self):
         # Checks forced configuration of embedded interpreter IO streams
         env = dict(os.environ, PYTHONIOENCODING="utf-8:surrogateescape")

Misc/NEWS.d/next/C API/2021-06-23-12-12-04.bpo-44441.3p14JB.rst

@@ -0,0 +1,4 @@
+:c:func:`Py_RunMain` now resets :c:data:`PyImport_Inittab` to its initial value
+at exit. It must be possible to call :c:func:`PyImport_AppendInittab` or
+:c:func:`PyImport_ExtendInittab` at each Python initialization.
+Patch by Victor Stinner.

Programs/_testembed.c

@@ -22,6 +22,8 @@
 /* Use path starting with "./" avoids a search along the PATH */
 #define PROGRAM_NAME L"./_testembed"
 
+#define INIT_LOOPS 16
+
 // Ignore Py_DEPRECATED() compiler warnings: deprecated functions are
 // tested on purpose here.
 _Py_COMP_DIAG_PUSH
@@ -67,9 +69,8 @@ static int test_repeated_init_and_subinterpreters(void)
 {
     PyThreadState *mainstate, *substate;
     PyGILState_STATE gilstate;
-    int i, j;
 
-    for (i=0; i<15; i++) {
+    for (int i=1; i <= INIT_LOOPS; i++) {
         printf("--- Pass %d ---\n", i);
         _testembed_Py_Initialize();
         mainstate = PyThreadState_Get();
@@ -80,7 +81,7 @@ static int test_repeated_init_and_subinterpreters(void)
         print_subinterp();
         PyThreadState_Swap(NULL);
 
-        for (j=0; j<3; j++) {
+        for (int j=0; j<3; j++) {
             substate = Py_NewInterpreter();
             print_subinterp();
             Py_EndInterpreter(substate);
@@ -96,6 +97,20 @@ static int test_repeated_init_and_subinterpreters(void)
     return 0;
 }
 
+#define EMBEDDED_EXT_NAME "embedded_ext"
+
+static PyModuleDef embedded_ext = {
+    PyModuleDef_HEAD_INIT,
+    .m_name = EMBEDDED_EXT_NAME,
+    .m_size = 0,
+};
+
+static PyObject*
+PyInit_embedded_ext(void)
+{
+    return PyModule_Create(&embedded_ext);
+}
+
 /*****************************************************
  * Test forcing a particular IO encoding
  *****************************************************/
@@ -1790,6 +1805,38 @@ static int list_frozen(void)
 }
 
 
+static int test_repeated_init_and_inittab(void)
+{
+    // bpo-44441: Py_RunMain() must reset PyImport_Inittab at exit.
+    // It must be possible to call PyImport_AppendInittab() or
+    // PyImport_ExtendInittab() before each Python initialization.
+    for (int i=1; i <= INIT_LOOPS; i++) {
+        printf("--- Pass %d ---\n", i);
+
+        // Call PyImport_AppendInittab() at each iteration
+        if (PyImport_AppendInittab(EMBEDDED_EXT_NAME,
+                                   &PyInit_embedded_ext) != 0) {
+            fprintf(stderr, "PyImport_AppendInittab() failed\n");
+            return 1;
+        }
+
+        // Initialize Python
+        wchar_t* argv[] = {PROGRAM_NAME, L"-c", L"pass"};
+        PyConfig config;
+        PyConfig_InitPythonConfig(&config);
+        config.isolated = 1;
+        config_set_argv(&config, Py_ARRAY_LENGTH(argv), argv);
+        init_from_config_clear(&config);
+
+        // Py_RunMain() calls _PyImport_Fini2() which resets PyImport_Inittab
+        int exitcode = Py_RunMain();
+        if (exitcode != 0) {
+            return exitcode;
+        }
+    }
+    return 0;
+}
+
 
 /* *********************************************************
  * List of test cases and the function that implements it.
@@ -1810,8 +1857,10 @@ struct TestCase
 };
 
 static struct TestCase TestCases[] = {
+    // Python initialization
     {"test_forced_io_encoding", test_forced_io_encoding},
     {"test_repeated_init_and_subinterpreters", test_repeated_init_and_subinterpreters},
+    {"test_repeated_init_and_inittab", test_repeated_init_and_inittab},
     {"test_pre_initialization_api", test_pre_initialization_api},
     {"test_pre_initialization_sys_options", test_pre_initialization_sys_options},
     {"test_bpo20891", test_bpo20891},
@@ -1851,6 +1900,7 @@ static struct TestCase TestCases[] = {
     {"test_run_main", test_run_main},
     {"test_get_argc_argv", test_get_argc_argv},
 
+    // Audit
     {"test_open_code_hook", test_open_code_hook},
     {"test_audit", test_audit},
     {"test_audit_subinterpreter", test_audit_subinterpreter},
@@ -1860,11 +1910,13 @@ static struct TestCase TestCases[] = {
     {"test_audit_run_startup", test_audit_run_startup},
     {"test_audit_run_stdin", test_audit_run_stdin},
 
+    // Specific C API
     {"test_unicode_id_init", test_unicode_id_init},
 #ifndef MS_WINDOWS
     {"test_frozenmain", test_frozenmain},
 #endif
 
+    // Command
     {"list_frozen", list_frozen},
     {NULL, NULL}
 };

Python/import.c

@@ -255,6 +255,9 @@ _PyImport_Fini2(void)
     PyMemAllocatorEx old_alloc;
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
+    // Reset PyImport_Inittab
+    PyImport_Inittab = _PyImport_Inittab;
+
     /* Free memory allocated by PyImport_ExtendInittab() */
     PyMem_RawFree(inittab_copy);
     inittab_copy = NULL;