bpo-45020: Add -X frozen_modules=[on|off] to explicitly control use of frozen modules.

Doc/using/cmdline.rst

@@ -480,6 +480,12 @@ Miscellaneous options
      objects and pyc files are desired as well as supressing the extra visual
      location indicators when the interpreter displays tracebacks. See also
      :envvar:`PYTHONNODEBUGRANGES`.
+   * ``-X frozen_modules`` determines whether or not frozen modules are
+     ignored by the import machinery.  A value of "on" means they get
+     imported and "off" means they are ignored.  The default is "on"
+     for non-debug builds (the normal case) and "off" for debug builds.
+     Note that the "importlib_bootstrap" and "importlib_bootstrap_external"
+     frozen modules are always used, even if this flag is set to "off".
 
    It also allows passing arbitrary values and retrieving them through the
    :data:`sys._xoptions` dictionary.
@@ -518,6 +524,9 @@ Miscellaneous options
    .. versionadded:: 3.11
       The ``-X no_debug_ranges`` option.
 
+   .. versionadded:: 3.11
+      The ``-X frozen_modules`` option.
+
 
 Options you shouldn't use
 ~~~~~~~~~~~~~~~~~~~~~~~~~

Include/cpython/initconfig.h

@@ -172,6 +172,7 @@ typedef struct PyConfig {
     int legacy_windows_stdio;
 #endif
     wchar_t *check_hash_pycs_mode;
+    int use_frozen_modules;
 
     /* --- Path configuration inputs ------------ */
     int pathconfig_warnings;

Include/internal/pycore_initconfig.h

@@ -155,6 +155,7 @@ extern PyStatus _PyConfig_Copy(
 extern PyStatus _PyConfig_InitPathConfig(
     PyConfig *config,
     int compute_path_config);
+extern PyStatus _PyConfig_InitImportConfig(PyConfig *config);
 extern PyStatus _PyConfig_Read(PyConfig *config, int compute_path_config);
 extern PyStatus _PyConfig_Write(const PyConfig *config,
     struct pyruntimestate *runtime);

Include/internal/pycore_interp.h

@@ -246,6 +246,9 @@ struct _is {
     PyObject *builtins;
     // importlib module
     PyObject *importlib;
+    // override for config->use_frozen_modules (for tests)
+    // (-1: "off", 1: "on", 0: no override)
+    int override_frozen_modules;
 
     /* Used in Modules/_threadmodule.c. */
     long num_threads;

Lib/ctypes/test/test_values.py

@@ -72,14 +72,16 @@ class struct_frozen(Structure):
                 self.assertGreater(abs(entry.size), 10)
                 self.assertTrue([entry.code[i] for i in range(abs(entry.size))])
                 # Check the module's package-ness.
-                spec = importlib.util.find_spec(modname)
+                with import_helper.frozen_modules():
+                    spec = importlib.util.find_spec(modname)
                 if entry.size < 0:
                     # It's a package.
                     self.assertIsNotNone(spec.submodule_search_locations)
                 else:
                     self.assertIsNone(spec.submodule_search_locations)
 
-        expected = imp._frozen_module_names()
+        with import_helper.frozen_modules():
+            expected = imp._frozen_module_names()
         self.maxDiff = None
         self.assertEqual(modules, expected, "PyImport_FrozenModules example "
             "in Doc/library/ctypes.rst may be out of date")

Lib/test/support/import_helper.py

@@ -1,4 +1,5 @@
 import contextlib
+import _imp
 import importlib
 import importlib.util
 import os
@@ -109,7 +110,24 @@ def _save_and_block_module(name, orig_modules):
     return saved
 
 
-def import_fresh_module(name, fresh=(), blocked=(), deprecated=False):
+@contextlib.contextmanager
+def frozen_modules(enabled=True):
+    """Force frozen modules to be used (or not).
+
+    This only applies to modules that haven't been imported yet.
+    Also, some essential modules will always be imported frozen.
+    """
+    _imp._override_frozen_modules_for_tests(1 if enabled else -1)
+    try:
+        yield
+    finally:
+        _imp._override_frozen_modules_for_tests(0)
+
+
+def import_fresh_module(name, fresh=(), blocked=(), *,
+                        deprecated=False,
+                        usefrozen=False,
+                        ):
     """Import and return a module, deliberately bypassing sys.modules.
 
     This function imports and returns a fresh copy of the named Python module
@@ -133,6 +151,9 @@ def import_fresh_module(name, fresh=(), blocked=(), deprecated=False):
 
     This function will raise ImportError if the named module cannot be
     imported.
+
+    If "usefrozen" is False (the default) then the frozen importer is
+    disabled (except for essential modules like importlib._bootstrap).
     """
     # NOTE: test_heapq, test_json and test_warnings include extra sanity checks
     # to make sure that this utility function is working as expected
@@ -148,7 +169,8 @@ def import_fresh_module(name, fresh=(), blocked=(), deprecated=False):
             for blocked_name in blocked:
                 if not _save_and_block_module(blocked_name, orig_modules):
                     names_to_remove.append(blocked_name)
-            fresh_module = importlib.import_module(name)
+            with frozen_modules(usefrozen):
+                fresh_module = importlib.import_module(name)
         except ImportError:
             fresh_module = None
         finally:
@@ -169,9 +191,12 @@ class CleanImport(object):
 
         with CleanImport("foo"):
             importlib.import_module("foo") # new reference
+
+    If "usefrozen" is False (the default) then the frozen importer is
+    disabled (except for essential modules like importlib._bootstrap).
     """
 
-    def __init__(self, *module_names):
+    def __init__(self, *module_names, usefrozen=False):
         self.original_modules = sys.modules.copy()
         for module_name in module_names:
             if module_name in sys.modules:
@@ -183,12 +208,15 @@ def __init__(self, *module_names):
                 if module.__name__ != module_name:
                     del sys.modules[module.__name__]
                 del sys.modules[module_name]
+        self._frozen_modules = frozen_modules(usefrozen)
 
     def __enter__(self):
+        self._frozen_modules.__enter__()
         return self
 
     def __exit__(self, *ignore_exc):
         sys.modules.update(self.original_modules)
+        self._frozen_modules.__exit__(*ignore_exc)
 
 
 class DirsOnSysPath(object):

Lib/test/support/os_helper.py

@@ -599,6 +599,10 @@ def set(self, envvar, value):
     def unset(self, envvar):
         del self[envvar]
 
+    def copy(self):
+        # We do what os.environ.copy() does.
+        return dict(self)
+
     def __enter__(self):
         return self
 

Lib/test/test_embed.py

@@ -12,6 +12,7 @@
 import shutil
 import subprocess
 import sys
+import sysconfig
 import tempfile
 import textwrap
 
@@ -426,6 +427,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'pathconfig_warnings': 1,
         '_init_main': 1,
         '_isolated_interpreter': 0,
+        'use_frozen_modules': False,
     }
     if MS_WINDOWS:
         CONFIG_COMPAT.update({

Lib/test/test_frozen.py

@@ -12,16 +12,17 @@
 
 import sys
 import unittest
-from test.support import captured_stdout
+from test.support import captured_stdout, import_helper
 
 
 class TestFrozen(unittest.TestCase):
     def test_frozen(self):
         name = '__hello__'
         if name in sys.modules:
             del sys.modules[name]
-        with captured_stdout() as out:
-            import __hello__
+        with import_helper.frozen_modules():
+            with captured_stdout() as out:
+                import __hello__
         self.assertEqual(out.getvalue(), 'Hello world!\n')
 
 

Lib/test/test_importlib/frozen/test_finder.py

@@ -6,14 +6,17 @@
 import unittest
 import warnings
 
+from test.support import import_helper
+
 
 class FindSpecTests(abc.FinderTests):
 
     """Test finding frozen modules."""
 
     def find(self, name, path=None):
         finder = self.machinery.FrozenImporter
-        return finder.find_spec(name, path)
+        with import_helper.frozen_modules():
+            return finder.find_spec(name, path)
 
     def test_module(self):
         name = '__hello__'
@@ -52,7 +55,8 @@ def find(self, name, path=None):
         finder = self.machinery.FrozenImporter
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", DeprecationWarning)
-            return finder.find_module(name, path)
+            with import_helper.frozen_modules():
+                return finder.find_module(name, path)
 
     def test_module(self):
         name = '__hello__'