bpo-45020: Freeze the modules imported during startup.

.gitattributes

@@ -47,6 +47,7 @@ Objects/clinic/*.h          linguist-generated=true
 PC/clinic/*.h               linguist-generated=true
 Python/clinic/*.h           linguist-generated=true
 Python/frozen_modules/*.h   linguist-generated=true
+Python/frozen_modules/MANIFEST  linguist-generated=true
 Include/internal/pycore_ast.h   linguist-generated=true
 Python/Python-ast.c         linguist-generated=true
 Include/opcode.h            linguist-generated=true

.gitignore

@@ -121,6 +121,13 @@ Tools/msi/obj
 Tools/ssl/amd64
 Tools/ssl/win32
 
+# TODO: Once we auto-regen frozem modules for Windows builds
+# we can drop the .h files from the repo and ignore them here.
+# At that point we will rely the frozen manifest file to identify
+# changed generated files.  We'll drop the entry for it then.
+# See: Tools/scripts/freeze_modules.py.
+#Python/frozen_modules/*.h
+
 # Two-trick pony for OSX and other case insensitive file systems:
 # Ignore ./python binary on Unix but still look into ./Python/ directory.
 /python

Doc/using/cmdline.rst

@@ -480,6 +480,13 @@ Miscellaneous options
      objects and pyc files are desired as well as supressing the extra visual
      location indicators when the interpreter displays tracebacks. See also
      :envvar:`PYTHONNODEBUGRANGES`.
+   * ``-X frozen_modules=[on|off]`` determines whether or not frozen modules
+     are ignored by the import machinery.  A value of "on" means they get
+     imported and "off" means they are ignored.  The default is "on"
+     if this is an installed Python (the normal case).  If it's under
+     development (running from the build dir) then the default is "off".
+     Note that the "importlib_bootstrap" and "importlib_bootstrap_external"
+     frozen modules are always used, even if this flag is set to "off".
 
    It also allows passing arbitrary values and retrieving them through the
    :data:`sys._xoptions` dictionary.
@@ -518,6 +525,9 @@ Miscellaneous options
    .. versionadded:: 3.11
       The ``-X no_debug_ranges`` option.
 
+   .. versionadded:: 3.11
+      The ``-X frozen_modules`` option.
+
 
 Options you shouldn't use
 ~~~~~~~~~~~~~~~~~~~~~~~~~

Include/cpython/initconfig.h

@@ -2,6 +2,8 @@
 #define Py_PYCORECONFIG_H
 #ifndef Py_LIMITED_API
 
+#include <stdbool.h>
+
 /* --- PyStatus ----------------------------------------------- */
 
 typedef struct {
@@ -172,6 +174,7 @@ typedef struct PyConfig {
     int legacy_windows_stdio;
 #endif
     wchar_t *check_hash_pycs_mode;
+    bool use_frozen_modules;
 
     /* --- Path configuration inputs ------------ */
     int pathconfig_warnings;

Include/internal/pycore_initconfig.h

@@ -155,6 +155,7 @@ extern PyStatus _PyConfig_Copy(
 extern PyStatus _PyConfig_InitPathConfig(
     PyConfig *config,
     int compute_path_config);
+extern PyStatus _PyConfig_InitImportConfig(PyConfig *config);
 extern PyStatus _PyConfig_Read(PyConfig *config, int compute_path_config);
 extern PyStatus _PyConfig_Write(const PyConfig *config,
     struct pyruntimestate *runtime);

Include/internal/pycore_pylifecycle.h

@@ -12,6 +12,7 @@ extern "C" {
 #include <signal.h>
 #endif
 
+#include <stdbool.h>
 #include "pycore_runtime.h"       // _PyRuntimeState
 
 #ifndef NSIG
@@ -122,7 +123,6 @@ PyAPI_FUNC(PyStatus) _Py_PreInitializeFromConfig(
     const PyConfig *config,
     const struct _PyArgv *args);
 
-
 PyAPI_FUNC(int) _Py_HandleSystemExit(int *exitcode_p);
 
 PyAPI_FUNC(PyObject*) _PyErr_WriteUnraisableDefaultHook(PyObject *unraisable);

Include/internal/pycore_pystate.h

@@ -30,6 +30,17 @@ _Py_IsMainInterpreter(PyInterpreterState *interp)
 }
 
 
+static inline const PyConfig *
+_Py_GetMainConfig(void)
+{
+    PyInterpreterState *interp = _PyRuntime.interpreters.main;
+    if (interp == NULL) {
+        return NULL;
+    }
+    return _PyInterpreterState_GetConfig(interp);
+}
+
+
 /* Only handle signals on the main thread of the main interpreter. */
 static inline int
 _Py_ThreadCanHandleSignals(PyInterpreterState *interp)

Lib/ctypes/test/test_values.py

@@ -2,9 +2,12 @@
 A testcase which accesses *values* in a dll.
 """
 
+import imp
+import importlib.util
 import unittest
 import sys
 from ctypes import *
+from test.support import import_helper, captured_stdout
 
 import _ctypes_test
 
@@ -55,41 +58,34 @@ class struct_frozen(Structure):
 
         ft = FrozenTable.in_dll(pythonapi, "PyImport_FrozenModules")
         # ft is a pointer to the struct_frozen entries:
-        items = []
-        # _frozen_importlib changes size whenever importlib._bootstrap
-        # changes, so it gets a special case.  We should make sure it's
-        # found, but don't worry about its size too much.  The same
-        # applies to _frozen_importlib_external.
-        bootstrap_seen = []
-        bootstrap_expected = [
-                b'_frozen_importlib',
-                b'_frozen_importlib_external',
-                b'zipimport',
-                ]
+        modules = []
         for entry in ft:
             # This is dangerous. We *can* iterate over a pointer, but
             # the loop will not terminate (maybe with an access
             # violation;-) because the pointer instance has no size.
             if entry.name is None:
                 break
-
-            if entry.name in bootstrap_expected:
-                bootstrap_seen.append(entry.name)
-                self.assertTrue(entry.size,
-                    "{!r} was reported as having no size".format(entry.name))
-                continue
-            items.append((entry.name.decode("ascii"), entry.size))
-
-        expected = [("__hello__", 164),
-                    ("__phello__", -164),
-                    ("__phello__.spam", 164),
-                    ]
-        self.assertEqual(items, expected, "PyImport_FrozenModules example "
+            modname = entry.name.decode("ascii")
+            modules.append(modname)
+            with self.subTest(modname):
+                # Do a sanity check on entry.size and entry.code.
+                self.assertGreater(abs(entry.size), 10)
+                self.assertTrue([entry.code[i] for i in range(abs(entry.size))])
+                # Check the module's package-ness.
+                with import_helper.frozen_modules(), captured_stdout():
+                    spec = importlib.util.find_spec(modname)
+                if entry.size < 0:
+                    # It's a package.
+                    self.assertIsNotNone(spec.submodule_search_locations)
+                else:
+                    self.assertIsNone(spec.submodule_search_locations)
+
+        with import_helper.frozen_modules():
+            expected = imp._frozen_module_names()
+        self.maxDiff = None
+        self.assertEqual(modules, expected, "PyImport_FrozenModules example "
             "in Doc/library/ctypes.rst may be out of date")
 
-        self.assertEqual(sorted(bootstrap_seen), bootstrap_expected,
-            "frozen bootstrap modules did not match PyImport_FrozenModules")
-
         from ctypes import _pointer_type_cache
         del _pointer_type_cache[struct_frozen]
 

Lib/idlelib/idle_test/test_query.py

@@ -136,8 +136,8 @@ def test_good_module_name(self):
         dialog = self.Dummy_ModuleName('idlelib')
         self.assertTrue(dialog.entry_ok().endswith('__init__.py'))
         self.assertEqual(dialog.entry_error['text'], '')
-        dialog = self.Dummy_ModuleName('os.path')
-        self.assertTrue(dialog.entry_ok().endswith('path.py'))
+        dialog = self.Dummy_ModuleName('idlelib.idle')
+        self.assertTrue(dialog.entry_ok().endswith('idle.py'))
         self.assertEqual(dialog.entry_error['text'], '')
 
 

Lib/imp.py

@@ -9,7 +9,7 @@
 from _imp import (lock_held, acquire_lock, release_lock,
                   get_frozen_object, is_frozen_package,
                   init_frozen, is_builtin, is_frozen,
-                  _fix_co_filename)
+                  _fix_co_filename, _frozen_module_names)
 try:
     from _imp import create_dynamic
 except ImportError:

Lib/test/support/import_helper.py

@@ -109,7 +109,20 @@ def _save_and_block_module(name, orig_modules):
     return saved
 
 
-def import_fresh_module(name, fresh=(), blocked=(), deprecated=False):
+@contextlib.contextmanager
+def frozen_modules(enabled=True):
+    # FYI: the env var will never show up in os.environ.
+    os.putenv('_PYTHONTESTFROZENMODULES', '1' if enabled else '0')
+    try:
+        yield
+    finally:
+        os.unsetenv('_PYTHONTESTFROZENMODULES')
+
+
+def import_fresh_module(name, fresh=(), blocked=(), *,
+                        deprecated=False,
+                        usefrozen=False,
+                        ):
     """Import and return a module, deliberately bypassing sys.modules.
 
     This function imports and returns a fresh copy of the named Python module
@@ -148,7 +161,8 @@ def import_fresh_module(name, fresh=(), blocked=(), deprecated=False):
             for blocked_name in blocked:
                 if not _save_and_block_module(blocked_name, orig_modules):
                     names_to_remove.append(blocked_name)
-            fresh_module = importlib.import_module(name)
+            with frozen_modules(usefrozen):
+                fresh_module = importlib.import_module(name)
         except ImportError:
             fresh_module = None
         finally:
@@ -171,7 +185,7 @@ class CleanImport(object):
             importlib.import_module("foo") # new reference
     """
 
-    def __init__(self, *module_names):
+    def __init__(self, *module_names, usefrozen=False):
         self.original_modules = sys.modules.copy()
         for module_name in module_names:
             if module_name in sys.modules:
@@ -183,12 +197,15 @@ def __init__(self, *module_names):
                 if module.__name__ != module_name:
                     del sys.modules[module.__name__]
                 del sys.modules[module_name]
+        self._frozen_modules = frozen_modules(usefrozen)
 
     def __enter__(self):
+        self._frozen_modules.__enter__()
         return self
 
     def __exit__(self, *ignore_exc):
         sys.modules.update(self.original_modules)
+        self._frozen_modules.__exit__(*ignore_exc)
 
 
 class DirsOnSysPath(object):

Lib/test/support/os_helper.py

@@ -599,6 +599,10 @@ def set(self, envvar, value):
     def unset(self, envvar):
         del self[envvar]
 
+    def copy(self):
+        # We do what os.environ.copy() does.
+        return dict(self)
+
     def __enter__(self):
         return self
 

Lib/test/test_cmd_line_script.py

@@ -474,7 +474,6 @@ def test_dash_m_errors(self):
                 br'ModuleNotFoundError'),
             ('builtins.x.y', br'Error while finding module specification.*'
                 br'ModuleNotFoundError.*No module named.*not a package'),
-            ('os.path', br'loader.*cannot handle'),
             ('importlib', br'No module named.*'
                 br'is a package and cannot be directly executed'),
             ('importlib.nonexistent', br'No module named'),

Lib/test/test_embed.py

@@ -12,6 +12,7 @@
 import shutil
 import subprocess
 import sys
+import sysconfig
 import tempfile
 import textwrap
 
@@ -426,11 +427,16 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'pathconfig_warnings': 1,
         '_init_main': 1,
         '_isolated_interpreter': 0,
+        'use_frozen_modules': False,
     }
     if MS_WINDOWS:
         CONFIG_COMPAT.update({
             'legacy_windows_stdio': 0,
         })
+    else:
+        config_args = sysconfig.get_config_var('CONFIG_ARGS') or ''
+        if '--with-address-sanitizer' in config_args:
+            CONFIG_COMPAT['use_frozen_modules'] = True
 
     CONFIG_PYTHON = dict(CONFIG_COMPAT,
         _config_init=API_PYTHON,
@@ -1244,7 +1250,9 @@ def test_init_setpythonhome(self):
             'pythonpath_env': paths_str,
         }
         self.default_program_name(config)
-        env = {'TESTHOME': home, 'PYTHONPATH': paths_str}
+        env = {'TESTHOME': home,
+               'PYTHONPATH': paths_str,
+               '_PYTHONTESTFROZENMODULES': '1'}
         self.check_all_configs("test_init_setpythonhome", config,
                                api=API_COMPAT, env=env)
 

Lib/test/test_frozen.py

@@ -12,16 +12,17 @@
 
 import sys
 import unittest
-from test.support import captured_stdout
+from test.support import captured_stdout, import_helper
 
 
 class TestFrozen(unittest.TestCase):
     def test_frozen(self):
         name = '__hello__'
         if name in sys.modules:
             del sys.modules[name]
-        with captured_stdout() as out:
-            import __hello__
+        with import_helper.frozen_modules():
+            with captured_stdout() as out:
+                import __hello__
         self.assertEqual(out.getvalue(), 'Hello world!\n')
 
 

Lib/test/test_imp.py

@@ -16,6 +16,9 @@
 import _imp
 
 
+OS_PATH_NAME = os.path.__name__
+
+
 def requires_load_dynamic(meth):
     """Decorator to skip a test if not running under CPython or lacking
     imp.load_dynamic()."""
@@ -213,15 +216,17 @@ def test_load_from_source(self):
         # state after reversion. Reinitialising the module contents
         # and just reverting os.environ to its previous state is an OK
         # workaround
-        orig_path = os.path
-        orig_getenv = os.getenv
-        with os_helper.EnvironmentVarGuard():
-            x = imp.find_module("os")
-            self.addCleanup(x[0].close)
-            new_os = imp.load_module("os", *x)
-            self.assertIs(os, new_os)
-            self.assertIs(orig_path, new_os.path)
-            self.assertIsNot(orig_getenv, new_os.getenv)
+        with import_helper.CleanImport('os', 'os.path', OS_PATH_NAME):
+            import os
+            orig_path = os.path
+            orig_getenv = os.getenv
+            with os_helper.EnvironmentVarGuard():
+                x = imp.find_module("os")
+                self.addCleanup(x[0].close)
+                new_os = imp.load_module("os", *x)
+                self.assertIs(os, new_os)
+                self.assertIs(orig_path, new_os.path)
+                self.assertIsNot(orig_getenv, new_os.getenv)
 
     @requires_load_dynamic
     def test_issue15828_load_extensions(self):

Lib/test/test_import/__init__.py

@@ -21,7 +21,7 @@
 from test.support import os_helper
 from test.support import (is_jython, swap_attr, swap_item, cpython_only)
 from test.support.import_helper import (
-    forget, make_legacy_pyc, unlink, unload, DirsOnSysPath)
+    forget, make_legacy_pyc, unlink, unload, DirsOnSysPath, CleanImport)
 from test.support.os_helper import (
     TESTFN, rmtree, temp_umask, TESTFN_UNENCODABLE, temp_dir)
 from test.support import script_helper
@@ -86,8 +86,10 @@ def test_from_import_missing_attr_raises_ImportError(self):
             from importlib import something_that_should_not_exist_anywhere
 
     def test_from_import_missing_attr_has_name_and_path(self):
-        with self.assertRaises(ImportError) as cm:
-            from os import i_dont_exist
+        with CleanImport('os'):
+            import os
+            with self.assertRaises(ImportError) as cm:
+                from os import i_dont_exist
         self.assertEqual(cm.exception.name, 'os')
         self.assertEqual(cm.exception.path, os.__file__)
         self.assertRegex(str(cm.exception), r"cannot import name 'i_dont_exist' from 'os' \(.*os.py\)")