bpo-45396: Always import custom frozen modules.

Doc/using/cmdline.rst

@@ -480,7 +480,7 @@ Miscellaneous options
      objects and pyc files are desired as well as supressing the extra visual
      location indicators when the interpreter displays tracebacks. See also
      :envvar:`PYTHONNODEBUGRANGES`.
-   * ``-X frozen_modules`` determines whether or not frozen modules are
+   * ``-X frozen_stdlib`` determines whether or not frozen stdlib modules are
      ignored by the import machinery.  A value of "on" means they get
      imported and "off" means they are ignored.  The default is "on"
      for non-debug builds (the normal case) and "off" for debug builds.
@@ -525,7 +525,7 @@ Miscellaneous options
       The ``-X no_debug_ranges`` option.
 
    .. versionadded:: 3.11
-      The ``-X frozen_modules`` option.
+      The ``-X frozen_stdlib`` option.
 
 
 Options you shouldn't use

Include/cpython/initconfig.h

@@ -172,7 +172,7 @@ typedef struct PyConfig {
     int legacy_windows_stdio;
 #endif
     wchar_t *check_hash_pycs_mode;
-    int use_frozen_modules;
+    int use_frozen_stdlib;
 
     /* --- Path configuration inputs ------------ */
     int pathconfig_warnings;

Include/internal/pycore_import.h

@@ -15,6 +15,7 @@ struct _module_alias {
     const char *orig;                 /* ASCII encoded string */
 };
 
+extern const struct _frozen * _PyImport_FrozenStdlib;
 extern const struct _module_alias * _PyImport_FrozenAliases;
 
 #ifdef __cplusplus

Include/internal/pycore_interp.h

@@ -246,9 +246,9 @@ struct _is {
     PyObject *builtins;
     // importlib module
     PyObject *importlib;
-    // override for config->use_frozen_modules (for tests)
+    // override for config->use_frozen_stdlib (for tests)
     // (-1: "off", 1: "on", 0: no override)
-    int override_frozen_modules;
+    int override_frozen_stdlib;
 
     /* Used in Modules/_threadmodule.c. */
     long num_threads;

Lib/test/support/import_helper.py

@@ -97,11 +97,11 @@ def frozen_modules(enabled=True):
     This only applies to modules that haven't been imported yet.
     Also, some essential modules will always be imported frozen.
     """
-    _imp._override_frozen_modules_for_tests(1 if enabled else -1)
+    _imp._override_frozen_stdlib_for_tests(1 if enabled else -1)
     try:
         yield
     finally:
-        _imp._override_frozen_modules_for_tests(0)
+        _imp._override_frozen_stdlib_for_tests(0)
 
 
 def import_fresh_module(name, fresh=(), blocked=(), *,

Lib/test/test_embed.py

@@ -435,7 +435,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'pathconfig_warnings': 1,
         '_init_main': 1,
         '_isolated_interpreter': 0,
-        'use_frozen_modules': 0,
+        'use_frozen_stdlib': 0,
     }
     if MS_WINDOWS:
         CONFIG_COMPAT.update({

Misc/NEWS.d/3.11.0a1.rst

@@ -260,8 +260,8 @@ Compiler now removes trailing unused constants from co_consts.
 .. nonce: ZPI_3L
 .. section: Core and Builtins
 
-Add a new command line option, "-X frozen_modules=[on|off]" to opt out of
-(or into) using optional frozen modules.  This defaults to "on" (or "off" if
+Add a new command line option, "-X frozen_stdlib=[on|off]" to opt out of
+(or into) using frozen stdlib modules.  This defaults to "on" (or "off" if
 it's a debug build).
 
 ..
@@ -4414,7 +4414,7 @@ before building.
 
 Freeze stdlib modules that are imported during startup.  This provides
 significant performance improvements to startup.  If necessary, use the
-previously added "-X frozen_modules=off" commandline option to force
+previously added "-X frozen_stdlib=off" commandline option to force
 importing the source modules.
 
 ..

Misc/NEWS.d/next/Core and Builtins/2021-10-06-14-27-51.bpo-45396.1xwQeD.rst

@@ -0,0 +1,2 @@
+Always import custom frozen modules.  This means "-X frozen_stdlib" is
+ignored for now (see bpo-45395).

Programs/_freeze_module.c

@@ -30,6 +30,7 @@ static const struct _module_alias aliases[] = {
 };
 
 const struct _frozen *PyImport_FrozenModules;
+const struct _frozen *_PyImport_FrozenStdlib;
 const struct _module_alias *_PyImport_FrozenAliases;
 
 static const char header[] =
@@ -188,6 +189,7 @@ main(int argc, char *argv[])
     const char *name, *inpath, *outpath;
 
     PyImport_FrozenModules = _PyImport_FrozenModules;
+    _PyImport_FrozenStdlib = _PyImport_FrozenModules;
     _PyImport_FrozenAliases = aliases;
 
     if (argc != 4) {

Python/frozen.c

@@ -105,6 +105,7 @@ static const struct _frozen _PyImport_FrozenModules[] = {
     {"__hello_only__", _Py_M__frozen_only, (int)sizeof(_Py_M__frozen_only)},
     {0, 0, 0} /* modules sentinel */
 };
+const struct _frozen *_PyImport_FrozenStdlib = _PyImport_FrozenModules;
 
 static const struct _module_alias aliases[] = {
     {"_frozen_importlib", "importlib._bootstrap"},

Python/import.c

@@ -1093,18 +1093,18 @@ is_essential_frozen_module(const char *name)
 }
 
 static bool
-use_frozen(void)
+use_frozen_stdlib(void)
 {
     PyInterpreterState *interp = _PyInterpreterState_GET();
-    int override = interp->override_frozen_modules;
+    int override = interp->override_frozen_stdlib;
     if (override > 0) {
         return true;
     }
     else if (override < 0) {
         return false;
     }
     else {
-        return interp->config.use_frozen_modules;
+        return interp->config.use_frozen_stdlib;
     }
 }
 
@@ -1115,7 +1115,9 @@ list_frozen_module_names()
     if (names == NULL) {
         return NULL;
     }
-    bool enabled = use_frozen();
+    bool enabled = PyImport_FrozenModules == _PyImport_FrozenStdlib
+                    ? use_frozen_stdlib()
+                    : true;
     for (const struct _frozen *p = PyImport_FrozenModules; ; p++) {
         if (p->name == NULL) {
             break;
@@ -1142,7 +1144,7 @@ typedef enum {
     FROZEN_OKAY,
     FROZEN_BAD_NAME,    // The given module name wasn't valid.
     FROZEN_NOT_FOUND,   // It wasn't in PyImport_FrozenModules.
-    FROZEN_DISABLED,    // -X frozen_modules=off (and not essential)
+    FROZEN_DISABLED,    // -X frozen_stdlib=off (and not essential)
     FROZEN_EXCLUDED,    // The PyImport_FrozenModules entry has NULL "code".
     FROZEN_INVALID,     // The PyImport_FrozenModules entry is bogus.
 } frozen_status;
@@ -1208,7 +1210,10 @@ find_frozen(PyObject *nameobj, struct frozen_info *info)
         return FROZEN_BAD_NAME;
     }
 
-    if (!use_frozen() && !is_essential_frozen_module(name)) {
+    bool enabled = PyImport_FrozenModules == _PyImport_FrozenStdlib
+                    ? use_frozen_stdlib()
+                    : true;
+    if (!enabled && !is_essential_frozen_module(name)) {
         return FROZEN_DISABLED;
     }
 
@@ -2213,23 +2218,23 @@ _imp__frozen_module_names_impl(PyObject *module)
 }
 
 /*[clinic input]
-_imp._override_frozen_modules_for_tests
+_imp._override_frozen_stdlib_for_tests
 
     override: int
     /
 
-(internal-only) Override PyConfig.use_frozen_modules.
+(internal-only) Override PyConfig.use_frozen_stdlib.
 
 (-1: "off", 1: "on", 0: no override)
 See frozen_modules() in Lib/test/support/import_helper.py.
 [clinic start generated code]*/
 
 static PyObject *
-_imp__override_frozen_modules_for_tests_impl(PyObject *module, int override)
-/*[clinic end generated code: output=36d5cb1594160811 input=8f1f95a3ef21aec3]*/
+_imp__override_frozen_stdlib_for_tests_impl(PyObject *module, int override)
+/*[clinic end generated code: output=3f769bf38d4fd4cf input=8c1be52885465f03]*/
 {
     PyInterpreterState *interp = _PyInterpreterState_GET();
-    interp->override_frozen_modules = override;
+    interp->override_frozen_stdlib = override;
     Py_RETURN_NONE;
 }
 
@@ -2395,7 +2400,7 @@ static PyMethodDef imp_methods[] = {
     _IMP_IS_BUILTIN_METHODDEF
     _IMP_IS_FROZEN_METHODDEF
     _IMP__FROZEN_MODULE_NAMES_METHODDEF
-    _IMP__OVERRIDE_FROZEN_MODULES_FOR_TESTS_METHODDEF
+    _IMP__OVERRIDE_FROZEN_STDLIB_FOR_TESTS_METHODDEF
     _IMP_CREATE_DYNAMIC_METHODDEF
     _IMP_EXEC_DYNAMIC_METHODDEF
     _IMP_EXEC_BUILTIN_METHODDEF

Python/initconfig.c

@@ -100,7 +100,7 @@ static const char usage_3[] = "\
             instruction in code objects. This is useful when smaller code objects and pyc \n\
             files are desired as well as supressing the extra visual location indicators \n\
             when the interpreter displays tracebacks.\n\
-         -X frozen_modules=[on|off]: whether or not frozen modules should be used.\n\
+         -X frozen_stdlib=[on|off]: whether or not frozen stdlib modules should be used.\n\
             The default is \"on\" (or \"off\" if you are running a local build).\n\
 \n\
 --check-hash-based-pycs always|default|never:\n\
@@ -948,7 +948,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2)
     COPY_ATTR(pathconfig_warnings);
     COPY_ATTR(_init_main);
     COPY_ATTR(_isolated_interpreter);
-    COPY_ATTR(use_frozen_modules);
+    COPY_ATTR(use_frozen_stdlib);
     COPY_WSTRLIST(orig_argv);
 
 #undef COPY_ATTR
@@ -1053,7 +1053,7 @@ _PyConfig_AsDict(const PyConfig *config)
     SET_ITEM_INT(_init_main);
     SET_ITEM_INT(_isolated_interpreter);
     SET_ITEM_WSTRLIST(orig_argv);
-    SET_ITEM_INT(use_frozen_modules);
+    SET_ITEM_INT(use_frozen_stdlib);
 
     return dict;
 
@@ -1337,7 +1337,7 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict)
     GET_UINT(_install_importlib);
     GET_UINT(_init_main);
     GET_UINT(_isolated_interpreter);
-    GET_UINT(use_frozen_modules);
+    GET_UINT(use_frozen_stdlib);
 
 #undef CHECK_VALUE
 #undef GET_UINT
@@ -2090,25 +2090,25 @@ config_init_import(PyConfig *config, int compute_path_config)
         return status;
     }
 
-    /* -X frozen_modules=[on|off] */
-    const wchar_t *value = config_get_xoption_value(config, L"frozen_modules");
+    /* -X frozen_stdlib=[on|off] */
+    const wchar_t *value = config_get_xoption_value(config, L"frozen_stdlib");
     if (value == NULL) {
         // For now we always default to "off".
         // In the near future we will be factoring in PGO and in-development.
-        config->use_frozen_modules = 0;
+        config->use_frozen_stdlib = 0;
     }
     else if (wcscmp(value, L"on") == 0) {
-        config->use_frozen_modules = 1;
+        config->use_frozen_stdlib = 1;
     }
     else if (wcscmp(value, L"off") == 0) {
-        config->use_frozen_modules = 0;
+        config->use_frozen_stdlib = 0;
     }
     else if (wcslen(value) == 0) {
-        // "-X frozen_modules" and "-X frozen_modules=" both imply "on".
-        config->use_frozen_modules = 1;
+        // "-X frozen_stdlib" and "-X frozen_stdlib=" both imply "on".
+        config->use_frozen_stdlib = 1;
     }
     else {
-        return PyStatus_Error("bad value for option -X frozen_modules "
+        return PyStatus_Error("bad value for option -X frozen_stdlib "
                               "(expected \"on\" or \"off\")");
     }