diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index b42518e8731ca8..3be0d4dcc72543 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -480,7 +480,7 @@ Miscellaneous options objects and pyc files are desired as well as supressing the extra visual location indicators when the interpreter displays tracebacks. See also :envvar:`PYTHONNODEBUGRANGES`. - * ``-X frozen_modules`` determines whether or not frozen modules are + * ``-X frozen_stdlib`` determines whether or not frozen stdlib modules are ignored by the import machinery. A value of "on" means they get imported and "off" means they are ignored. The default is "on" for non-debug builds (the normal case) and "off" for debug builds. @@ -525,7 +525,7 @@ Miscellaneous options The ``-X no_debug_ranges`` option. .. versionadded:: 3.11 - The ``-X frozen_modules`` option. + The ``-X frozen_stdlib`` option. Options you shouldn't use diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 05641001bcd749..c8c0eae10aa911 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -172,7 +172,7 @@ typedef struct PyConfig { int legacy_windows_stdio; #endif wchar_t *check_hash_pycs_mode; - int use_frozen_modules; + int use_frozen_stdlib; /* --- Path configuration inputs ------------ */ int pathconfig_warnings; diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 6439b7369fb593..fcb7760b5f66fe 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -15,6 +15,7 @@ struct _module_alias { const char *orig; /* ASCII encoded string */ }; +extern const struct _frozen * _PyImport_FrozenStdlib; extern const struct _module_alias * _PyImport_FrozenAliases; #ifdef __cplusplus diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 0e6edf4ec26702..6010f4bdf53cce 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -246,9 +246,9 @@ struct _is { PyObject *builtins; // importlib module PyObject *importlib; - // override for config->use_frozen_modules (for tests) + // override for config->use_frozen_stdlib (for tests) // (-1: "off", 1: "on", 0: no override) - int override_frozen_modules; + int override_frozen_stdlib; /* Used in Modules/_threadmodule.c. */ long num_threads; diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 9bce29895249ae..5bc8019df14aea 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -97,11 +97,11 @@ def frozen_modules(enabled=True): This only applies to modules that haven't been imported yet. Also, some essential modules will always be imported frozen. """ - _imp._override_frozen_modules_for_tests(1 if enabled else -1) + _imp._override_frozen_stdlib_for_tests(1 if enabled else -1) try: yield finally: - _imp._override_frozen_modules_for_tests(0) + _imp._override_frozen_stdlib_for_tests(0) def import_fresh_module(name, fresh=(), blocked=(), *, diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index aa2b3d7efbf996..e7b38b56db1197 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -435,7 +435,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'pathconfig_warnings': 1, '_init_main': 1, '_isolated_interpreter': 0, - 'use_frozen_modules': 0, + 'use_frozen_stdlib': 0, } if MS_WINDOWS: CONFIG_COMPAT.update({ diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index ba07ef95b48016..0167f3f25e67b9 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -260,8 +260,8 @@ Compiler now removes trailing unused constants from co_consts. .. nonce: ZPI_3L .. section: Core and Builtins -Add a new command line option, "-X frozen_modules=[on|off]" to opt out of -(or into) using optional frozen modules. This defaults to "on" (or "off" if +Add a new command line option, "-X frozen_stdlib=[on|off]" to opt out of +(or into) using frozen stdlib modules. This defaults to "on" (or "off" if it's a debug build). .. @@ -4414,7 +4414,7 @@ before building. Freeze stdlib modules that are imported during startup. This provides significant performance improvements to startup. If necessary, use the -previously added "-X frozen_modules=off" commandline option to force +previously added "-X frozen_stdlib=off" commandline option to force importing the source modules. .. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-06-14-27-51.bpo-45396.1xwQeD.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-06-14-27-51.bpo-45396.1xwQeD.rst new file mode 100644 index 00000000000000..e158424810e52e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-10-06-14-27-51.bpo-45396.1xwQeD.rst @@ -0,0 +1,2 @@ +Always import custom frozen modules. This means "-X frozen_stdlib" is +ignored for now (see bpo-45395). diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index dd90d92e512c09..32c3dae5aeeeed 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -30,6 +30,7 @@ static const struct _module_alias aliases[] = { }; const struct _frozen *PyImport_FrozenModules; +const struct _frozen *_PyImport_FrozenStdlib; const struct _module_alias *_PyImport_FrozenAliases; static const char header[] = @@ -188,6 +189,7 @@ main(int argc, char *argv[]) const char *name, *inpath, *outpath; PyImport_FrozenModules = _PyImport_FrozenModules; + _PyImport_FrozenStdlib = _PyImport_FrozenModules; _PyImport_FrozenAliases = aliases; if (argc != 4) { diff --git a/Python/clinic/import.c.h b/Python/clinic/import.c.h index dfb59de3b5ce80..9d1d3189263b04 100644 --- a/Python/clinic/import.c.h +++ b/Python/clinic/import.c.h @@ -365,23 +365,23 @@ _imp__frozen_module_names(PyObject *module, PyObject *Py_UNUSED(ignored)) return _imp__frozen_module_names_impl(module); } -PyDoc_STRVAR(_imp__override_frozen_modules_for_tests__doc__, -"_override_frozen_modules_for_tests($module, override, /)\n" +PyDoc_STRVAR(_imp__override_frozen_stdlib_for_tests__doc__, +"_override_frozen_stdlib_for_tests($module, override, /)\n" "--\n" "\n" -"(internal-only) Override PyConfig.use_frozen_modules.\n" +"(internal-only) Override PyConfig.use_frozen_stdlib.\n" "\n" "(-1: \"off\", 1: \"on\", 0: no override)\n" "See frozen_modules() in Lib/test/support/import_helper.py."); -#define _IMP__OVERRIDE_FROZEN_MODULES_FOR_TESTS_METHODDEF \ - {"_override_frozen_modules_for_tests", (PyCFunction)_imp__override_frozen_modules_for_tests, METH_O, _imp__override_frozen_modules_for_tests__doc__}, +#define _IMP__OVERRIDE_FROZEN_STDLIB_FOR_TESTS_METHODDEF \ + {"_override_frozen_stdlib_for_tests", (PyCFunction)_imp__override_frozen_stdlib_for_tests, METH_O, _imp__override_frozen_stdlib_for_tests__doc__}, static PyObject * -_imp__override_frozen_modules_for_tests_impl(PyObject *module, int override); +_imp__override_frozen_stdlib_for_tests_impl(PyObject *module, int override); static PyObject * -_imp__override_frozen_modules_for_tests(PyObject *module, PyObject *arg) +_imp__override_frozen_stdlib_for_tests(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; int override; @@ -390,7 +390,7 @@ _imp__override_frozen_modules_for_tests(PyObject *module, PyObject *arg) if (override == -1 && PyErr_Occurred()) { goto exit; } - return_value = _imp__override_frozen_modules_for_tests_impl(module, override); + return_value = _imp__override_frozen_stdlib_for_tests_impl(module, override); exit: return return_value; @@ -548,4 +548,4 @@ _imp_source_hash(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb #ifndef _IMP_EXEC_DYNAMIC_METHODDEF #define _IMP_EXEC_DYNAMIC_METHODDEF #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */ -/*[clinic end generated code: output=8c8dd08158f9ac7c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=494288a40db2310a input=a9049054013a1b77]*/ diff --git a/Python/frozen.c b/Python/frozen.c index 499b3b99570573..006ce41546902f 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -105,6 +105,7 @@ static const struct _frozen _PyImport_FrozenModules[] = { {"__hello_only__", _Py_M__frozen_only, (int)sizeof(_Py_M__frozen_only)}, {0, 0, 0} /* modules sentinel */ }; +const struct _frozen *_PyImport_FrozenStdlib = _PyImport_FrozenModules; static const struct _module_alias aliases[] = { {"_frozen_importlib", "importlib._bootstrap"}, diff --git a/Python/import.c b/Python/import.c index a6170a39c7fdbd..514a7c8e875b34 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1093,10 +1093,10 @@ is_essential_frozen_module(const char *name) } static bool -use_frozen(void) +use_frozen_stdlib(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); - int override = interp->override_frozen_modules; + int override = interp->override_frozen_stdlib; if (override > 0) { return true; } @@ -1104,7 +1104,7 @@ use_frozen(void) return false; } else { - return interp->config.use_frozen_modules; + return interp->config.use_frozen_stdlib; } } @@ -1115,7 +1115,9 @@ list_frozen_module_names() if (names == NULL) { return NULL; } - bool enabled = use_frozen(); + bool enabled = PyImport_FrozenModules == _PyImport_FrozenStdlib + ? use_frozen_stdlib() + : true; for (const struct _frozen *p = PyImport_FrozenModules; ; p++) { if (p->name == NULL) { break; @@ -1142,7 +1144,7 @@ typedef enum { FROZEN_OKAY, FROZEN_BAD_NAME, // The given module name wasn't valid. FROZEN_NOT_FOUND, // It wasn't in PyImport_FrozenModules. - FROZEN_DISABLED, // -X frozen_modules=off (and not essential) + FROZEN_DISABLED, // -X frozen_stdlib=off (and not essential) FROZEN_EXCLUDED, // The PyImport_FrozenModules entry has NULL "code". FROZEN_INVALID, // The PyImport_FrozenModules entry is bogus. } frozen_status; @@ -1208,7 +1210,10 @@ find_frozen(PyObject *nameobj, struct frozen_info *info) return FROZEN_BAD_NAME; } - if (!use_frozen() && !is_essential_frozen_module(name)) { + bool enabled = PyImport_FrozenModules == _PyImport_FrozenStdlib + ? use_frozen_stdlib() + : true; + if (!enabled && !is_essential_frozen_module(name)) { return FROZEN_DISABLED; } @@ -2213,23 +2218,23 @@ _imp__frozen_module_names_impl(PyObject *module) } /*[clinic input] -_imp._override_frozen_modules_for_tests +_imp._override_frozen_stdlib_for_tests override: int / -(internal-only) Override PyConfig.use_frozen_modules. +(internal-only) Override PyConfig.use_frozen_stdlib. (-1: "off", 1: "on", 0: no override) See frozen_modules() in Lib/test/support/import_helper.py. [clinic start generated code]*/ static PyObject * -_imp__override_frozen_modules_for_tests_impl(PyObject *module, int override) -/*[clinic end generated code: output=36d5cb1594160811 input=8f1f95a3ef21aec3]*/ +_imp__override_frozen_stdlib_for_tests_impl(PyObject *module, int override) +/*[clinic end generated code: output=3f769bf38d4fd4cf input=8c1be52885465f03]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); - interp->override_frozen_modules = override; + interp->override_frozen_stdlib = override; Py_RETURN_NONE; } @@ -2395,7 +2400,7 @@ static PyMethodDef imp_methods[] = { _IMP_IS_BUILTIN_METHODDEF _IMP_IS_FROZEN_METHODDEF _IMP__FROZEN_MODULE_NAMES_METHODDEF - _IMP__OVERRIDE_FROZEN_MODULES_FOR_TESTS_METHODDEF + _IMP__OVERRIDE_FROZEN_STDLIB_FOR_TESTS_METHODDEF _IMP_CREATE_DYNAMIC_METHODDEF _IMP_EXEC_DYNAMIC_METHODDEF _IMP_EXEC_BUILTIN_METHODDEF diff --git a/Python/initconfig.c b/Python/initconfig.c index 9fa202a7da5c02..4b9b46ac7fd37c 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -100,7 +100,7 @@ static const char usage_3[] = "\ instruction in code objects. This is useful when smaller code objects and pyc \n\ files are desired as well as supressing the extra visual location indicators \n\ when the interpreter displays tracebacks.\n\ - -X frozen_modules=[on|off]: whether or not frozen modules should be used.\n\ + -X frozen_stdlib=[on|off]: whether or not frozen stdlib modules should be used.\n\ The default is \"on\" (or \"off\" if you are running a local build).\n\ \n\ --check-hash-based-pycs always|default|never:\n\ @@ -948,7 +948,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2) COPY_ATTR(pathconfig_warnings); COPY_ATTR(_init_main); COPY_ATTR(_isolated_interpreter); - COPY_ATTR(use_frozen_modules); + COPY_ATTR(use_frozen_stdlib); COPY_WSTRLIST(orig_argv); #undef COPY_ATTR @@ -1053,7 +1053,7 @@ _PyConfig_AsDict(const PyConfig *config) SET_ITEM_INT(_init_main); SET_ITEM_INT(_isolated_interpreter); SET_ITEM_WSTRLIST(orig_argv); - SET_ITEM_INT(use_frozen_modules); + SET_ITEM_INT(use_frozen_stdlib); return dict; @@ -1337,7 +1337,7 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict) GET_UINT(_install_importlib); GET_UINT(_init_main); GET_UINT(_isolated_interpreter); - GET_UINT(use_frozen_modules); + GET_UINT(use_frozen_stdlib); #undef CHECK_VALUE #undef GET_UINT @@ -2090,25 +2090,25 @@ config_init_import(PyConfig *config, int compute_path_config) return status; } - /* -X frozen_modules=[on|off] */ - const wchar_t *value = config_get_xoption_value(config, L"frozen_modules"); + /* -X frozen_stdlib=[on|off] */ + const wchar_t *value = config_get_xoption_value(config, L"frozen_stdlib"); if (value == NULL) { // For now we always default to "off". // In the near future we will be factoring in PGO and in-development. - config->use_frozen_modules = 0; + config->use_frozen_stdlib = 0; } else if (wcscmp(value, L"on") == 0) { - config->use_frozen_modules = 1; + config->use_frozen_stdlib = 1; } else if (wcscmp(value, L"off") == 0) { - config->use_frozen_modules = 0; + config->use_frozen_stdlib = 0; } else if (wcslen(value) == 0) { - // "-X frozen_modules" and "-X frozen_modules=" both imply "on". - config->use_frozen_modules = 1; + // "-X frozen_stdlib" and "-X frozen_stdlib=" both imply "on". + config->use_frozen_stdlib = 1; } else { - return PyStatus_Error("bad value for option -X frozen_modules " + return PyStatus_Error("bad value for option -X frozen_stdlib " "(expected \"on\" or \"off\")"); }