Skip to content

Commit b9cdd0f

Browse files
bpo-45020: Default to using frozen modules unless running from source tree. (gh-28940)
The default was "off". Switching it to "on" means users get the benefit of frozen stdlib modules without having to do anything. There's a special-case for running-in-source-tree, so contributors don't get surprised when their stdlib changes don't get used. https://bugs.python.org/issue45020
1 parent fe0d9e2 commit b9cdd0f

File tree

7 files changed

+91
-25
lines changed

7 files changed

+91
-25
lines changed

Doc/using/cmdline.rst

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -483,7 +483,8 @@ Miscellaneous options
483483
* ``-X frozen_modules`` determines whether or not frozen modules are
484484
ignored by the import machinery. A value of "on" means they get
485485
imported and "off" means they are ignored. The default is "on"
486-
for non-debug builds (the normal case) and "off" for debug builds.
486+
if this is an installed Python (the normal case). If it's under
487+
development (running from the source tree) then the default is "off".
487488
Note that the "importlib_bootstrap" and "importlib_bootstrap_external"
488489
frozen modules are always used, even if this flag is set to "off".
489490

Include/internal/pycore_fileutils.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ extern wchar_t * _Py_join_relfile(const wchar_t *dirname,
7979
extern int _Py_add_relfile(wchar_t *dirname,
8080
const wchar_t *relfile,
8181
size_t bufsize);
82+
extern size_t _Py_find_basename(const wchar_t *filename);
8283

8384
// Macros to protect CRT calls against instant termination when passed an
8485
// invalid parameter (bpo-23524). IPH stands for Invalid Parameter Handler.

Lib/site.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -418,8 +418,10 @@ def setcopyright():
418418
files, dirs = [], []
419419
# Not all modules are required to have a __file__ attribute. See
420420
# PEP 420 for more details.
421-
if hasattr(os, '__file__'):
421+
here = getattr(sys, '_stdlib_dir', None)
422+
if not here and hasattr(os, '__file__'):
422423
here = os.path.dirname(os.__file__)
424+
if here:
423425
files.extend(["LICENSE.txt", "LICENSE"])
424426
dirs.extend([os.path.join(here, os.pardir), here, os.curdir])
425427
builtins.license = _sitebuiltins._Printer(

Lib/test/test_embed.py

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,12 +53,13 @@ def remove_python_envvars():
5353
class EmbeddingTestsMixin:
5454
def setUp(self):
5555
exename = "_testembed"
56+
builddir = os.path.dirname(sys.executable)
5657
if MS_WINDOWS:
5758
ext = ("_d" if debug_build(sys.executable) else "") + ".exe"
5859
exename += ext
59-
exepath = os.path.dirname(sys.executable)
60+
exepath = builddir
6061
else:
61-
exepath = os.path.join(support.REPO_ROOT, "Programs")
62+
exepath = os.path.join(builddir, 'Programs')
6263
self.test_exe = exe = os.path.join(exepath, exename)
6364
if not os.path.exists(exe):
6465
self.skipTest("%r doesn't exist" % exe)
@@ -434,7 +435,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
434435
'pathconfig_warnings': 1,
435436
'_init_main': 1,
436437
'_isolated_interpreter': 0,
437-
'use_frozen_modules': 0,
438+
'use_frozen_modules': 1,
438439
}
439440
if MS_WINDOWS:
440441
CONFIG_COMPAT.update({
@@ -1146,6 +1147,7 @@ def test_init_setpath(self):
11461147
# The current getpath.c doesn't determine the stdlib dir
11471148
# in this case.
11481149
'stdlib_dir': '',
1150+
'use_frozen_modules': -1,
11491151
}
11501152
self.default_program_name(config)
11511153
env = {'TESTPATH': os.path.pathsep.join(paths)}
@@ -1169,6 +1171,7 @@ def test_init_setpath_config(self):
11691171
# The current getpath.c doesn't determine the stdlib dir
11701172
# in this case.
11711173
'stdlib_dir': '',
1174+
'use_frozen_modules': -1,
11721175
# overridden by PyConfig
11731176
'program_name': 'conf_program_name',
11741177
'base_executable': 'conf_executable',
@@ -1265,6 +1268,8 @@ def test_init_setpythonhome(self):
12651268
'stdlib_dir': stdlib,
12661269
}
12671270
self.default_program_name(config)
1271+
if not config['executable']:
1272+
config['use_frozen_modules'] = -1
12681273
env = {'TESTHOME': home, 'PYTHONPATH': paths_str}
12691274
self.check_all_configs("test_init_setpythonhome", config,
12701275
api=API_COMPAT, env=env)
@@ -1303,6 +1308,7 @@ def test_init_pybuilddir(self):
13031308
# The current getpath.c doesn't determine the stdlib dir
13041309
# in this case.
13051310
'stdlib_dir': None,
1311+
'use_frozen_modules': -1,
13061312
}
13071313
env = self.copy_paths_by_env(config)
13081314
self.check_all_configs("test_init_compat_config", config,
@@ -1361,6 +1367,7 @@ def test_init_pyvenv_cfg(self):
13611367
config['base_prefix'] = pyvenv_home
13621368
config['prefix'] = pyvenv_home
13631369
config['stdlib_dir'] = os.path.join(pyvenv_home, 'lib')
1370+
config['use_frozen_modules'] = 1
13641371

13651372
ver = sys.version_info
13661373
dll = f'python{ver.major}'
@@ -1373,6 +1380,7 @@ def test_init_pyvenv_cfg(self):
13731380
# The current getpath.c doesn't determine the stdlib dir
13741381
# in this case.
13751382
config['stdlib_dir'] = None
1383+
config['use_frozen_modules'] = -1
13761384

13771385
env = self.copy_paths_by_env(config)
13781386
self.check_all_configs("test_init_compat_config", config,

Misc/NEWS.d/3.11.0a1.rst

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -262,7 +262,7 @@ Compiler now removes trailing unused constants from co_consts.
262262
263263
Add a new command line option, "-X frozen_modules=[on|off]" to opt out of
264264
(or into) using optional frozen modules. This defaults to "on" (or "off" if
265-
it's a debug build).
265+
it's running out of the source tree).
266266

267267
..
268268

Python/fileutils.c

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2169,6 +2169,18 @@ _Py_add_relfile(wchar_t *dirname, const wchar_t *relfile, size_t bufsize)
21692169
}
21702170

21712171

2172+
size_t
2173+
_Py_find_basename(const wchar_t *filename)
2174+
{
2175+
for (size_t i = wcslen(filename); i > 0; --i) {
2176+
if (filename[i] == SEP) {
2177+
return i + 1;
2178+
}
2179+
}
2180+
return 0;
2181+
}
2182+
2183+
21722184
/* Get the current directory. buflen is the buffer size in wide characters
21732185
including the null character. Decode the path from the locale encoding.
21742186

Python/initconfig.c

Lines changed: 61 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -739,6 +739,7 @@ _PyConfig_InitCompatConfig(PyConfig *config)
739739
#ifdef MS_WINDOWS
740740
config->legacy_windows_stdio = -1;
741741
#endif
742+
config->use_frozen_modules = -1;
742743
}
743744

744745

@@ -2090,6 +2091,44 @@ config_init_fs_encoding(PyConfig *config, const PyPreConfig *preconfig)
20902091
}
20912092

20922093

2094+
/* Determine if the current build is a "development" build (e.g. running
2095+
out of the source tree) or not.
2096+
2097+
A return value of -1 indicates that we do not know.
2098+
*/
2099+
static int
2100+
is_dev_env(PyConfig *config)
2101+
{
2102+
// This should only ever get called early in runtime initialization,
2103+
// before the global path config is written. Otherwise we would
2104+
// use Py_GetProgramFullPath() and _Py_GetStdlibDir().
2105+
assert(config != NULL);
2106+
2107+
const wchar_t *executable = config->executable;
2108+
const wchar_t *stdlib = config->stdlib_dir;
2109+
if (executable == NULL || *executable == L'\0' ||
2110+
stdlib == NULL || *stdlib == L'\0') {
2111+
// _PyPathConfig_Calculate() hasn't run yet.
2112+
return -1;
2113+
}
2114+
size_t len = _Py_find_basename(executable);
2115+
if (wcscmp(executable + len, L"python") != 0 &&
2116+
wcscmp(executable + len, L"python.exe") != 0) {
2117+
return 0;
2118+
}
2119+
/* If dirname() is the same for both then it is a dev build. */
2120+
if (len != _Py_find_basename(stdlib)) {
2121+
return 0;
2122+
}
2123+
// We do not bother normalizing the two filenames first since
2124+
// for config_init_import() is does the right thing as-is.
2125+
if (wcsncmp(stdlib, executable, len) != 0) {
2126+
return 0;
2127+
}
2128+
return 1;
2129+
}
2130+
2131+
20932132
static PyStatus
20942133
config_init_import(PyConfig *config, int compute_path_config)
20952134
{
@@ -2101,25 +2140,28 @@ config_init_import(PyConfig *config, int compute_path_config)
21012140
}
21022141

21032142
/* -X frozen_modules=[on|off] */
2104-
const wchar_t *value = config_get_xoption_value(config, L"frozen_modules");
2105-
if (value == NULL) {
2106-
// For now we always default to "off".
2107-
// In the near future we will be factoring in PGO and in-development.
2108-
config->use_frozen_modules = 0;
2109-
}
2110-
else if (wcscmp(value, L"on") == 0) {
2111-
config->use_frozen_modules = 1;
2112-
}
2113-
else if (wcscmp(value, L"off") == 0) {
2114-
config->use_frozen_modules = 0;
2115-
}
2116-
else if (wcslen(value) == 0) {
2117-
// "-X frozen_modules" and "-X frozen_modules=" both imply "on".
2118-
config->use_frozen_modules = 1;
2119-
}
2120-
else {
2121-
return PyStatus_Error("bad value for option -X frozen_modules "
2122-
"(expected \"on\" or \"off\")");
2143+
if (config->use_frozen_modules < 0) {
2144+
const wchar_t *value = config_get_xoption_value(config, L"frozen_modules");
2145+
if (value == NULL) {
2146+
int isdev = is_dev_env(config);
2147+
if (isdev >= 0) {
2148+
config->use_frozen_modules = !isdev;
2149+
}
2150+
}
2151+
else if (wcscmp(value, L"on") == 0) {
2152+
config->use_frozen_modules = 1;
2153+
}
2154+
else if (wcscmp(value, L"off") == 0) {
2155+
config->use_frozen_modules = 0;
2156+
}
2157+
else if (wcslen(value) == 0) {
2158+
// "-X frozen_modules" and "-X frozen_modules=" both imply "on".
2159+
config->use_frozen_modules = 1;
2160+
}
2161+
else {
2162+
return PyStatus_Error("bad value for option -X frozen_modules "
2163+
"(expected \"on\" or \"off\")");
2164+
}
21232165
}
21242166

21252167
return _PyStatus_OK();

0 commit comments

Comments
 (0)