From cc59209d29ee079195cf6d317d1f3e78db386a9c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 31 Aug 2021 17:45:02 -0600 Subject: [PATCH 01/11] Add a NEWS entry. --- Misc/NEWS.d/3.11.0a1.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index e3d2acc4999689..a64a3e74ccb410 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -262,7 +262,7 @@ Compiler now removes trailing unused constants from co_consts. Add a new command line option, "-X frozen_modules=[on|off]" to opt out of (or into) using optional frozen modules. This defaults to "on" (or "off" if -it's a debug build). +it's running out of the source tree). .. From 0cefeaffd5965ac0f8b7e25008aebf1eb1053de6 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Aug 2021 18:31:39 -0600 Subject: [PATCH 02/11] Identify whether or not the executable is running installed (adding _Py_IsDevelopmentEnv()). --- Include/internal/pycore_pylifecycle.h | 2 ++ Python/pathconfig.c | 44 +++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 4f12fef8d65466..0dccd4956a0144 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -12,6 +12,7 @@ extern "C" { #include #endif +#include #include "pycore_runtime.h" // _PyRuntimeState #ifndef NSIG @@ -123,6 +124,7 @@ PyAPI_FUNC(PyStatus) _Py_PreInitializeFromConfig( const struct _PyArgv *args); PyAPI_FUNC(wchar_t *) _Py_GetStdlibDir(void); +PyAPI_FUNC(bool) _Py_IsDevelopmentEnv(void); PyAPI_FUNC(int) _Py_HandleSystemExit(int *exitcode_p); diff --git a/Python/pathconfig.c b/Python/pathconfig.c index ad22222e000fc0..d72ad5719ed779 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -7,6 +7,7 @@ #include "pycore_pathconfig.h" #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include +#include #ifdef MS_WINDOWS # include // GetFullPathNameW(), MAX_PATH #endif @@ -35,6 +36,17 @@ copy_wstr(wchar_t **dst, const wchar_t *src) return 0; } +static size_t +find_basename(const wchar_t *filename) +{ + for (size_t i = wcslen(filename); i > 0; --i) { + if (filename[i] == SEP) { + return i + 1; + } + } + return 0; +} + static void pathconfig_clear(_PyPathConfig *config) @@ -632,6 +644,38 @@ Py_GetProgramName(void) return _Py_path_config.program_name; } + +bool +_Py_IsDevelopmentEnv(void) +{ + // XXX Could this be called early enough during init that + // _Py_path_config.program_full_path isn't set yet? + const wchar_t *executable = Py_GetProgramFullPath(); + if (executable == NULL) { + return false; + } + size_t len = find_basename(executable); + if (wcscmp(executable + len, L"python") != 0 && + wcscmp(executable + len, L"python.exe") != 0) { + return false; + } + /* If dirname() is the same for both then it is a local (dev) build. */ + const wchar_t *stdlib = _Py_GetStdlibDir(); + if (stdlib == NULL) { + return false; + } + // XXX This doesn't work on Windows. + if (len != find_basename(stdlib)) { + return false; + } + // XXX Could either have .. in them? + if (wcsncmp(stdlib, executable, len) != 0) { + return false; + } + return true; +} + + /* Compute module search path from argv[0] or the current working directory ("-m module" case) which will be prepended to sys.argv: sys.path[0]. From 03d7d2609f474ab28a6e948bb228b08cc99874bc Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 31 Aug 2021 09:43:56 -0600 Subject: [PATCH 03/11] Default to "off" if in development. --- Doc/using/cmdline.rst | 3 ++- Lib/test/test_embed.py | 9 +++++++-- Python/initconfig.c | 4 +--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 23a645a036aedf..d341ea8bb43c88 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -483,7 +483,8 @@ Miscellaneous options * ``-X frozen_modules`` determines whether or not frozen modules are ignored by the import machinery. A value of "on" means they get imported and "off" means they are ignored. The default is "on" - for non-debug builds (the normal case) and "off" for debug builds. + if this is an installed Python (the normal case). If it's under + development (running from the source tree) then the default is "off". Note that the "importlib_bootstrap" and "importlib_bootstrap_external" frozen modules are always used, even if this flag is set to "off". diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 4cbb4c2c1ce366..5b6ce078dc8f14 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -434,7 +434,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'pathconfig_warnings': 1, '_init_main': 1, '_isolated_interpreter': 0, - 'use_frozen_modules': 0, + 'use_frozen_modules': GET_DEFAULT_CONFIG, } if MS_WINDOWS: CONFIG_COMPAT.update({ @@ -1265,7 +1265,9 @@ def test_init_setpythonhome(self): 'stdlib_dir': stdlib, } self.default_program_name(config) - env = {'TESTHOME': home, 'PYTHONPATH': paths_str} + env = {'TESTHOME': home, + 'PYTHONPATH': paths_str, + '_PYTHONTESTFROZENMODULES': '1'} self.check_all_configs("test_init_setpythonhome", config, api=API_COMPAT, env=env) @@ -1355,6 +1357,9 @@ def test_init_pyvenv_cfg(self): 'base_executable': executable, 'executable': executable, 'module_search_paths': paths, + # The current getpath.c doesn't determine the stdlib dir + # in this case. + 'stdlib_dir': None, } path_config = {} if MS_WINDOWS: diff --git a/Python/initconfig.c b/Python/initconfig.c index b0d54b0472fb09..be80f8cef92621 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -2103,9 +2103,7 @@ config_init_import(PyConfig *config, int compute_path_config) /* -X frozen_modules=[on|off] */ const wchar_t *value = config_get_xoption_value(config, L"frozen_modules"); if (value == NULL) { - // For now we always default to "off". - // In the near future we will be factoring in PGO and in-development. - config->use_frozen_modules = 0; + config->use_frozen_modules = _Py_IsDevelopmentEnv(); } else if (wcscmp(value, L"on") == 0) { config->use_frozen_modules = 1; From 5db164b0ee111dacba00e9a1342590df7758a368 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 13 Oct 2021 08:39:24 -0600 Subject: [PATCH 04/11] Fix test_embed so it runs with out-of-tree builds. --- Lib/test/test_embed.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 5b6ce078dc8f14..ff82009daeef2f 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -53,12 +53,13 @@ def remove_python_envvars(): class EmbeddingTestsMixin: def setUp(self): exename = "_testembed" + builddir = os.path.dirname(sys.executable) if MS_WINDOWS: ext = ("_d" if debug_build(sys.executable) else "") + ".exe" exename += ext - exepath = os.path.dirname(sys.executable) + exepath = builddir else: - exepath = os.path.join(support.REPO_ROOT, "Programs") + exepath = os.path.join(builddir, 'Programs') self.test_exe = exe = os.path.join(exepath, exename) if not os.path.exists(exe): self.skipTest("%r doesn't exist" % exe) From e9cce77d6cecd7df64c6a7d51285e22e1136bcb8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 13 Oct 2021 14:01:15 -0600 Subject: [PATCH 05/11] Use the active config to decide "is dev env". --- Include/internal/pycore_fileutils.h | 1 + Include/internal/pycore_pylifecycle.h | 2 - Lib/test/test_embed.py | 6 ++- Python/fileutils.c | 12 +++++ Python/initconfig.c | 78 +++++++++++++++++++++------ Python/pathconfig.c | 42 --------------- 6 files changed, 79 insertions(+), 62 deletions(-) diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 3464477bce5755..ab436ae9b007ac 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -79,6 +79,7 @@ extern wchar_t * _Py_join_relfile(const wchar_t *dirname, extern int _Py_add_relfile(wchar_t *dirname, const wchar_t *relfile, size_t bufsize); +extern size_t _Py_find_basename(const wchar_t *filename); // Macros to protect CRT calls against instant termination when passed an // invalid parameter (bpo-23524). IPH stands for Invalid Parameter Handler. diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 0dccd4956a0144..4f12fef8d65466 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -12,7 +12,6 @@ extern "C" { #include #endif -#include #include "pycore_runtime.h" // _PyRuntimeState #ifndef NSIG @@ -124,7 +123,6 @@ PyAPI_FUNC(PyStatus) _Py_PreInitializeFromConfig( const struct _PyArgv *args); PyAPI_FUNC(wchar_t *) _Py_GetStdlibDir(void); -PyAPI_FUNC(bool) _Py_IsDevelopmentEnv(void); PyAPI_FUNC(int) _Py_HandleSystemExit(int *exitcode_p); diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index ff82009daeef2f..4163d31fde7f5f 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -435,7 +435,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'pathconfig_warnings': 1, '_init_main': 1, '_isolated_interpreter': 0, - 'use_frozen_modules': GET_DEFAULT_CONFIG, + 'use_frozen_modules': 1, } if MS_WINDOWS: CONFIG_COMPAT.update({ @@ -1147,6 +1147,7 @@ def test_init_setpath(self): # The current getpath.c doesn't determine the stdlib dir # in this case. 'stdlib_dir': '', + 'use_frozen_modules': -1, } self.default_program_name(config) env = {'TESTPATH': os.path.pathsep.join(paths)} @@ -1170,6 +1171,7 @@ def test_init_setpath_config(self): # The current getpath.c doesn't determine the stdlib dir # in this case. 'stdlib_dir': '', + 'use_frozen_modules': -1, # overridden by PyConfig 'program_name': 'conf_program_name', 'base_executable': 'conf_executable', @@ -1306,6 +1308,7 @@ def test_init_pybuilddir(self): # The current getpath.c doesn't determine the stdlib dir # in this case. 'stdlib_dir': None, + 'use_frozen_modules': -1, } env = self.copy_paths_by_env(config) self.check_all_configs("test_init_compat_config", config, @@ -1361,6 +1364,7 @@ def test_init_pyvenv_cfg(self): # The current getpath.c doesn't determine the stdlib dir # in this case. 'stdlib_dir': None, + 'use_frozen_modules': -1, } path_config = {} if MS_WINDOWS: diff --git a/Python/fileutils.c b/Python/fileutils.c index 173d34dd23f18b..3d8f3a4f16326c 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2169,6 +2169,18 @@ _Py_add_relfile(wchar_t *dirname, const wchar_t *relfile, size_t bufsize) } +size_t +_Py_find_basename(const wchar_t *filename) +{ + for (size_t i = wcslen(filename); i > 0; --i) { + if (filename[i] == SEP) { + return i + 1; + } + } + return 0; +} + + /* Get the current directory. buflen is the buffer size in wide characters including the null character. Decode the path from the locale encoding. diff --git a/Python/initconfig.c b/Python/initconfig.c index be80f8cef92621..e753e8b9a8f66c 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -739,6 +739,7 @@ _PyConfig_InitCompatConfig(PyConfig *config) #ifdef MS_WINDOWS config->legacy_windows_stdio = -1; #endif + config->use_frozen_modules = -1; } @@ -2090,6 +2091,44 @@ config_init_fs_encoding(PyConfig *config, const PyPreConfig *preconfig) } +/* Determine if the current build is a "development" build (e.g. running + out of the source tree) or not. + + A return value of -1 indicates that we do not know. + */ +static int +is_dev_env(PyConfig *config) +{ + // This should only ever get called early in runtime initialization, + // before the global path config is written. Otherwise we would + // use Py_GetProgramFullPath() and _Py_GetStdlibDir(). + assert(config != NULL); + + const wchar_t *executable = config->executable; + const wchar_t *stdlib = config->stdlib_dir; + if (executable == NULL || *executable == L'\0' || + stdlib == NULL || *stdlib == L'\0') { + // _PyPathConfig_Calculate() hasn't run yet. + return -1; + } + size_t len = _Py_find_basename(executable); + if (wcscmp(executable + len, L"python") != 0 && + wcscmp(executable + len, L"python.exe") != 0) { + return 0; + } + /* If dirname() is the same for both then it is a dev build. */ + // XXX This doesn't work on Windows. + if (len != _Py_find_basename(stdlib)) { + return 0; + } + // XXX Could either have .. in them? + if (wcsncmp(stdlib, executable, len) != 0) { + return 0; + } + return 1; +} + + static PyStatus config_init_import(PyConfig *config, int compute_path_config) { @@ -2101,23 +2140,28 @@ config_init_import(PyConfig *config, int compute_path_config) } /* -X frozen_modules=[on|off] */ - const wchar_t *value = config_get_xoption_value(config, L"frozen_modules"); - if (value == NULL) { - config->use_frozen_modules = _Py_IsDevelopmentEnv(); - } - else if (wcscmp(value, L"on") == 0) { - config->use_frozen_modules = 1; - } - else if (wcscmp(value, L"off") == 0) { - config->use_frozen_modules = 0; - } - else if (wcslen(value) == 0) { - // "-X frozen_modules" and "-X frozen_modules=" both imply "on". - config->use_frozen_modules = 1; - } - else { - return PyStatus_Error("bad value for option -X frozen_modules " - "(expected \"on\" or \"off\")"); + if (config->use_frozen_modules < 0) { + const wchar_t *value = config_get_xoption_value(config, L"frozen_modules"); + if (value == NULL) { + int isdev = is_dev_env(config); + if (isdev >= 0) { + config->use_frozen_modules = !isdev; + } + } + else if (wcscmp(value, L"on") == 0) { + config->use_frozen_modules = 1; + } + else if (wcscmp(value, L"off") == 0) { + config->use_frozen_modules = 0; + } + else if (wcslen(value) == 0) { + // "-X frozen_modules" and "-X frozen_modules=" both imply "on". + config->use_frozen_modules = 1; + } + else { + return PyStatus_Error("bad value for option -X frozen_modules " + "(expected \"on\" or \"off\")"); + } } return _PyStatus_OK(); diff --git a/Python/pathconfig.c b/Python/pathconfig.c index d72ad5719ed779..7885c5c9e1f705 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -36,17 +36,6 @@ copy_wstr(wchar_t **dst, const wchar_t *src) return 0; } -static size_t -find_basename(const wchar_t *filename) -{ - for (size_t i = wcslen(filename); i > 0; --i) { - if (filename[i] == SEP) { - return i + 1; - } - } - return 0; -} - static void pathconfig_clear(_PyPathConfig *config) @@ -645,37 +634,6 @@ Py_GetProgramName(void) } -bool -_Py_IsDevelopmentEnv(void) -{ - // XXX Could this be called early enough during init that - // _Py_path_config.program_full_path isn't set yet? - const wchar_t *executable = Py_GetProgramFullPath(); - if (executable == NULL) { - return false; - } - size_t len = find_basename(executable); - if (wcscmp(executable + len, L"python") != 0 && - wcscmp(executable + len, L"python.exe") != 0) { - return false; - } - /* If dirname() is the same for both then it is a local (dev) build. */ - const wchar_t *stdlib = _Py_GetStdlibDir(); - if (stdlib == NULL) { - return false; - } - // XXX This doesn't work on Windows. - if (len != find_basename(stdlib)) { - return false; - } - // XXX Could either have .. in them? - if (wcsncmp(stdlib, executable, len) != 0) { - return false; - } - return true; -} - - /* Compute module search path from argv[0] or the current working directory ("-m module" case) which will be prepended to sys.argv: sys.path[0]. From dc832f5f67a1aa24391b24a853a52679714a2aae Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 13 Oct 2021 16:34:16 -0600 Subject: [PATCH 06/11] Drop extraneous changes. --- Python/pathconfig.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 7885c5c9e1f705..ad22222e000fc0 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -7,7 +7,6 @@ #include "pycore_pathconfig.h" #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include -#include #ifdef MS_WINDOWS # include // GetFullPathNameW(), MAX_PATH #endif @@ -633,7 +632,6 @@ Py_GetProgramName(void) return _Py_path_config.program_name; } - /* Compute module search path from argv[0] or the current working directory ("-m module" case) which will be prepended to sys.argv: sys.path[0]. From e26cc55b71177b8642bc047bab9bbd3c128abac1 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 13 Oct 2021 16:58:48 -0600 Subject: [PATCH 07/11] Fix test_embed on Windows. --- Lib/test/test_embed.py | 6 ++---- Python/initconfig.c | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 4163d31fde7f5f..f760c1825522bd 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1361,16 +1361,13 @@ def test_init_pyvenv_cfg(self): 'base_executable': executable, 'executable': executable, 'module_search_paths': paths, - # The current getpath.c doesn't determine the stdlib dir - # in this case. - 'stdlib_dir': None, - 'use_frozen_modules': -1, } path_config = {} if MS_WINDOWS: config['base_prefix'] = pyvenv_home config['prefix'] = pyvenv_home config['stdlib_dir'] = os.path.join(pyvenv_home, 'lib') + config['use_frozen_modules'] = 1 ver = sys.version_info dll = f'python{ver.major}' @@ -1383,6 +1380,7 @@ def test_init_pyvenv_cfg(self): # The current getpath.c doesn't determine the stdlib dir # in this case. config['stdlib_dir'] = None + config['use_frozen_modules'] = -1 env = self.copy_paths_by_env(config) self.check_all_configs("test_init_compat_config", config, diff --git a/Python/initconfig.c b/Python/initconfig.c index e753e8b9a8f66c..b94258ead642e0 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -2117,7 +2117,6 @@ is_dev_env(PyConfig *config) return 0; } /* If dirname() is the same for both then it is a dev build. */ - // XXX This doesn't work on Windows. if (len != _Py_find_basename(stdlib)) { return 0; } From 69242c027b457180a8f699a948a22c682bc8478c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 13 Oct 2021 17:00:23 -0600 Subject: [PATCH 08/11] Drop an outdated TODO comment. --- Python/initconfig.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/initconfig.c b/Python/initconfig.c index b94258ead642e0..c916e2f7c0714f 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -2120,7 +2120,8 @@ is_dev_env(PyConfig *config) if (len != _Py_find_basename(stdlib)) { return 0; } - // XXX Could either have .. in them? + // We do not bother normalizing the two filenames first since + // for config_init_import() is does the right thing as-is. if (wcsncmp(stdlib, executable, len) != 0) { return 0; } From b2aa67d3f123ce4f3b86c3f861673cd17718f162 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 13 Oct 2021 17:37:49 -0600 Subject: [PATCH 09/11] Use sys._stdlib_dir to find the stdlib dir. --- Lib/site.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/site.py b/Lib/site.py index 939893eb5ee93b..e129f3b4851f3d 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -418,8 +418,10 @@ def setcopyright(): files, dirs = [], [] # Not all modules are required to have a __file__ attribute. See # PEP 420 for more details. - if hasattr(os, '__file__'): + here = getattr(sys, '_stdlib_dir', None) + if not here and hasattr(os, '__file__'): here = os.path.dirname(os.__file__) + if here: files.extend(["LICENSE.txt", "LICENSE"]) dirs.extend([os.path.join(here, os.pardir), here, os.curdir]) builtins.license = _sitebuiltins._Printer( From fc7a72fb1251990f8a791d9a4a2e8ed9eca9a21a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 13 Oct 2021 17:59:44 -0600 Subject: [PATCH 10/11] Drop an outdated env var. --- Lib/test/test_embed.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index f760c1825522bd..17414dade28d62 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1268,9 +1268,7 @@ def test_init_setpythonhome(self): 'stdlib_dir': stdlib, } self.default_program_name(config) - env = {'TESTHOME': home, - 'PYTHONPATH': paths_str, - '_PYTHONTESTFROZENMODULES': '1'} + env = {'TESTHOME': home, 'PYTHONPATH': paths_str} self.check_all_configs("test_init_setpythonhome", config, api=API_COMPAT, env=env) From f1d38785b2b15168bcbcd25819fa9e2d66189354 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 15 Oct 2021 19:31:08 -0600 Subject: [PATCH 11/11] Fix test_embed when "python3" isn't on $PATH. --- Lib/test/test_embed.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 17414dade28d62..4b4396efb5cadb 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1268,6 +1268,8 @@ def test_init_setpythonhome(self): 'stdlib_dir': stdlib, } self.default_program_name(config) + if not config['executable']: + config['use_frozen_modules'] = -1 env = {'TESTHOME': home, 'PYTHONPATH': paths_str} self.check_all_configs("test_init_setpythonhome", config, api=API_COMPAT, env=env)