From b24ecb7639a9f558240d4d69df0e4094eba6a51b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 27 Sep 2023 12:32:12 +0200 Subject: [PATCH 1/4] gh-109615: Fix support test_copy_python_src_ignore() (#109958) Fix the test when run on an installed Python: use "abs_srcdir" of sysconfig, and skip the test if the Python source code cannot be found. * Tools/patchcheck/patchcheck.py, Tools/freeze/test/freeze.py and Lib/test/libregrtest/utils.py now first try to get "abs_srcdir" from sysconfig, before getting "srcdir" from sysconfig. * test.pythoninfo logs sysconfig "abs_srcdir". (cherry picked from commit b89ed9df39851348fbb1552294644f99f6b17d2c) --- Lib/test/libregrtest/main.py | 12 +++++++----- Lib/test/pythoninfo.py | 2 ++ Lib/test/test_support.py | 7 ++++++- Tools/freeze/test/freeze.py | 9 ++++++++- Tools/patchcheck/patchcheck.py | 9 ++++++++- 5 files changed, 31 insertions(+), 8 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 7cfea548ee2180..56d562d0b068dd 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -747,11 +747,13 @@ def set_temp_dir(self): if sysconfig.is_python_build(): self.tmp_dir = sysconfig.get_config_var('abs_builddir') if self.tmp_dir is None: - # gh-74470: On Windows, only srcdir is available. Using - # abs_builddir mostly matters on UNIX when building Python - # out of the source tree, especially when the source tree - # is read only. - self.tmp_dir = sysconfig.get_config_var('srcdir') + self.tmp_dir = sysconfig.get_config_var('abs_srcdir') + if not self.tmp_dir: + # gh-74470: On Windows, only srcdir is available. Using + # abs_builddir mostly matters on UNIX when building + # Python out of the source tree, especially when the + # source tree is read only. + self.tmp_dir = sysconfig.get_config_var('srcdir') self.tmp_dir = os.path.join(self.tmp_dir, 'build') else: self.tmp_dir = tempfile.gettempdir() diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 860c4cc4e0a818..1ec04de3c0c94f 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -516,6 +516,8 @@ def collect_sysconfig(info_add): 'Py_ENABLE_SHARED', 'SHELL', 'SOABI', + 'abs_builddir', + 'abs_srcdir', 'prefix', ): value = sysconfig.get_config_var(name) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index d3b2922d9b0ad1..fa33d328d82c6e 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -767,8 +767,13 @@ def recursive_function(depth): #self.assertEqual(available, 2) def test_copy_python_src_ignore(self): - src_dir = sysconfig.get_config_var('srcdir') + src_dir = sysconfig.get_config_var('abs_srcdir') + if not src_dir: + src_dir = sysconfig.get_config_var('srcdir') src_dir = os.path.abspath(src_dir) + if not os.path.exists(src_dir): + self.skipTest(f"cannot access Python source code directory:" + f" {src_dir!r}") ignored = {'.git', '__pycache__'} diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index bb15941464e3d1..cdf77c57bbb6ae 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -7,9 +7,16 @@ from test import support +def get_python_source_dir(): + src_dir = sysconfig.get_config_var('abs_srcdir') + if not src_dir: + src_dir = sysconfig.get_config_var('srcdir') + return os.path.abspath(src_dir) + + TESTS_DIR = os.path.dirname(__file__) TOOL_ROOT = os.path.dirname(TESTS_DIR) -SRCDIR = os.path.abspath(sysconfig.get_config_var('srcdir')) +SRCDIR = get_python_source_dir() MAKE = shutil.which('make') FREEZE = os.path.join(TOOL_ROOT, 'freeze.py') diff --git a/Tools/patchcheck/patchcheck.py b/Tools/patchcheck/patchcheck.py index fa3a43af6e6048..e3959ce428c7c5 100755 --- a/Tools/patchcheck/patchcheck.py +++ b/Tools/patchcheck/patchcheck.py @@ -11,6 +11,13 @@ import untabify +def get_python_source_dir(): + src_dir = sysconfig.get_config_var('abs_srcdir') + if not src_dir: + src_dir = sysconfig.get_config_var('srcdir') + return os.path.abspath(src_dir) + + # Excluded directories which are copies of external libraries: # don't check their coding style EXCLUDE_DIRS = [ @@ -18,7 +25,7 @@ os.path.join('Modules', 'expat'), os.path.join('Modules', 'zlib'), ] -SRCDIR = sysconfig.get_config_var('srcdir') +SRCDIR = get_python_source_dir() def n_files_str(count): From c310ecb0534aeed2373993eaca32222cdf809e27 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 27 Sep 2023 17:29:20 +0200 Subject: [PATCH 2/4] gh-109615: Fix support test_copy_python_src_ignore() on WASM (#109970) Not only check if src_dir exists, but look also for Lib/os.py landmark. (cherry picked from commit cc54bcf17b5b5f7681f52baf3acef75b995fa1fd) --- Lib/test/test_support.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index fa33d328d82c6e..3e06d2b24004e3 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -767,17 +767,25 @@ def recursive_function(depth): #self.assertEqual(available, 2) def test_copy_python_src_ignore(self): + # Get source directory src_dir = sysconfig.get_config_var('abs_srcdir') if not src_dir: src_dir = sysconfig.get_config_var('srcdir') src_dir = os.path.abspath(src_dir) + + # Check that the source code is available if not os.path.exists(src_dir): self.skipTest(f"cannot access Python source code directory:" f" {src_dir!r}") + landmark = os.path.join(src_dir, 'Lib', 'os.py') + if not os.path.exists(landmark): + self.skipTest(f"cannot access Python source code directory:" + f" {landmark!r} landmark is missing") - ignored = {'.git', '__pycache__'} + # Test support.copy_python_src_ignore() # Source code directory + ignored = {'.git', '__pycache__'} names = os.listdir(src_dir) self.assertEqual(support.copy_python_src_ignore(src_dir, names), ignored | {'build'}) From 4f4c799e83721c1a2c2c7fe1068c2496242ecc8d Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 29 Sep 2023 16:24:38 +0100 Subject: [PATCH 3/4] gh-109615: Look for 'Modules' as landmark for test_copy_python_src_ignore (GH-110108) (cherry picked from commit 20bc5f7c28a6f8a2e156c4a748ffabb5efc7c761) --- Lib/test/test_support.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 3e06d2b24004e3..8dc1175f30cd95 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -777,7 +777,9 @@ def test_copy_python_src_ignore(self): if not os.path.exists(src_dir): self.skipTest(f"cannot access Python source code directory:" f" {src_dir!r}") - landmark = os.path.join(src_dir, 'Lib', 'os.py') + # Check that the landmark copy_python_src_ignore() expects is available + # (Previously we looked for 'Lib\os.py', which is always present on Windows.) + landmark = os.path.join(src_dir, 'Modules') if not os.path.exists(landmark): self.skipTest(f"cannot access Python source code directory:" f" {landmark!r} landmark is missing") From a3340576a25462ef4acec6b74cbdf188e35bfe6d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 30 Sep 2023 20:23:26 +0200 Subject: [PATCH 4/4] gh-109748: Fix again venv test_zippath_from_non_installed_posix() (#110149) Call also copy_python_src_ignore() on listdir() names. shutil.copytree(): replace set() with an empty tuple. An empty tuple becomes a constant in the compiler and checking if an item is in an empty tuple is cheap. (cherry picked from commit 0def8c712bb6f66f1081cab71deb3681566b846d) --- Lib/shutil.py | 2 +- Lib/test/test_support.py | 2 +- Lib/test/test_venv.py | 6 +++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Lib/shutil.py b/Lib/shutil.py index 5bcfa563fea9af..a278b74fab2ddb 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -481,7 +481,7 @@ def _copytree(entries, src, dst, symlinks, ignore, copy_function, if ignore is not None: ignored_names = ignore(os.fspath(src), [x.name for x in entries]) else: - ignored_names = set() + ignored_names = () os.makedirs(dst, exist_ok=dirs_exist_ok) errors = [] diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 8dc1175f30cd95..b9b05fc4306a31 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -797,7 +797,7 @@ def test_copy_python_src_ignore(self): self.assertEqual(support.copy_python_src_ignore(path, os.listdir(path)), ignored | {'build', 'venv'}) - # An other directory + # Another directory path = os.path.join(src_dir, 'Objects') self.assertEqual(support.copy_python_src_ignore(path, os.listdir(path)), ignored) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 6942bf246b61d1..fea16568afebae 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -571,7 +571,11 @@ def test_zippath_from_non_installed_posix(self): eachpath, os.path.join(non_installed_dir, platlibdir)) elif os.path.isfile(os.path.join(eachpath, "os.py")): - for name in os.listdir(eachpath): + names = os.listdir(eachpath) + ignored_names = copy_python_src_ignore(eachpath, names) + for name in names: + if name in ignored_names: + continue if name == "site-packages": continue fn = os.path.join(eachpath, name)