From 496acc1a0d8018c830b30a3a28826c9b101975fa Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 12 Sep 2023 15:05:00 -0400 Subject: [PATCH 1/3] Replace static zip fixtures with dynamically generated zip fixtures built from the same modules as found on the file system. --- importlib_resources/tests/test_read.py | 6 +- importlib_resources/tests/test_resource.py | 77 ++++-------------- importlib_resources/tests/update-zips.py | 53 ------------ importlib_resources/tests/util.py | 48 ++++------- importlib_resources/tests/zip.py | 29 +++++++ .../tests/zipdata01/__init__.py | 0 .../tests/zipdata01/ziptestdata.zip | Bin 876 -> 0 bytes .../tests/zipdata02/__init__.py | 0 .../tests/zipdata02/ziptestdata.zip | Bin 698 -> 0 bytes 9 files changed, 66 insertions(+), 147 deletions(-) delete mode 100755 importlib_resources/tests/update-zips.py create mode 100755 importlib_resources/tests/zip.py delete mode 100644 importlib_resources/tests/zipdata01/__init__.py delete mode 100644 importlib_resources/tests/zipdata01/ziptestdata.zip delete mode 100644 importlib_resources/tests/zipdata02/__init__.py delete mode 100644 importlib_resources/tests/zipdata02/ziptestdata.zip diff --git a/importlib_resources/tests/test_read.py b/importlib_resources/tests/test_read.py index b5aaec45..5b83221e 100644 --- a/importlib_resources/tests/test_read.py +++ b/importlib_resources/tests/test_read.py @@ -58,15 +58,13 @@ class ReadDiskTests(ReadTests, unittest.TestCase): class ReadZipTests(ReadTests, util.ZipSetup, unittest.TestCase): def test_read_submodule_resource(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') result = resources.files(submodule).joinpath('binary.file').read_bytes() self.assertEqual(result, b'\0\1\2\3') def test_read_submodule_resource_by_name(self): result = ( - resources.files('ziptestdata.subdirectory') - .joinpath('binary.file') - .read_bytes() + resources.files('data01.subdirectory').joinpath('binary.file').read_bytes() ) self.assertEqual(result, b'\0\1\2\3') diff --git a/importlib_resources/tests/test_resource.py b/importlib_resources/tests/test_resource.py index 677110cd..4069b6ca 100644 --- a/importlib_resources/tests/test_resource.py +++ b/importlib_resources/tests/test_resource.py @@ -1,15 +1,11 @@ -import contextlib import sys import unittest import importlib_resources as resources -import uuid import pathlib from . import data01 -from . import zipdata01, zipdata02 from . import util from importlib import import_module -from ._compat import import_helper, os_helper, unlink class ResourceTests: @@ -89,34 +85,32 @@ def test_package_has_no_reader_fallback(self): class ResourceFromZipsTest01(util.ZipSetupBase, unittest.TestCase): - ZIP_MODULE = zipdata01 # type: ignore + ZIP_MODULE = 'data01' def test_is_submodule_resource(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') self.assertTrue(resources.files(submodule).joinpath('binary.file').is_file()) def test_read_submodule_resource_by_name(self): self.assertTrue( - resources.files('ziptestdata.subdirectory') - .joinpath('binary.file') - .is_file() + resources.files('data01.subdirectory').joinpath('binary.file').is_file() ) def test_submodule_contents(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') self.assertEqual( names(resources.files(submodule)), {'__init__.py', 'binary.file'} ) def test_submodule_contents_by_name(self): self.assertEqual( - names(resources.files('ziptestdata.subdirectory')), + names(resources.files('data01.subdirectory')), {'__init__.py', 'binary.file'}, ) def test_as_file_directory(self): - with resources.as_file(resources.files('ziptestdata')) as data: - assert data.name == 'ziptestdata' + with resources.as_file(resources.files('data01')) as data: + assert data.name == 'data01' assert data.is_dir() assert data.joinpath('subdirectory').is_dir() assert len(list(data.iterdir())) @@ -124,7 +118,7 @@ def test_as_file_directory(self): class ResourceFromZipsTest02(util.ZipSetupBase, unittest.TestCase): - ZIP_MODULE = zipdata02 # type: ignore + ZIP_MODULE = 'data02' def test_unrelated_contents(self): """ @@ -132,80 +126,45 @@ def test_unrelated_contents(self): distinct resources. Ref python/importlib_resources#44. """ self.assertEqual( - names(resources.files('ziptestdata.one')), + names(resources.files('data02.one')), {'__init__.py', 'resource1.txt'}, ) self.assertEqual( - names(resources.files('ziptestdata.two')), + names(resources.files('data02.two')), {'__init__.py', 'resource2.txt'}, ) -@contextlib.contextmanager -def zip_on_path(dir): - data_path = pathlib.Path(zipdata01.__file__) - source_zip_path = data_path.parent.joinpath('ziptestdata.zip') - zip_path = pathlib.Path(dir) / f'{uuid.uuid4()}.zip' - zip_path.write_bytes(source_zip_path.read_bytes()) - sys.path.append(str(zip_path)) - import_module('ziptestdata') - - try: - yield - finally: - with contextlib.suppress(ValueError): - sys.path.remove(str(zip_path)) - - with contextlib.suppress(KeyError): - del sys.path_importer_cache[str(zip_path)] - del sys.modules['ziptestdata'] - - with contextlib.suppress(OSError): - unlink(zip_path) - - -class DeletingZipsTest(unittest.TestCase): +class DeletingZipsTest(util.ZipSetupBase, unittest.TestCase): """Having accessed resources in a zip file should not keep an open reference to the zip. """ - def setUp(self): - self.fixtures = contextlib.ExitStack() - self.addCleanup(self.fixtures.close) - - modules = import_helper.modules_setup() - self.addCleanup(import_helper.modules_cleanup, *modules) - - temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) - self.fixtures.enter_context(zip_on_path(temp_dir)) - def test_iterdir_does_not_keep_open(self): - [item.name for item in resources.files('ziptestdata').iterdir()] + [item.name for item in resources.files('data01').iterdir()] def test_is_file_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('binary.file').is_file() + resources.files('data01').joinpath('binary.file').is_file() def test_is_file_failure_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('not-present').is_file() + resources.files('data01').joinpath('not-present').is_file() @unittest.skip("Desired but not supported.") def test_as_file_does_not_keep_open(self): # pragma: no cover - resources.as_file(resources.files('ziptestdata') / 'binary.file') + resources.as_file(resources.files('data01') / 'binary.file') def test_entered_path_does_not_keep_open(self): """ Mimic what certifi does on import to make its bundle available for the process duration. """ - resources.as_file(resources.files('ziptestdata') / 'binary.file').__enter__() + resources.as_file(resources.files('data01') / 'binary.file').__enter__() def test_read_binary_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('binary.file').read_bytes() + resources.files('data01').joinpath('binary.file').read_bytes() def test_read_text_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('utf-8.file').read_text( - encoding='utf-8' - ) + resources.files('data01').joinpath('utf-8.file').read_text(encoding='utf-8') class ResourceFromNamespaceTest01(unittest.TestCase): diff --git a/importlib_resources/tests/update-zips.py b/importlib_resources/tests/update-zips.py deleted file mode 100755 index 231334aa..00000000 --- a/importlib_resources/tests/update-zips.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -Generate the zip test data files. - -Run to build the tests/zipdataNN/ziptestdata.zip files from -files in tests/dataNN. - -Replaces the file with the working copy, but does commit anything -to the source repo. -""" - -import contextlib -import os -import pathlib -import zipfile - - -def main(): - """ - >>> from unittest import mock - >>> monkeypatch = getfixture('monkeypatch') - >>> monkeypatch.setattr(zipfile, 'ZipFile', mock.MagicMock()) - >>> print(); main() # print workaround for bpo-32509 - - ...data01... -> ziptestdata/... - ... - ...data02... -> ziptestdata/... - ... - """ - suffixes = '01', '02' - tuple(map(generate, suffixes)) - - -def generate(suffix): - root = pathlib.Path(__file__).parent.relative_to(os.getcwd()) - zfpath = root / f'zipdata{suffix}/ziptestdata.zip' - with zipfile.ZipFile(zfpath, 'w') as zf: - for src, rel in walk(root / f'data{suffix}'): - dst = 'ziptestdata' / pathlib.PurePosixPath(rel.as_posix()) - print(src, '->', dst) - zf.write(src, dst) - - -def walk(datapath): - for dirpath, dirnames, filenames in os.walk(datapath): - with contextlib.suppress(ValueError): - dirnames.remove('__pycache__') - for filename in filenames: - res = pathlib.Path(dirpath) / filename - rel = res.relative_to(datapath) - yield res, rel - - -__name__ == '__main__' and main() diff --git a/importlib_resources/tests/util.py b/importlib_resources/tests/util.py index ce0e6fad..066f4113 100644 --- a/importlib_resources/tests/util.py +++ b/importlib_resources/tests/util.py @@ -4,11 +4,12 @@ import sys import types import pathlib +import contextlib from . import data01 -from . import zipdata01 from ..abc import ResourceReader -from ._compat import import_helper +from ._compat import import_helper, os_helper +from . import zip as zip_ from importlib.machinery import ModuleSpec @@ -141,39 +142,24 @@ def test_useless_loader(self): class ZipSetupBase: - ZIP_MODULE = None - - @classmethod - def setUpClass(cls): - data_path = pathlib.Path(cls.ZIP_MODULE.__file__) - data_dir = data_path.parent - cls._zip_path = str(data_dir / 'ziptestdata.zip') - sys.path.append(cls._zip_path) - cls.data = importlib.import_module('ziptestdata') - - @classmethod - def tearDownClass(cls): - try: - sys.path.remove(cls._zip_path) - except ValueError: - pass - - try: - del sys.path_importer_cache[cls._zip_path] - del sys.modules[cls.data.__name__] - except KeyError: - pass - - try: - del cls.data - del cls._zip_path - except AttributeError: - pass + ZIP_MODULE = 'data01' def setUp(self): + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + modules = import_helper.modules_setup() self.addCleanup(import_helper.modules_cleanup, *modules) + temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + modules = pathlib.Path(temp_dir) / 'zipped modules.zip' + src_path = pathlib.Path(__file__).parent.joinpath(self.ZIP_MODULE) + self.fixtures.enter_context( + import_helper.DirsOnSysPath(str(zip_.make_zip_file(src_path, modules))) + ) + + self.data = importlib.import_module(self.ZIP_MODULE) + class ZipSetup(ZipSetupBase): - ZIP_MODULE = zipdata01 # type: ignore + pass diff --git a/importlib_resources/tests/zip.py b/importlib_resources/tests/zip.py new file mode 100755 index 00000000..6ab914b0 --- /dev/null +++ b/importlib_resources/tests/zip.py @@ -0,0 +1,29 @@ +""" +Generate zip test data files. +""" + +import contextlib +import os +import pathlib +import zipfile + + +def make_zip_file(src, dst): + """ + Zip the files in src into a new zipfile at dst. + """ + with zipfile.ZipFile(dst, 'w') as zf: + for src_path, rel in walk(src): + dst_name = src.name / pathlib.PurePosixPath(rel.as_posix()) + zf.write(src_path, dst_name) + return dst + + +def walk(datapath): + for dirpath, dirnames, filenames in os.walk(datapath): + with contextlib.suppress(ValueError): + dirnames.remove('__pycache__') + for filename in filenames: + res = pathlib.Path(dirpath) / filename + rel = res.relative_to(datapath) + yield res, rel diff --git a/importlib_resources/tests/zipdata01/__init__.py b/importlib_resources/tests/zipdata01/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/importlib_resources/tests/zipdata01/ziptestdata.zip b/importlib_resources/tests/zipdata01/ziptestdata.zip deleted file mode 100644 index 9a3bb0739f87e97c1084b94d7d153680f6727738..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 876 zcmWIWW@Zs#00HOCX@Q%&m27l?Y!DU);;PJolGNgol*E!m{nC;&T|+ayw9K5;|NlG~ zQWMD z9;rDw`8o=rA#S=B3g!7lIVp-}COK17UPc zNtt;*xhM-3R!jMEPhCreO-3*u>5Df}T7+BJ{639e$2uhfsIs`pJ5Qf}C xGXyDE@VNvOv@o!wQJfLgCAgysx3f@9jKpUmiW^zkK<;1z!tFpk^MROw0RS~O%0&PG diff --git a/importlib_resources/tests/zipdata02/__init__.py b/importlib_resources/tests/zipdata02/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/importlib_resources/tests/zipdata02/ziptestdata.zip b/importlib_resources/tests/zipdata02/ziptestdata.zip deleted file mode 100644 index d63ff512d2807ef2fd259455283b81b02e0e45fb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 698 zcmWIWW@Zs#00HOCX@Ot{ln@8fRhb1Psl_EJi6x2p@$s2?nI-Y@dIgmMI5kP5Y0A$_ z#jWw|&p#`9ff_(q7K_HB)Z+ZoqU2OVy^@L&ph*fa0WRVlP*R?c+X1opI-R&20MZDv z&j{oIpa8N17@0(vaR(gGH(;=&5k%n(M%;#g0ulz6G@1gL$cA79E2=^00gEsw4~s!C zUxI@ZWaIMqz|BszK;s4KsL2<9jRy!Q2E6`2cTLHjr{wAk1ZCU@!+_ G1_l6Bc%f?m From 023d2c13e5c3598e187470369d94bcf39ed411b8 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 12 Sep 2023 15:37:11 -0400 Subject: [PATCH 2/3] Separate 'disk' concern of namespace tests. --- importlib_resources/tests/test_resource.py | 24 ++++++++++++---------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/importlib_resources/tests/test_resource.py b/importlib_resources/tests/test_resource.py index 4069b6ca..40368056 100644 --- a/importlib_resources/tests/test_resource.py +++ b/importlib_resources/tests/test_resource.py @@ -167,17 +167,7 @@ def test_read_text_does_not_keep_open(self): resources.files('data01').joinpath('utf-8.file').read_text(encoding='utf-8') -class ResourceFromNamespaceTest01(unittest.TestCase): - site_dir = str(pathlib.Path(__file__).parent) - - @classmethod - def setUpClass(cls): - sys.path.append(cls.site_dir) - - @classmethod - def tearDownClass(cls): - sys.path.remove(cls.site_dir) - +class ResourceFromNamespaceTests: def test_is_submodule_resource(self): self.assertTrue( resources.files(import_module('namespacedata01')) @@ -207,5 +197,17 @@ def test_submodule_contents_by_name(self): self.assertEqual(contents, {'binary.file', 'utf-8.file', 'utf-16.file'}) +class ResourceFromNamespaceDiskTests(ResourceFromNamespaceTests, unittest.TestCase): + site_dir = str(pathlib.Path(__file__).parent) + + @classmethod + def setUpClass(cls): + sys.path.append(cls.site_dir) + + @classmethod + def tearDownClass(cls): + sys.path.remove(cls.site_dir) + + if __name__ == '__main__': unittest.main() From d3a7f69e44043719330fb68e6183ce0da31b47d3 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 18 Sep 2023 11:38:27 -0400 Subject: [PATCH 3/3] In zip namespace fixtures, explicitly generate the directory entries implied by children. Workaround for python/cpython#59110. --- importlib_resources/tests/zip.py | 3 +++ setup.cfg | 1 + 2 files changed, 4 insertions(+) diff --git a/importlib_resources/tests/zip.py b/importlib_resources/tests/zip.py index 6ab914b0..962195a9 100755 --- a/importlib_resources/tests/zip.py +++ b/importlib_resources/tests/zip.py @@ -7,6 +7,8 @@ import pathlib import zipfile +import zipp + def make_zip_file(src, dst): """ @@ -16,6 +18,7 @@ def make_zip_file(src, dst): for src_path, rel in walk(src): dst_name = src.name / pathlib.PurePosixPath(rel.as_posix()) zf.write(src_path, dst_name) + zipp.CompleteDirs.inject(zf) return dst diff --git a/setup.cfg b/setup.cfg index 52ec4c25..571356dd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -44,6 +44,7 @@ testing = pytest-ruff # local + zipp >= 3.17 docs = # upstream