diff --git a/.gitattributes b/.gitattributes index 68566e899249f6..fd303806dac21a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -28,7 +28,6 @@ Lib/test/test_email/data/*.txt -text Lib/test/xmltestdata/* -text Lib/test/coding20731.py -text Lib/test/test_importlib/data01/* -text -Lib/test/test_importlib/namespacedata01/* -text # CRLF files *.bat text eol=crlf diff --git a/Lib/importlib/_adapters.py b/Lib/importlib/_adapters.py index 9907b148b396d2..eedde49dd03ad9 100644 --- a/Lib/importlib/_adapters.py +++ b/Lib/importlib/_adapters.py @@ -1,5 +1,4 @@ from contextlib import suppress -from io import TextIOWrapper from . import abc @@ -26,119 +25,32 @@ def __init__(self, spec): self.spec = spec def get_resource_reader(self, name): - return CompatibilityFiles(self.spec)._native() + return DegenerateFiles(self.spec)._native() -def _io_wrapper(file, mode='r', *args, **kwargs): - if mode == 'r': - return TextIOWrapper(file, *args, **kwargs) - elif mode == 'rb': - return file - raise ValueError( - "Invalid mode value '{}', only 'r' and 'rb' are supported".format(mode) - ) - - -class CompatibilityFiles: +class DegenerateFiles: """ Adapter for an existing or non-existant resource reader - to provide a compability .files(). + to provide a degenerate .files(). """ - class SpecPath(abc.Traversable): - """ - Path tied to a module spec. - Can be read and exposes the resource reader children. - """ - - def __init__(self, spec, reader): - self._spec = spec - self._reader = reader - - def iterdir(self): - if not self._reader: - return iter(()) - return iter( - CompatibilityFiles.ChildPath(self._reader, path) - for path in self._reader.contents() - ) - - def is_file(self): - return False - - is_dir = is_file - - def joinpath(self, other): - if not self._reader: - return CompatibilityFiles.OrphanPath(other) - return CompatibilityFiles.ChildPath(self._reader, other) - - @property - def name(self): - return self._spec.name - - def open(self, mode='r', *args, **kwargs): - return _io_wrapper(self._reader.open_resource(None), mode, *args, **kwargs) - - class ChildPath(abc.Traversable): - """ - Path tied to a resource reader child. - Can be read but doesn't expose any meaningfull children. - """ - - def __init__(self, reader, name): - self._reader = reader - self._name = name - + class Path(abc.Traversable): def iterdir(self): return iter(()) - def is_file(self): - return self._reader.is_resource(self.name) - def is_dir(self): - return not self.is_file() - - def joinpath(self, other): - return CompatibilityFiles.OrphanPath(self.name, other) - - @property - def name(self): - return self._name - - def open(self, mode='r', *args, **kwargs): - return _io_wrapper( - self._reader.open_resource(self.name), mode, *args, **kwargs - ) - - class OrphanPath(abc.Traversable): - """ - Orphan path, not tied to a module spec or resource reader. - Can't be read and doesn't expose any meaningful children. - """ - - def __init__(self, *path_parts): - if len(path_parts) < 1: - raise ValueError('Need at least one path part to construct a path') - self._path = path_parts - - def iterdir(self): - return iter(()) - - def is_file(self): return False - is_dir = is_file + is_file = exists = is_dir # type: ignore def joinpath(self, other): - return CompatibilityFiles.OrphanPath(*self._path, other) + return DegenerateFiles.Path() - @property def name(self): - return self._path[-1] + return '' - def open(self, mode='r', *args, **kwargs): - raise FileNotFoundError("Can't open orphan path") + def open(self): + raise ValueError() def __init__(self, spec): self.spec = spec @@ -159,7 +71,7 @@ def __getattr__(self, attr): return getattr(self._reader, attr) def files(self): - return CompatibilityFiles.SpecPath(self.spec, self._reader) + return DegenerateFiles.Path() def wrap_spec(package): diff --git a/Lib/importlib/_common.py b/Lib/importlib/_common.py index 74654b34ed5a9d..549fee379a415f 100644 --- a/Lib/importlib/_common.py +++ b/Lib/importlib/_common.py @@ -12,7 +12,6 @@ from ._adapters import wrap_spec Package = Union[types.ModuleType, str] -Resource = Union[str, os.PathLike] def files(package): @@ -94,7 +93,7 @@ def _tempfile(reader, suffix=''): finally: try: os.remove(raw_path) - except (FileNotFoundError, PermissionError): + except FileNotFoundError: pass diff --git a/Lib/importlib/_itertools.py b/Lib/importlib/_itertools.py deleted file mode 100644 index dd45f2f0966302..00000000000000 --- a/Lib/importlib/_itertools.py +++ /dev/null @@ -1,19 +0,0 @@ -from itertools import filterfalse - - -def unique_everseen(iterable, key=None): - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBCcAD', str.lower) --> A B C D - seen = set() - seen_add = seen.add - if key is None: - for element in filterfalse(seen.__contains__, iterable): - seen_add(element) - yield element - else: - for element in iterable: - k = key(element) - if k not in seen: - seen_add(k) - yield element diff --git a/Lib/importlib/_legacy.py b/Lib/importlib/_legacy.py deleted file mode 100644 index 2ddec5f90a323b..00000000000000 --- a/Lib/importlib/_legacy.py +++ /dev/null @@ -1,84 +0,0 @@ -import os -import pathlib -import types - -from typing import Union, Iterable, ContextManager, BinaryIO, TextIO - -from . import _common - -Package = Union[types.ModuleType, str] -Resource = Union[str, os.PathLike] - - -def open_binary(package: Package, resource: Resource) -> BinaryIO: - """Return a file-like object opened for binary reading of the resource.""" - return (_common.files(package) / _common.normalize_path(resource)).open('rb') - - -def read_binary(package: Package, resource: Resource) -> bytes: - """Return the binary contents of the resource.""" - return (_common.files(package) / _common.normalize_path(resource)).read_bytes() - - -def open_text( - package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict', -) -> TextIO: - """Return a file-like object opened for text reading of the resource.""" - return (_common.files(package) / _common.normalize_path(resource)).open( - 'r', encoding=encoding, errors=errors - ) - - -def read_text( - package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict', -) -> str: - """Return the decoded string of the resource. - - The decoding-related arguments have the same semantics as those of - bytes.decode(). - """ - with open_text(package, resource, encoding, errors) as fp: - return fp.read() - - -def contents(package: Package) -> Iterable[str]: - """Return an iterable of entries in `package`. - - Note that not all entries are resources. Specifically, directories are - not considered resources. Use `is_resource()` on each entry returned here - to check if it is a resource or not. - """ - return [path.name for path in _common.files(package).iterdir()] - - -def is_resource(package: Package, name: str) -> bool: - """True if `name` is a resource inside `package`. - - Directories are *not* resources. - """ - resource = _common.normalize_path(name) - return any( - traversable.name == resource and traversable.is_file() - for traversable in _common.files(package).iterdir() - ) - - -def path( - package: Package, - resource: Resource, -) -> ContextManager[pathlib.Path]: - """A context manager providing a file path object to the resource. - - If the resource does not already exist on its own on the file system, - a temporary file will be created. If the file was created, the file - will be deleted upon exiting the context manager (no exception is - raised if the file was deleted prior to the context manager - exiting). - """ - return _common.as_file(_common.files(package) / _common.normalize_path(resource)) diff --git a/Lib/importlib/readers.py b/Lib/importlib/readers.py index b470a2062b2b3a..41089c071d8683 100644 --- a/Lib/importlib/readers.py +++ b/Lib/importlib/readers.py @@ -1,12 +1,8 @@ import collections -import operator -import pathlib import zipfile - +import pathlib from . import abc -from ._itertools import unique_everseen - def remove_duplicates(items): return iter(collections.OrderedDict.fromkeys(items)) @@ -67,8 +63,13 @@ def __init__(self, *paths): raise NotADirectoryError('MultiplexedPath only supports directories') def iterdir(self): - files = (file for path in self._paths for file in path.iterdir()) - return unique_everseen(files, key=operator.attrgetter('name')) + visited = [] + for path in self._paths: + for file in path.iterdir(): + if file.name in visited: + continue + visited.append(file.name) + yield file def read_bytes(self): raise FileNotFoundError(f'{self} is not a file') diff --git a/Lib/importlib/resources.py b/Lib/importlib/resources.py index 6cc46283ba0a21..bb5c354d9f00ab 100644 --- a/Lib/importlib/resources.py +++ b/Lib/importlib/resources.py @@ -1,23 +1,19 @@ -"""Read resources contained within a package.""" +import os +import io -from ._common import ( - as_file, - files, - Package, - Resource, -) - -from ._legacy import ( - contents, - open_binary, - read_binary, - open_text, - read_text, - is_resource, - path, -) - -from importlib.abc import ResourceReader +from . import _common +from ._common import as_file, files +from .abc import ResourceReader +from contextlib import suppress +from importlib.abc import ResourceLoader +from importlib.machinery import ModuleSpec +from io import BytesIO, TextIOWrapper +from pathlib import Path +from types import ModuleType +from typing import ContextManager, Iterable, Union +from typing import cast, BinaryIO, TextIO +from collections.abc import Sequence +from functools import singledispatch __all__ = [ @@ -34,3 +30,155 @@ 'read_binary', 'read_text', ] + + +Package = Union[str, ModuleType] +Resource = Union[str, os.PathLike] + + +def open_binary(package: Package, resource: Resource) -> BinaryIO: + """Return a file-like object opened for binary reading of the resource.""" + resource = _common.normalize_path(resource) + package = _common.get_package(package) + reader = _common.get_resource_reader(package) + if reader is not None: + return reader.open_resource(resource) + spec = cast(ModuleSpec, package.__spec__) + # Using pathlib doesn't work well here due to the lack of 'strict' + # argument for pathlib.Path.resolve() prior to Python 3.6. + if spec.submodule_search_locations is not None: + paths = spec.submodule_search_locations + elif spec.origin is not None: + paths = [os.path.dirname(os.path.abspath(spec.origin))] + + for package_path in paths: + full_path = os.path.join(package_path, resource) + try: + return open(full_path, mode='rb') + except OSError: + # Just assume the loader is a resource loader; all the relevant + # importlib.machinery loaders are and an AttributeError for + # get_data() will make it clear what is needed from the loader. + loader = cast(ResourceLoader, spec.loader) + data = None + if hasattr(spec.loader, 'get_data'): + with suppress(OSError): + data = loader.get_data(full_path) + if data is not None: + return BytesIO(data) + + raise FileNotFoundError(f'{resource!r} resource not found in {spec.name!r}') + + +def open_text( + package: Package, + resource: Resource, + encoding: str = 'utf-8', + errors: str = 'strict', +) -> TextIO: + """Return a file-like object opened for text reading of the resource.""" + return TextIOWrapper( + open_binary(package, resource), encoding=encoding, errors=errors + ) + + +def read_binary(package: Package, resource: Resource) -> bytes: + """Return the binary contents of the resource.""" + with open_binary(package, resource) as fp: + return fp.read() + + +def read_text( + package: Package, + resource: Resource, + encoding: str = 'utf-8', + errors: str = 'strict', +) -> str: + """Return the decoded string of the resource. + + The decoding-related arguments have the same semantics as those of + bytes.decode(). + """ + with open_text(package, resource, encoding, errors) as fp: + return fp.read() + + +def path( + package: Package, + resource: Resource, +) -> 'ContextManager[Path]': + """A context manager providing a file path object to the resource. + + If the resource does not already exist on its own on the file system, + a temporary file will be created. If the file was created, the file + will be deleted upon exiting the context manager (no exception is + raised if the file was deleted prior to the context manager + exiting). + """ + reader = _common.get_resource_reader(_common.get_package(package)) + return ( + _path_from_reader(reader, _common.normalize_path(resource)) + if reader + else _common.as_file( + _common.files(package).joinpath(_common.normalize_path(resource)) + ) + ) + + +def _path_from_reader(reader, resource): + return _path_from_resource_path(reader, resource) or _path_from_open_resource( + reader, resource + ) + + +def _path_from_resource_path(reader, resource): + with suppress(FileNotFoundError): + return Path(reader.resource_path(resource)) + + +def _path_from_open_resource(reader, resource): + saved = io.BytesIO(reader.open_resource(resource).read()) + return _common._tempfile(saved.read, suffix=resource) + + +def is_resource(package: Package, name: str) -> bool: + """True if 'name' is a resource inside 'package'. + + Directories are *not* resources. + """ + package = _common.get_package(package) + _common.normalize_path(name) + reader = _common.get_resource_reader(package) + if reader is not None: + return reader.is_resource(name) + package_contents = set(contents(package)) + if name not in package_contents: + return False + return (_common.from_package(package) / name).is_file() + + +def contents(package: Package) -> Iterable[str]: + """Return an iterable of entries in 'package'. + + Note that not all entries are resources. Specifically, directories are + not considered resources. Use `is_resource()` on each entry returned here + to check if it is a resource or not. + """ + package = _common.get_package(package) + reader = _common.get_resource_reader(package) + if reader is not None: + return _ensure_sequence(reader.contents()) + transversable = _common.from_package(package) + if transversable.is_dir(): + return list(item.name for item in transversable.iterdir()) + return [] + + +@singledispatch +def _ensure_sequence(iterable): + return list(iterable) + + +@_ensure_sequence.register(Sequence) +def _(iterable): + return iterable diff --git a/Lib/importlib/simple.py b/Lib/importlib/simple.py deleted file mode 100644 index da073cbdb11e6c..00000000000000 --- a/Lib/importlib/simple.py +++ /dev/null @@ -1,116 +0,0 @@ -""" -Interface adapters for low-level readers. -""" - -import abc -import io -import itertools -from typing import BinaryIO, List - -from .abc import Traversable, TraversableResources - - -class SimpleReader(abc.ABC): - """ - The minimum, low-level interface required from a resource - provider. - """ - - @abc.abstractproperty - def package(self): - # type: () -> str - """ - The name of the package for which this reader loads resources. - """ - - @abc.abstractmethod - def children(self): - # type: () -> List['SimpleReader'] - """ - Obtain an iterable of SimpleReader for available - child containers (e.g. directories). - """ - - @abc.abstractmethod - def resources(self): - # type: () -> List[str] - """ - Obtain available named resources for this virtual package. - """ - - @abc.abstractmethod - def open_binary(self, resource): - # type: (str) -> BinaryIO - """ - Obtain a File-like for a named resource. - """ - - @property - def name(self): - return self.package.split('.')[-1] - - -class ResourceHandle(Traversable): - """ - Handle to a named resource in a ResourceReader. - """ - - def __init__(self, parent, name): - # type: (ResourceContainer, str) -> None - self.parent = parent - self.name = name # type: ignore - - def is_file(self): - return True - - def is_dir(self): - return False - - def open(self, mode='r', *args, **kwargs): - stream = self.parent.reader.open_binary(self.name) - if 'b' not in mode: - stream = io.TextIOWrapper(*args, **kwargs) - return stream - - def joinpath(self, name): - raise RuntimeError("Cannot traverse into a resource") - - -class ResourceContainer(Traversable): - """ - Traversable container for a package's resources via its reader. - """ - - def __init__(self, reader): - # type: (SimpleReader) -> None - self.reader = reader - - def is_dir(self): - return True - - def is_file(self): - return False - - def iterdir(self): - files = (ResourceHandle(self, name) for name in self.reader.resources) - dirs = map(ResourceContainer, self.reader.children()) - return itertools.chain(files, dirs) - - def open(self, *args, **kwargs): - raise IsADirectoryError() - - def joinpath(self, name): - return next( - traversable for traversable in self.iterdir() if traversable.name == name - ) - - -class TraversableReader(TraversableResources, SimpleReader): - """ - A TraversableResources based on SimpleReader. Resource providers - may derive from this class to provide the TraversableResources - interface by supplying the SimpleReader interface. - """ - - def files(self): - return ResourceContainer(self) diff --git a/Lib/test/test_importlib/resources/__init__.py b/Lib/test/test_importlib/resources/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_importlib/resources/util.py b/Lib/test/test_importlib/resources/util.py deleted file mode 100644 index d7a049bf8044cc..00000000000000 --- a/Lib/test/test_importlib/resources/util.py +++ /dev/null @@ -1,190 +0,0 @@ -import abc -import importlib -import io -import sys -import types -from pathlib import Path, PurePath - -from .. import data01 -from .. import zipdata01 -from importlib.abc import ResourceReader -from test.support import import_helper - - -from importlib.machinery import ModuleSpec - - -class Reader(ResourceReader): - def __init__(self, **kwargs): - vars(self).update(kwargs) - - def get_resource_reader(self, package): - return self - - def open_resource(self, path): - self._path = path - if isinstance(self.file, Exception): - raise self.file - return self.file - - def resource_path(self, path_): - self._path = path_ - if isinstance(self.path, Exception): - raise self.path - return self.path - - def is_resource(self, path_): - self._path = path_ - if isinstance(self.path, Exception): - raise self.path - - def part(entry): - return entry.split('/') - - return any( - len(parts) == 1 and parts[0] == path_ for parts in map(part, self._contents) - ) - - def contents(self): - if isinstance(self.path, Exception): - raise self.path - yield from self._contents - - -def create_package_from_loader(loader, is_package=True): - name = 'testingpackage' - module = types.ModuleType(name) - spec = ModuleSpec(name, loader, origin='does-not-exist', is_package=is_package) - module.__spec__ = spec - module.__loader__ = loader - return module - - -def create_package(file=None, path=None, is_package=True, contents=()): - return create_package_from_loader( - Reader(file=file, path=path, _contents=contents), - is_package, - ) - - -class CommonTests(metaclass=abc.ABCMeta): - """ - Tests shared by test_open, test_path, and test_read. - """ - - @abc.abstractmethod - def execute(self, package, path): - """ - Call the pertinent legacy API function (e.g. open_text, path) - on package and path. - """ - - def test_package_name(self): - # Passing in the package name should succeed. - self.execute(data01.__name__, 'utf-8.file') - - def test_package_object(self): - # Passing in the package itself should succeed. - self.execute(data01, 'utf-8.file') - - def test_string_path(self): - # Passing in a string for the path should succeed. - path = 'utf-8.file' - self.execute(data01, path) - - def test_pathlib_path(self): - # Passing in a pathlib.PurePath object for the path should succeed. - path = PurePath('utf-8.file') - self.execute(data01, path) - - def test_absolute_path(self): - # An absolute path is a ValueError. - path = Path(__file__) - full_path = path.parent / 'utf-8.file' - with self.assertRaises(ValueError): - self.execute(data01, full_path) - - def test_relative_path(self): - # A reative path is a ValueError. - with self.assertRaises(ValueError): - self.execute(data01, '../data01/utf-8.file') - - def test_importing_module_as_side_effect(self): - # The anchor package can already be imported. - del sys.modules[data01.__name__] - self.execute(data01.__name__, 'utf-8.file') - - def test_non_package_by_name(self): - # The anchor package cannot be a module. - with self.assertRaises(TypeError): - self.execute(__name__, 'utf-8.file') - - def test_non_package_by_package(self): - # The anchor package cannot be a module. - with self.assertRaises(TypeError): - module = sys.modules['test.test_importlib.resources.util'] - self.execute(module, 'utf-8.file') - - def test_missing_path(self): - # Attempting to open or read or request the path for a - # non-existent path should succeed if open_resource - # can return a viable data stream. - bytes_data = io.BytesIO(b'Hello, world!') - package = create_package(file=bytes_data, path=FileNotFoundError()) - self.execute(package, 'utf-8.file') - self.assertEqual(package.__loader__._path, 'utf-8.file') - - def test_extant_path(self): - # Attempting to open or read or request the path when the - # path does exist should still succeed. Does not assert - # anything about the result. - bytes_data = io.BytesIO(b'Hello, world!') - # any path that exists - path = __file__ - package = create_package(file=bytes_data, path=path) - self.execute(package, 'utf-8.file') - self.assertEqual(package.__loader__._path, 'utf-8.file') - - def test_useless_loader(self): - package = create_package(file=FileNotFoundError(), path=FileNotFoundError()) - with self.assertRaises(FileNotFoundError): - self.execute(package, 'utf-8.file') - - -class ZipSetupBase: - ZIP_MODULE = None - - @classmethod - def setUpClass(cls): - data_path = Path(cls.ZIP_MODULE.__file__) - data_dir = data_path.parent - cls._zip_path = str(data_dir / 'ziptestdata.zip') - sys.path.append(cls._zip_path) - cls.data = importlib.import_module('ziptestdata') - - @classmethod - def tearDownClass(cls): - try: - sys.path.remove(cls._zip_path) - except ValueError: - pass - - try: - del sys.path_importer_cache[cls._zip_path] - del sys.modules[cls.data.__name__] - except KeyError: - pass - - try: - del cls.data - del cls._zip_path - except AttributeError: - pass - - def setUp(self): - modules = import_helper.modules_setup() - self.addCleanup(import_helper.modules_cleanup, *modules) - - -class ZipSetup(ZipSetupBase): - ZIP_MODULE = zipdata01 # type: ignore diff --git a/Lib/test/test_importlib/test_compatibilty_files.py b/Lib/test/test_importlib/test_compatibilty_files.py deleted file mode 100644 index d703c060c44073..00000000000000 --- a/Lib/test/test_importlib/test_compatibilty_files.py +++ /dev/null @@ -1,102 +0,0 @@ -import io -import unittest - -from importlib import resources - -from importlib._adapters import ( - CompatibilityFiles, - wrap_spec, -) - -from .resources import util - - -class CompatibilityFilesTests(unittest.TestCase): - @property - def package(self): - bytes_data = io.BytesIO(b'Hello, world!') - return util.create_package( - file=bytes_data, - path='some_path', - contents=('a', 'b', 'c'), - ) - - @property - def files(self): - return resources.files(self.package) - - def test_spec_path_iter(self): - self.assertEqual( - sorted(path.name for path in self.files.iterdir()), - ['a', 'b', 'c'], - ) - - def test_child_path_iter(self): - self.assertEqual(list((self.files / 'a').iterdir()), []) - - def test_orphan_path_iter(self): - self.assertEqual(list((self.files / 'a' / 'a').iterdir()), []) - self.assertEqual(list((self.files / 'a' / 'a' / 'a').iterdir()), []) - - def test_spec_path_is(self): - self.assertFalse(self.files.is_file()) - self.assertFalse(self.files.is_dir()) - - def test_child_path_is(self): - self.assertTrue((self.files / 'a').is_file()) - self.assertFalse((self.files / 'a').is_dir()) - - def test_orphan_path_is(self): - self.assertFalse((self.files / 'a' / 'a').is_file()) - self.assertFalse((self.files / 'a' / 'a').is_dir()) - self.assertFalse((self.files / 'a' / 'a' / 'a').is_file()) - self.assertFalse((self.files / 'a' / 'a' / 'a').is_dir()) - - def test_spec_path_name(self): - self.assertEqual(self.files.name, 'testingpackage') - - def test_child_path_name(self): - self.assertEqual((self.files / 'a').name, 'a') - - def test_orphan_path_name(self): - self.assertEqual((self.files / 'a' / 'b').name, 'b') - self.assertEqual((self.files / 'a' / 'b' / 'c').name, 'c') - - def test_spec_path_open(self): - self.assertEqual(self.files.read_bytes(), b'Hello, world!') - self.assertEqual(self.files.read_text(), 'Hello, world!') - - def test_child_path_open(self): - self.assertEqual((self.files / 'a').read_bytes(), b'Hello, world!') - self.assertEqual((self.files / 'a').read_text(), 'Hello, world!') - - def test_orphan_path_open(self): - with self.assertRaises(FileNotFoundError): - (self.files / 'a' / 'b').read_bytes() - with self.assertRaises(FileNotFoundError): - (self.files / 'a' / 'b' / 'c').read_bytes() - - def test_open_invalid_mode(self): - with self.assertRaises(ValueError): - self.files.open('0') - - def test_orphan_path_invalid(self): - with self.assertRaises(ValueError): - CompatibilityFiles.OrphanPath() - - def test_wrap_spec(self): - spec = wrap_spec(self.package) - self.assertIsInstance(spec.loader.get_resource_reader(None), CompatibilityFiles) - - -class CompatibilityFilesNoReaderTests(unittest.TestCase): - @property - def package(self): - return util.create_package_from_loader(None) - - @property - def files(self): - return resources.files(self.package) - - def test_spec_path_joinpath(self): - self.assertIsInstance(self.files / 'a', CompatibilityFiles.OrphanPath) diff --git a/Lib/test/test_importlib/test_contents.py b/Lib/test/test_importlib/test_contents.py deleted file mode 100644 index 0f3aa84f5b5127..00000000000000 --- a/Lib/test/test_importlib/test_contents.py +++ /dev/null @@ -1,42 +0,0 @@ -import unittest -from importlib import resources - -from . import data01 -from .resources import util - - -class ContentsTests: - expected = { - '__init__.py', - 'binary.file', - 'subdirectory', - 'utf-16.file', - 'utf-8.file', - } - - def test_contents(self): - assert self.expected <= set(resources.contents(self.data)) - - -class ContentsDiskTests(ContentsTests, unittest.TestCase): - def setUp(self): - self.data = data01 - - -class ContentsZipTests(ContentsTests, util.ZipSetup, unittest.TestCase): - pass - - -class ContentsNamespaceTests(ContentsTests, unittest.TestCase): - expected = { - # no __init__ because of namespace design - # no subdirectory as incidental difference in fixture - 'binary.file', - 'utf-16.file', - 'utf-8.file', - } - - def setUp(self): - from . import namespacedata01 - - self.data = namespacedata01 diff --git a/Lib/test/test_importlib/test_files.py b/Lib/test/test_importlib/test_files.py index b9170d83bea912..481829b7422853 100644 --- a/Lib/test/test_importlib/test_files.py +++ b/Lib/test/test_importlib/test_files.py @@ -4,7 +4,7 @@ from importlib import resources from importlib.abc import Traversable from . import data01 -from .resources import util +from . import util class FilesTests: @@ -35,12 +35,5 @@ class OpenZipTests(FilesTests, util.ZipSetup, unittest.TestCase): pass -class OpenNamespaceTests(FilesTests, unittest.TestCase): - def setUp(self): - from . import namespacedata01 - - self.data = namespacedata01 - - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_open.py b/Lib/test/test_importlib/test_open.py index 6f88ff78b73a0c..b75675f43b63fd 100644 --- a/Lib/test/test_importlib/test_open.py +++ b/Lib/test/test_importlib/test_open.py @@ -2,16 +2,16 @@ from importlib import resources from . import data01 -from .resources import util +from . import util -class CommonBinaryTests(util.CommonTests, unittest.TestCase): +class CommonBinaryTests(util.CommonResourceTests, unittest.TestCase): def execute(self, package, path): with resources.open_binary(package, path): pass -class CommonTextTests(util.CommonTests, unittest.TestCase): +class CommonTextTests(util.CommonResourceTests, unittest.TestCase): def execute(self, package, path): with resources.open_text(package, path): pass diff --git a/Lib/test/test_importlib/test_path.py b/Lib/test/test_importlib/test_path.py index 4436d7f34ef411..d6ed09a9e0d0fa 100644 --- a/Lib/test/test_importlib/test_path.py +++ b/Lib/test/test_importlib/test_path.py @@ -3,10 +3,10 @@ from importlib import resources from . import data01 -from .resources import util +from . import util -class CommonTests(util.CommonTests, unittest.TestCase): +class CommonTests(util.CommonResourceTests, unittest.TestCase): def execute(self, package, path): with resources.path(package, path): pass diff --git a/Lib/test/test_importlib/test_read.py b/Lib/test/test_importlib/test_read.py index 357980132b6ec6..f6ec13af62d13c 100644 --- a/Lib/test/test_importlib/test_read.py +++ b/Lib/test/test_importlib/test_read.py @@ -2,15 +2,15 @@ from importlib import import_module, resources from . import data01 -from .resources import util +from . import util -class CommonBinaryTests(util.CommonTests, unittest.TestCase): +class CommonBinaryTests(util.CommonResourceTests, unittest.TestCase): def execute(self, package, path): resources.read_binary(package, path) -class CommonTextTests(util.CommonTests, unittest.TestCase): +class CommonTextTests(util.CommonResourceTests, unittest.TestCase): def execute(self, package, path): resources.read_text(package, path) @@ -55,12 +55,5 @@ def test_read_submodule_resource_by_name(self): self.assertEqual(result, b'\0\1\2\3') -class ReadNamespaceTests(ReadTests, unittest.TestCase): - def setUp(self): - from . import namespacedata01 - - self.data = namespacedata01 - - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_resource.py b/Lib/test/test_importlib/test_resource.py index 612bada5e0316c..003f7e95ad9122 100644 --- a/Lib/test/test_importlib/test_resource.py +++ b/Lib/test/test_importlib/test_resource.py @@ -5,7 +5,7 @@ from . import data01 from . import zipdata01, zipdata02 -from .resources import util +from . import util from importlib import resources, import_module from test.support import import_helper from test.support.os_helper import unlink @@ -33,14 +33,14 @@ def test_contents(self): # are not germane to this test, so just filter them out. contents.discard('__pycache__') self.assertEqual( - sorted(contents), - [ + contents, + { '__init__.py', - 'binary.file', 'subdirectory', - 'utf-16.file', 'utf-8.file', - ], + 'binary.file', + 'utf-16.file', + }, ) diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index c07ac2a64c289f..ca0d8c9b6eb352 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -1,11 +1,17 @@ +import abc import builtins import contextlib import errno import functools +import importlib from importlib import machinery, util, invalidate_caches +from importlib.abc import ResourceReader +import io import marshal import os import os.path +from pathlib import Path, PurePath +from test import support from test.support import import_helper from test.support import os_helper import unittest @@ -13,6 +19,9 @@ import tempfile import types +from . import data01 +from . import zipdata01 + BUILTINS = types.SimpleNamespace() BUILTINS.good_name = None @@ -408,3 +417,166 @@ def caseok_env_changed(self, *, should_exist): if any(x in self.importlib._bootstrap_external._os.environ for x in possibilities) != should_exist: self.skipTest('os.environ changes not reflected in _os.environ') + + +def create_package(file, path, is_package=True, contents=()): + class Reader(ResourceReader): + def get_resource_reader(self, package): + return self + + def open_resource(self, path): + self._path = path + if isinstance(file, Exception): + raise file + else: + return file + + def resource_path(self, path_): + self._path = path_ + if isinstance(path, Exception): + raise path + else: + return path + + def is_resource(self, path_): + self._path = path_ + if isinstance(path, Exception): + raise path + for entry in contents: + parts = entry.split('/') + if len(parts) == 1 and parts[0] == path_: + return True + return False + + def contents(self): + if isinstance(path, Exception): + raise path + # There's no yield from in baseball, er, Python 2. + for entry in contents: + yield entry + + name = 'testingpackage' + # Unfortunately importlib.util.module_from_spec() was not introduced until + # Python 3.5. + module = types.ModuleType(name) + loader = Reader() + spec = machinery.ModuleSpec( + name, loader, + origin='does-not-exist', + is_package=is_package) + module.__spec__ = spec + module.__loader__ = loader + return module + + +class CommonResourceTests(abc.ABC): + @abc.abstractmethod + def execute(self, package, path): + raise NotImplementedError + + def test_package_name(self): + # Passing in the package name should succeed. + self.execute(data01.__name__, 'utf-8.file') + + def test_package_object(self): + # Passing in the package itself should succeed. + self.execute(data01, 'utf-8.file') + + def test_string_path(self): + # Passing in a string for the path should succeed. + path = 'utf-8.file' + self.execute(data01, path) + + @unittest.skipIf(sys.version_info < (3, 6), 'requires os.PathLike support') + def test_pathlib_path(self): + # Passing in a pathlib.PurePath object for the path should succeed. + path = PurePath('utf-8.file') + self.execute(data01, path) + + def test_absolute_path(self): + # An absolute path is a ValueError. + path = Path(__file__) + full_path = path.parent/'utf-8.file' + with self.assertRaises(ValueError): + self.execute(data01, full_path) + + def test_relative_path(self): + # A relative path is a ValueError. + with self.assertRaises(ValueError): + self.execute(data01, '../data01/utf-8.file') + + def test_importing_module_as_side_effect(self): + # The anchor package can already be imported. + del sys.modules[data01.__name__] + self.execute(data01.__name__, 'utf-8.file') + + def test_non_package_by_name(self): + # The anchor package cannot be a module. + with self.assertRaises(TypeError): + self.execute(__name__, 'utf-8.file') + + def test_non_package_by_package(self): + # The anchor package cannot be a module. + with self.assertRaises(TypeError): + module = sys.modules['test.test_importlib.util'] + self.execute(module, 'utf-8.file') + + @unittest.skipIf(sys.version_info < (3,), 'No ResourceReader in Python 2') + def test_resource_opener(self): + bytes_data = io.BytesIO(b'Hello, world!') + package = create_package(file=bytes_data, path=FileNotFoundError()) + self.execute(package, 'utf-8.file') + self.assertEqual(package.__loader__._path, 'utf-8.file') + + @unittest.skipIf(sys.version_info < (3,), 'No ResourceReader in Python 2') + def test_resource_path(self): + bytes_data = io.BytesIO(b'Hello, world!') + path = __file__ + package = create_package(file=bytes_data, path=path) + self.execute(package, 'utf-8.file') + self.assertEqual(package.__loader__._path, 'utf-8.file') + + def test_useless_loader(self): + package = create_package(file=FileNotFoundError(), + path=FileNotFoundError()) + with self.assertRaises(FileNotFoundError): + self.execute(package, 'utf-8.file') + + +class ZipSetupBase: + ZIP_MODULE = None + + @classmethod + def setUpClass(cls): + data_path = Path(cls.ZIP_MODULE.__file__) + data_dir = data_path.parent + cls._zip_path = str(data_dir / 'ziptestdata.zip') + sys.path.append(cls._zip_path) + cls.data = importlib.import_module('ziptestdata') + + @classmethod + def tearDownClass(cls): + try: + sys.path.remove(cls._zip_path) + except ValueError: + pass + + try: + del sys.path_importer_cache[cls._zip_path] + del sys.modules[cls.data.__name__] + except KeyError: + pass + + try: + del cls.data + del cls._zip_path + except AttributeError: + pass + + def setUp(self): + modules = import_helper.modules_setup() + self.addCleanup(import_helper.modules_cleanup, *modules) + + +class ZipSetup(ZipSetupBase): + ZIP_MODULE = zipdata01 # type: ignore diff --git a/Misc/NEWS.d/next/Library/2021-07-28-22-53-18.bpo-44771.BvLdnU.rst b/Misc/NEWS.d/next/Library/2021-07-28-22-53-18.bpo-44771.BvLdnU.rst deleted file mode 100644 index 0d47a55a7d74f9..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-28-22-53-18.bpo-44771.BvLdnU.rst +++ /dev/null @@ -1,5 +0,0 @@ -Added ``importlib.simple`` module implementing adapters from a low-level -resources reader interface to a ``TraversableResources`` interface. Legacy -API (``path``, ``contents``, ...) is now supported entirely by the -``.files()`` API with a compatibility shim supplied for resource loaders -without that functionality. Feature parity with ``importlib_resources`` 5.2.