Skip to content

Add stubs for importlib.(resources.)readers #10928

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Oct 26, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pyrightconfig.stricter.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
"stubs/**/@tests/test_cases",
"stdlib/distutils/command",
"stdlib/distutils/dist.pyi",
"stdlib/importlib/readers.pyi",
"stdlib/lib2to3/fixes/*.pyi",
"stdlib/_tkinter.pyi",
"stdlib/tkinter/__init__.pyi",
Expand Down
2 changes: 2 additions & 0 deletions stdlib/VERSIONS
Original file line number Diff line number Diff line change
Expand Up @@ -152,8 +152,10 @@ imp: 2.7-3.11
importlib: 2.7-
importlib.metadata: 3.8-
importlib.metadata._meta: 3.10-
importlib.readers: 3.10-
importlib.resources: 3.7-
importlib.resources.abc: 3.11-
importlib.resources.readers: 3.11-
inspect: 2.7-
io: 2.7-
ipaddress: 3.3-
Expand Down
12 changes: 8 additions & 4 deletions stdlib/importlib/abc.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ from abc import ABCMeta, abstractmethod
from collections.abc import Iterator, Mapping, Sequence
from importlib.machinery import ModuleSpec
from io import BufferedRandom, BufferedReader, BufferedWriter, FileIO, TextIOWrapper
from typing import IO, Any, BinaryIO, NoReturn, Protocol, overload, runtime_checkable
from typing import IO, Any, BinaryIO, Protocol, overload, runtime_checkable
from typing_extensions import Literal

if sys.version_info >= (3, 11):
Expand Down Expand Up @@ -203,8 +203,12 @@ if sys.version_info >= (3, 9):
@property
@abstractmethod
def name(self) -> str: ...
@abstractmethod
def __truediv__(self, child: str) -> Traversable: ...
if sys.version_info >= (3, 10):
def __truediv__(self, child: str) -> Traversable: ...
else:
@abstractmethod
def __truediv__(self, child: str) -> Traversable: ...

@abstractmethod
def read_bytes(self) -> bytes: ...
@abstractmethod
Expand All @@ -214,6 +218,6 @@ if sys.version_info >= (3, 9):
@abstractmethod
def files(self) -> Traversable: ...
def open_resource(self, resource: str) -> BufferedReader: ...
def resource_path(self, resource: Any) -> NoReturn: ...
def resource_path(self, resource: Any) -> str: ...
def is_resource(self, path: str) -> bool: ...
def contents(self) -> Iterator[str]: ...
61 changes: 61 additions & 0 deletions stdlib/importlib/readers.pyi
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
# On py311+, things are actually defined in importlib.resources.readers,
# and re-exported here,
# but doing it this way leads to less code duplication for us

import pathlib
import sys
from _typeshed import Incomplete, StrPath
from collections.abc import Iterable, Iterator
from io import BufferedReader
from typing import NoReturn, TypeVar
from typing_extensions import Literal, Never

if sys.version_info >= (3, 11):
import importlib.resources.abc as abc
else:
import importlib.abc as abc

if sys.version_info >= (3, 10):
if sys.version_info >= (3, 11):
__all__ = ["FileReader", "ZipReader", "MultiplexedPath", "NamespaceReader"]

if sys.version_info < (3, 11):
_T = TypeVar("_T")

def remove_duplicates(items: Iterable[_T]) -> Iterator[_T]: ...

class FileReader(abc.TraversableResources):
path: pathlib.Path
def __init__(self, loader) -> None: ...
def resource_path(self, resource: StrPath) -> str: ...
def files(self) -> pathlib.Path: ...

class ZipReader(abc.TraversableResources):
prefix: str
archive: Incomplete
def __init__(self, loader, module: str) -> None: ...
def open_resource(self, resource: str) -> BufferedReader: ...
def is_resource(self, path: StrPath) -> bool: ...
def files(self): ...
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This obviously returns a zipfile.Path at runtime. Unfortunately, if you add that annotation right now, mypy complains:

stdlib\importlib\readers.pyi:39: error: Return type "Path" of "files" incompatible with return type "Traversable" in supertype "TraversableResources"  [override]

This points to an error either in our zipfile stubs or in our importlib.(resources.)abc stubs. Type checkers should probably understand zipfile.Path as being a subtype of Traversable; I believe that's the intention of the authors of these modules.

Anyway, that looks non-trivial to fix, so I'd rather defer it to another PR for now.


class MultiplexedPath(abc.Traversable):
def __init__(self, *paths: abc.Traversable) -> None: ...
def iterdir(self) -> Iterator[abc.Traversable]: ...
def read_bytes(self) -> NoReturn: ...
def read_text(self, *args: Never, **kwargs: Never) -> NoReturn: ... # type: ignore[override]
def is_dir(self) -> Literal[True]: ...
def is_file(self) -> Literal[False]: ...
if sys.version_info >= (3, 12):
def joinpath(self, *descendants: str) -> abc.Traversable: ...
else:
def joinpath(self, child: str) -> abc.Traversable: ... # type: ignore[override]
__truediv__ = joinpath
def open(self, *args: Never, **kwargs: Never) -> NoReturn: ... # type: ignore[override]
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
def open(self, *args: Never, **kwargs: Never) -> NoReturn: ... # type: ignore[override]
def open(self, *args: Unused, **kwargs: Unused) -> NoReturn: ... # type: ignore[override]

Never would disallow arguably allowed calls like .open("r") (even if those always raise an exception, of course).

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, that was a deliberate choice... I thought maybe it would be useful for type checkers to warn you if you provided arguments to this function. Since it always fails at runtime, it seems like you're probably making a mistake if you're passing arguments to it?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But that should already be caught by the NoReturn.While this is a far-fetched example (using a non-existent typing feature), this disallows things like this:

trav_open: SignatureOf[Traversible.open] = MultiplexedPath.open

I wonder why no type checker complains about using Never, since this breaks Traversible's contract.

That said, I don't feel strongly about this.

Copy link
Member Author

@AlexWaygood AlexWaygood Oct 26, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But that should already be caught by the NoReturn

If we only have the NoReturn return annotation, type checkers will only complain when you call this method if you have mypy's --warn-unreachable setting turned on (or the equivalent setting in other type checkers), and most people don't, unfortunately. Whereas annotating the arguments with Never means you'll get an error from the type checker even if you only have the default settings enabled.

I'll concede that this is a bit of a hack, and that we're really working around the lack of something like python/typing#1043 -- but I think it's useful for the time being

@property
def name(self) -> str: ...

class NamespaceReader(abc.TraversableResources):
path: MultiplexedPath
def __init__(self, namespace_path) -> None: ...
def resource_path(self, resource: str) -> str: ...
def files(self) -> MultiplexedPath: ...
14 changes: 14 additions & 0 deletions stdlib/importlib/resources/readers.pyi
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# On py311+, things are actually defined here
# and re-exported from importlib.readers,
# but doing it this way leads to less code duplication for us

import sys
from collections.abc import Iterable, Iterator
from typing import TypeVar

if sys.version_info >= (3, 11):
from importlib.readers import *

_T = TypeVar("_T")

def remove_duplicates(items: Iterable[_T]) -> Iterator[_T]: ...
3 changes: 0 additions & 3 deletions tests/stubtest_allowlists/py310.txt
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,6 @@ types.GenericAlias.__call__ # Would be complicated to fix properly, Any could s
typing._SpecialForm.__mro_entries__
weakref.ProxyType.__reversed__ # Doesn't really exist

# Modules that exist at runtime, but are missing from typeshed
importlib.readers

# Modules that exist at runtime, but shouldn't be added to typeshed
ctypes.test
ctypes\.test\..+
Expand Down
2 changes: 0 additions & 2 deletions tests/stubtest_allowlists/py311.txt
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@ typing.NewType.__call__
typing.NewType.__mro_entries__

# Modules that exist at runtime, but are missing from typeshed
importlib.readers
importlib.resources.readers
importlib.resources.simple
importlib.simple

Expand Down
2 changes: 0 additions & 2 deletions tests/stubtest_allowlists/py312.txt
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
# Modules that exist at runtime, but are missing from typeshed
zipfile._path.glob
importlib.readers
importlib.resources.readers
importlib.resources.simple
importlib.simple

Expand Down