diff --git a/news/8588.feature b/news/8588.feature new file mode 100644 index 00000000000..273715bb009 --- /dev/null +++ b/news/8588.feature @@ -0,0 +1,3 @@ +Allow the new resolver to obtain dependency information through wheels +lazily downloaded using HTTP range requests. To enable this feature, +invoke ``pip`` with ``--use-feature=fast-deps``. diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index b2f63bb264e..ed42c5f5ae7 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -913,7 +913,7 @@ def check_list_path_option(options): metavar='feature', action='append', default=[], - choices=['2020-resolver'], + choices=['2020-resolver', 'fast-deps'], help='Enable new functionality, that may be backward incompatible.', ) # type: Callable[..., Option] diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index 50a60c8ed5a..78b5ce6a141 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -271,6 +271,7 @@ def make_resolver( force_reinstall=force_reinstall, upgrade_strategy=upgrade_strategy, py_version_info=py_version_info, + lazy_wheel='fast-deps' in options.features_enabled, ) import pip._internal.resolution.legacy.resolver return pip._internal.resolution.legacy.resolver.Resolver( diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index 1ee46430292..c289bb5839c 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -1,16 +1,22 @@ import logging import sys +from pip._vendor.contextlib2 import suppress from pip._vendor.packaging.specifiers import InvalidSpecifier, SpecifierSet from pip._vendor.packaging.utils import canonicalize_name from pip._vendor.packaging.version import Version from pip._internal.exceptions import HashError, MetadataInconsistent +from pip._internal.network.lazy_wheel import ( + HTTPRangeRequestUnsupported, + dist_from_wheel_url, +) from pip._internal.req.constructors import ( install_req_from_editable, install_req_from_line, ) from pip._internal.req.req_install import InstallRequirement +from pip._internal.utils.logging import indent_log from pip._internal.utils.misc import dist_is_editable, normalize_version_info from pip._internal.utils.packaging import get_requires_python from pip._internal.utils.typing import MYPY_CHECK_RUNNING @@ -142,6 +148,7 @@ def __init__( self._name = name self._version = version self._dist = None # type: Optional[Distribution] + self._prepared = False def __repr__(self): # type: () -> str @@ -197,11 +204,23 @@ def _prepare_abstract_distribution(self): # type: () -> AbstractDistribution raise NotImplementedError("Override in subclass") + def _check_metadata_consistency(self): + # type: () -> None + """Check for consistency of project name and version of dist.""" + # TODO: (Longer term) Rather than abort, reject this candidate + # and backtrack. This would need resolvelib support. + dist = self._dist # type: Distribution + name = canonicalize_name(dist.project_name) + if self._name is not None and self._name != name: + raise MetadataInconsistent(self._ireq, "name", dist.project_name) + version = dist.parsed_version + if self._version is not None and self._version != version: + raise MetadataInconsistent(self._ireq, "version", dist.version) + def _prepare(self): # type: () -> None - if self._dist is not None: + if self._prepared: return - try: abstract_dist = self._prepare_abstract_distribution() except HashError as e: @@ -210,24 +229,36 @@ def _prepare(self): self._dist = abstract_dist.get_pkg_resources_distribution() assert self._dist is not None, "Distribution already installed" + self._check_metadata_consistency() + self._prepared = True - # TODO: (Longer term) Rather than abort, reject this candidate - # and backtrack. This would need resolvelib support. - name = canonicalize_name(self._dist.project_name) - if self._name is not None and self._name != name: - raise MetadataInconsistent( - self._ireq, "name", self._dist.project_name, - ) - version = self._dist.parsed_version - if self._version is not None and self._version != version: - raise MetadataInconsistent( - self._ireq, "version", self._dist.version, - ) + def _fetch_metadata(self): + # type: () -> None + """Fetch metadata, using lazy wheel if possible.""" + preparer = self._factory.preparer + use_lazy_wheel = self._factory.use_lazy_wheel + remote_wheel = self._link.is_wheel and not self._link.is_file + if use_lazy_wheel and remote_wheel and not preparer.require_hashes: + assert self._name is not None + logger.info('Collecting %s', self._ireq.req or self._ireq) + # If HTTPRangeRequestUnsupported is raised, fallback silently. + with indent_log(), suppress(HTTPRangeRequestUnsupported): + logger.info( + 'Obtaining dependency information from %s %s', + self._name, self._version, + ) + url = self._link.url.split('#', 1)[0] + session = preparer.downloader._session + self._dist = dist_from_wheel_url(self._name, url, session) + self._check_metadata_consistency() + if self._dist is None: + self._prepare() @property def dist(self): # type: () -> Distribution - self._prepare() + if self._dist is None: + self._fetch_metadata() return self._dist def _get_requires_python_specifier(self): diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 0d73ff978ef..d10e2eb0009 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -82,6 +82,7 @@ def __init__( ignore_installed, # type: bool ignore_requires_python, # type: bool py_version_info=None, # type: Optional[Tuple[int, ...]] + lazy_wheel=False, # type: bool ): # type: (...) -> None self._finder = finder @@ -92,6 +93,7 @@ def __init__( self._use_user_site = use_user_site self._force_reinstall = force_reinstall self._ignore_requires_python = ignore_requires_python + self.use_lazy_wheel = lazy_wheel self._link_candidate_cache = {} # type: Cache[LinkCandidate] self._editable_candidate_cache = {} # type: Cache[EditableCandidate] diff --git a/src/pip/_internal/resolution/resolvelib/resolver.py b/src/pip/_internal/resolution/resolvelib/resolver.py index d2ac9d0418a..43ea248632d 100644 --- a/src/pip/_internal/resolution/resolvelib/resolver.py +++ b/src/pip/_internal/resolution/resolvelib/resolver.py @@ -49,8 +49,16 @@ def __init__( force_reinstall, # type: bool upgrade_strategy, # type: str py_version_info=None, # type: Optional[Tuple[int, ...]] + lazy_wheel=False, # type: bool ): super(Resolver, self).__init__() + if lazy_wheel: + logger.warning( + 'pip is using lazily downloaded wheels using HTTP ' + 'range requests to obtain dependency information. ' + 'This experimental feature is enabled through ' + '--use-feature=fast-deps and it is not ready for production.' + ) assert upgrade_strategy in self._allowed_strategies @@ -64,6 +72,7 @@ def __init__( ignore_installed=ignore_installed, ignore_requires_python=ignore_requires_python, py_version_info=py_version_info, + lazy_wheel=lazy_wheel, ) self.ignore_dependencies = ignore_dependencies self.upgrade_strategy = upgrade_strategy diff --git a/tests/data/packages/requiresPaste/pyproject.toml b/tests/data/packages/requiresPaste/pyproject.toml new file mode 100644 index 00000000000..7ca1bcc79c9 --- /dev/null +++ b/tests/data/packages/requiresPaste/pyproject.toml @@ -0,0 +1,9 @@ +[build-system] +requires = ['flit_core >=2,<4'] +build-backend = 'flit_core.buildapi' + +[tool.flit.metadata] +module = 'requiresPaste' +author = 'A. Random Developer' +author-email = 'author@example.com' +requires = ['Paste==3.4.2'] diff --git a/tests/data/packages/requiresPaste/requiresPaste.py b/tests/data/packages/requiresPaste/requiresPaste.py new file mode 100644 index 00000000000..c74209e44fe --- /dev/null +++ b/tests/data/packages/requiresPaste/requiresPaste.py @@ -0,0 +1,3 @@ +"""Module requiring Paste to test dependencies download of pip wheel.""" + +__version__ = '3.1.4' diff --git a/tests/functional/test_fast_deps.py b/tests/functional/test_fast_deps.py new file mode 100644 index 00000000000..b41055c5606 --- /dev/null +++ b/tests/functional/test_fast_deps.py @@ -0,0 +1,50 @@ +import fnmatch +import json +from os.path import basename + +from pip._vendor.packaging.utils import canonicalize_name +from pytest import mark + + +def pip(script, command, requirement): + return script.pip( + command, '--prefer-binary', '--no-cache-dir', + '--use-feature=fast-deps', requirement, + allow_stderr_warning=True, + ) + + +def assert_installed(script, names): + list_output = json.loads(script.pip('list', '--format=json').stdout) + installed = {canonicalize_name(item['name']) for item in list_output} + assert installed.issuperset(map(canonicalize_name, names)) + + +@mark.network +@mark.parametrize(('requirement', 'expected'), ( + ('Paste==3.4.2', ('Paste', 'six')), + ('Paste[flup]==3.4.2', ('Paste', 'six', 'flup')), +)) +def test_install_from_pypi(requirement, expected, script): + pip(script, 'install', requirement) + assert_installed(script, expected) + + +@mark.network +@mark.parametrize(('requirement', 'expected'), ( + ('Paste==3.4.2', ('Paste-3.4.2-*.whl', 'six-*.whl')), + ('Paste[flup]==3.4.2', ('Paste-3.4.2-*.whl', 'six-*.whl', 'flup-*')), +)) +def test_download_from_pypi(requirement, expected, script): + result = pip(script, 'download', requirement) + created = list(map(basename, result.files_created)) + assert all(fnmatch.filter(created, f) for f in expected) + + +@mark.network +def test_build_wheel_with_deps(data, script): + result = pip(script, 'wheel', data.packages/'requiresPaste') + created = list(map(basename, result.files_created)) + assert fnmatch.filter(created, 'requiresPaste-3.1.4-*.whl') + assert fnmatch.filter(created, 'Paste-3.4.2-*.whl') + assert fnmatch.filter(created, 'six-*.whl')