Merge pull request #10550 from jbylund/joe/cache_requirement_creation

Author: pradyunsg

news/10550.feature.rst

@@ -0,0 +1 @@
+Cache requirement objects, to improve performance reducing reparses of requirement strings.

src/pip/_internal/req/constructors.py

@@ -26,6 +26,7 @@
 from pip._internal.req.req_install import InstallRequirement
 from pip._internal.utils.filetypes import is_archive_file
 from pip._internal.utils.misc import is_installable_dir
+from pip._internal.utils.packaging import get_requirement
 from pip._internal.utils.urls import path_to_url
 from pip._internal.vcs import is_url, vcs
 
@@ -54,7 +55,7 @@ def _strip_extras(path: str) -> Tuple[str, Optional[str]]:
 def convert_extras(extras: Optional[str]) -> Set[str]:
     if not extras:
         return set()
-    return Requirement("placeholder" + extras.lower()).extras
+    return get_requirement("placeholder" + extras.lower()).extras
 
 
 def parse_editable(editable_req: str) -> Tuple[Optional[str], str, Set[str]]:
@@ -83,7 +84,7 @@ def parse_editable(editable_req: str) -> Tuple[Optional[str], str, Set[str]]:
             return (
                 package_name,
                 url_no_extras,
-                Requirement("placeholder" + extras.lower()).extras,
+                get_requirement("placeholder" + extras.lower()).extras,
             )
         else:
             return package_name, url_no_extras, set()
@@ -309,7 +310,7 @@ def with_source(text: str) -> str:
 
     def _parse_req_string(req_as_string: str) -> Requirement:
         try:
-            req = Requirement(req_as_string)
+            req = get_requirement(req_as_string)
         except InvalidRequirement:
             if os.path.sep in req_as_string:
                 add_msg = "It looks like a path."
@@ -386,7 +387,7 @@ def install_req_from_req_string(
     user_supplied: bool = False,
 ) -> InstallRequirement:
     try:
-        req = Requirement(req_string)
+        req = get_requirement(req_string)
     except InvalidRequirement:
         raise InstallationError(f"Invalid requirement: '{req_string}'")
 

src/pip/_internal/resolution/resolvelib/factory.py

@@ -19,7 +19,6 @@
 )
 
 from pip._vendor.packaging.requirements import InvalidRequirement
-from pip._vendor.packaging.requirements import Requirement as PackagingRequirement
 from pip._vendor.packaging.specifiers import SpecifierSet
 from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
 from pip._vendor.resolvelib import ResolutionImpossible
@@ -46,6 +45,7 @@
 from pip._internal.resolution.base import InstallRequirementProvider
 from pip._internal.utils.compatibility_tags import get_supported
 from pip._internal.utils.hashes import Hashes
+from pip._internal.utils.packaging import get_requirement
 from pip._internal.utils.virtualenv import running_under_virtualenv
 
 from .base import Candidate, CandidateVersion, Constraint, Requirement
@@ -365,7 +365,7 @@ def find_candidates(
         # If the current identifier contains extras, add explicit candidates
         # from entries from extra-less identifier.
         with contextlib.suppress(InvalidRequirement):
-            parsed_requirement = PackagingRequirement(identifier)
+            parsed_requirement = get_requirement(identifier)
             explicit_candidates.update(
                 self._iter_explicit_candidates_from_base(
                     requirements.get(parsed_requirement.name, ()),

src/pip/_internal/utils/packaging.py

@@ -1,10 +1,12 @@
+import functools
 import logging
 from email.message import Message
 from email.parser import FeedParser
 from typing import Optional, Tuple
 
 from pip._vendor import pkg_resources
 from pip._vendor.packaging import specifiers, version
+from pip._vendor.packaging.requirements import Requirement
 from pip._vendor.pkg_resources import Distribution
 
 from pip._internal.exceptions import NoneMetadataError
@@ -69,3 +71,14 @@ def get_installer(dist: Distribution) -> str:
             if line.strip():
                 return line.strip()
     return ""
+
+
+@functools.lru_cache(maxsize=512)
+def get_requirement(req_string: str) -> Requirement:
+    """Construct a packaging.Requirement object with caching"""
+    # Parsing requirement strings is expensive, and is also expected to happen
+    # with a low diversity of different arguments (at least relative the number
+    # constructed). This method adds a cache to requirement object creation to
+    # minimize repeated parsing of the same string to construct equivalent
+    # Requirement objects.
+    return Requirement(req_string)

tests/unit/test_packaging.py

@@ -2,8 +2,9 @@
 
 import pytest
 from pip._vendor.packaging import specifiers
+from pip._vendor.packaging.requirements import Requirement
 
-from pip._internal.utils.packaging import check_requires_python
+from pip._internal.utils.packaging import check_requires_python, get_requirement
 
 
 @pytest.mark.parametrize(
@@ -27,3 +28,17 @@ def test_check_requires_python__invalid() -> None:
     """
     with pytest.raises(specifiers.InvalidSpecifier):
         check_requires_python("invalid", (3, 6, 5))
+
+
+def test_get_or_create_caching() -> None:
+    """test caching of get_or_create requirement"""
+    teststr = "affinegap==1.10"
+    from_helper = get_requirement(teststr)
+    freshly_made = Requirement(teststr)
+
+    # Requirement doesn't have an equality operator (yet) so test
+    # equality of attribute for list of attributes
+    for iattr in ["name", "url", "extras", "specifier", "marker"]:
+        assert getattr(from_helper, iattr) == getattr(freshly_made, iattr)
+    assert get_requirement(teststr) is not Requirement(teststr)
+    assert get_requirement(teststr) is get_requirement(teststr)