Introduce pip.cache.Cache base class for pip's caches (#4623)

Author: pradyunsg

pip/cache.py

@@ -16,34 +16,28 @@
 logger = logging.getLogger(__name__)
 
 
-class WheelCache(object):
-    """A cache of wheels for future installs."""
+class Cache(object):
+    """An abstract class - provides cache directories for data from links
 
-    def __init__(self, cache_dir, format_control):
-        """Create a wheel cache.
 
         :param cache_dir: The root of the cache.
         :param format_control: A pip.index.FormatControl object to limit
             binaries being read from the cache.
-        """
-        self._cache_dir = expanduser(cache_dir) if cache_dir else None
-        self._format_control = format_control
-
-    def get_cache_path_for_link(self, link):
-        """
-        Return a directory to store cached wheels in for link.
+        :param allowed_formats: which formats of files the cache should store.
+            ('binary' and 'source' are the only allowed values)
+    """
 
-        Because there are M wheels for any one sdist, we provide a directory
-        to cache them in, and then consult that directory when looking up
-        cache hits.
+    def __init__(self, cache_dir, format_control, allowed_formats):
+        super(Cache, self).__init__()
+        self.cache_dir = expanduser(cache_dir) if cache_dir else None
+        self.format_control = format_control
+        self.allowed_formats = allowed_formats
 
-        We only insert things into the cache if they have plausible version
-        numbers, so that we don't contaminate the cache with things that were
-        not unique. E.g. ./package might have dozens of installs done for it
-        and build a version of 0.0...and if we built and cached a wheel, we'd
-        end up using the same wheel even if the source has been edited.
+        _valid_formats = {"source", "binary"}
+        assert self.allowed_formats.union(_valid_formats) == _valid_formats
 
-        :param link: The link of the sdist for which this will cache wheels.
+    def _get_cache_path_parts(self, link):
+        """Get parts of part that must be os.path.joined with cache_dir
         """
 
         # We want to generate an url to use as our cache key, we don't want to
@@ -65,37 +59,82 @@ def get_cache_path_for_link(self, link):
         # FS.
         parts = [hashed[:2], hashed[2:4], hashed[4:6], hashed[6:]]
 
-        # Inside of the base location for cached wheels, expand our parts and
-        # join them all together.
-        return os.path.join(self._cache_dir, "wheels", *parts)
-
-    def cached_wheel(self, link, package_name):
-        not_cached = (
-            not self._cache_dir or
-            not link or
-            link.is_wheel or
-            not link.is_artifact or
-            not package_name
-        )
+        return parts
 
-        if not_cached:
-            return link
+    def _get_candidates(self, link, package_name):
+        can_not_cache = (
+            not self.cache_dir or
+            not package_name or
+            not link
+        )
+        if can_not_cache:
+            return []
 
         canonical_name = canonicalize_name(package_name)
         formats = pip.index.fmt_ctl_formats(
-            self._format_control, canonical_name
+            self.format_control, canonical_name
         )
-        if "binary" not in formats:
-            return link
-        root = self.get_cache_path_for_link(link)
+        if not self.allowed_formats.intersection(formats):
+            return []
+
+        root = self.get_path_for_link(link)
         try:
-            wheel_names = os.listdir(root)
+            return os.listdir(root)
         except OSError as err:
             if err.errno in {errno.ENOENT, errno.ENOTDIR}:
-                return link
+                return []
             raise
+
+    def get_path_for_link(self, link):
+        """Return a directory to store cached items in for link.
+        """
+        raise NotImplementedError()
+
+    def get(self, link, package_name):
+        """Returns a link to a cached item if it exists, otherwise returns the
+        passed link.
+        """
+        raise NotImplementedError()
+
+    def _link_for_candidate(self, link, candidate):
+        root = self.get_path_for_link(link)
+        path = os.path.join(root, candidate)
+
+        return pip.index.Link(path_to_url(path))
+
+
+class WheelCache(Cache):
+    """A cache of wheels for future installs.
+    """
+
+    def __init__(self, cache_dir, format_control):
+        super(WheelCache, self).__init__(cache_dir, format_control, {"binary"})
+
+    def get_path_for_link(self, link):
+        """Return a directory to store cached wheels for link
+
+        Because there are M wheels for any one sdist, we provide a directory
+        to cache them in, and then consult that directory when looking up
+        cache hits.
+
+        We only insert things into the cache if they have plausible version
+        numbers, so that we don't contaminate the cache with things that were
+        not unique. E.g. ./package might have dozens of installs done for it
+        and build a version of 0.0...and if we built and cached a wheel, we'd
+        end up using the same wheel even if the source has been edited.
+
+        :param link: The link of the sdist for which this will cache wheels.
+        """
+        parts = self._get_cache_path_parts(link)
+
+        # Inside of the base location for cached wheels, expand our parts and
+        # join them all together.
+        return os.path.join(self.cache_dir, "wheels", *parts)
+
+    def get(self, link, package_name):
         candidates = []
-        for wheel_name in wheel_names:
+
+        for wheel_name in self._get_candidates(link, package_name):
             try:
                 wheel = Wheel(wheel_name)
             except InvalidWheelFilename:
@@ -104,8 +143,8 @@ def cached_wheel(self, link, package_name):
                 # Built for a different python/arch/etc
                 continue
             candidates.append((wheel.support_index_min(), wheel_name))
+
         if not candidates:
             return link
-        candidates.sort()
-        path = os.path.join(root, candidates[0][1])
-        return pip.index.Link(path_to_url(path))
+
+        return self._link_for_candidate(link, min(candidates)[1])

pip/req/req_install.py

@@ -298,7 +298,7 @@ def populate_link(self, finder, upgrade, require_hashes):
             self.link = finder.find_requirement(self, upgrade)
         if self._wheel_cache is not None and not require_hashes:
             old_link = self.link
-            self.link = self._wheel_cache.cached_wheel(self.link, self.name)
+            self.link = self._wheel_cache.get(self.link, self.name)
             if old_link != self.link:
                 logger.debug('Using cached wheel link: %s', self.link)
 

pip/wheel.py

@@ -727,7 +727,7 @@ def build(self, session, autobuilding=False):
         :return: True if all the wheels built correctly.
         """
         building_is_possible = self._wheel_dir or (
-            autobuilding and self.wheel_cache._cache_dir
+            autobuilding and self.wheel_cache.cache_dir
         )
         assert building_is_possible
 
@@ -778,9 +778,7 @@ def build(self, session, autobuilding=False):
                 python_tag = None
                 if autobuilding:
                     python_tag = pep425tags.implementation_tag
-                    output_dir = self.wheel_cache.get_cache_path_for_link(
-                        req.link
-                    )
+                    output_dir = self.wheel_cache.get_path_for_link(req.link)
                     try:
                         ensure_dir(output_dir)
                     except OSError as e:

tests/unit/test_cache.py

@@ -6,8 +6,8 @@ class TestWheelCache:
 
     def test_expands_path(self):
         wc = WheelCache("~/.foo/", None)
-        assert wc._cache_dir == expanduser("~/.foo/")
+        assert wc.cache_dir == expanduser("~/.foo/")
 
     def test_falsey_path_none(self):
         wc = WheelCache(False, None)
-        assert wc._cache_dir is None
+        assert wc.cache_dir is None