Automatic vendoring (#4093)

Author: xavfernandez

pip/_vendor/README.rst

@@ -15,7 +15,8 @@ Policy
   pure Python.
 
 * Any modifications made to libraries **MUST** be noted in
-  ``pip/_vendor/README.rst``.
+  ``pip/_vendor/README.rst`` and their corresponding patches **MUST** be
+  included ``tasks/vendoring/patches``.
 
 
 Rationale
@@ -92,23 +93,28 @@ situation that we expect pip to be used and not mandate some external mechanism
 such as OS packages.
 
 
-pkg_resources
--------------
-
-pkg_resources has been pulled in from setuptools 28.8.0
-
-
 Modifications
 -------------
 
 * html5lib has been modified to import six from pip._vendor
+* setuptools is completely stripped to only keep pkg_resources
 * pkg_resources has been modified to import its externs from pip._vendor
 * CacheControl has been modified to import its dependencies from pip._vendor
 * packaging has been modified to import its dependencies from pip._vendor
 * requests has been modified *not* to optionally load any C dependencies.
 * Modified distro to delay importing argparse to avoid errors on 2.6
 
 
+Automatic Vendoring
+-------------------
+
+Vendoring is automated via the ``vendoring.update`` task (defined in
+``tasks/vendoring/__init__.py``) from the content of
+``pip/_vendor/vendor.txt`` and the different patches in
+``tasks/vendoring/patches/``.
+Launch it via ``invoke vendoring.update`` (you will need ``invoke>=0.13.0``).
+
+
 Debundling
 ----------
 

pip/_vendor/re-vendor.py

@@ -13,4 +13,5 @@ ipaddress==1.0.17  # Only needed on 2.6 and 2.7
 packaging==16.8
 pyparsing==2.1.10
 retrying==1.3.3
+setuptools==28.8.0
 webencodings==0.5

pip/_vendor/vendor.txt

@@ -1,5 +1,6 @@
 import invoke
 
 from . import generate
+from . import vendoring
 
-ns = invoke.Collection(generate)
+ns = invoke.Collection(generate, vendoring)

tasks/__init__.py

@@ -4,12 +4,12 @@
 
 
 @invoke.task
-def authors():
+def authors(ctx):
     print("[generate.authors] Generating AUTHORS")
 
     # Get our list of authors
     print("[generate.authors] Collecting author names")
-    r = invoke.run("git log --use-mailmap --format'=%aN <%aE>'", hide=True)
+    r = ctx.run("git log --use-mailmap --format'=%aN <%aE>'", hide=True)
     authors = []
     seen_authors = set()
     for author in r.stdout.splitlines():

tasks/generate.py

@@ -0,0 +1,129 @@
+""""Vendoring script, python 3.5 needed"""
+
+from pathlib import Path
+import re
+import shutil
+
+import invoke
+
+TASK_NAME = 'update'
+
+FILE_WHITE_LIST = (
+    'Makefile',
+    'vendor.txt',
+    '__init__.py',
+    'README.rst',
+)
+
+
+def drop_dir(path):
+    shutil.rmtree(str(path))
+
+
+def remove_all(paths):
+    for path in paths:
+        if path.is_dir():
+            drop_dir(path)
+        else:
+            path.unlink()
+
+
+def log(msg):
+    print('[vendoring.%s] %s' % (TASK_NAME, msg))
+
+
+def clean_vendor(ctx, vendor_dir):
+    # Old _vendor cleanup
+    remove_all(vendor_dir.glob('*.pyc'))
+    log('Cleaning %s' % vendor_dir)
+    for item in vendor_dir.iterdir():
+        if item.is_dir():
+            shutil.rmtree(str(item))
+        elif item.name not in FILE_WHITE_LIST:
+            item.unlink()
+        else:
+            log('Skipping %s' % item)
+
+
+def rewrite_imports(package_dir, vendored_libs):
+    for item in package_dir.iterdir():
+        if item.is_dir():
+            rewrite_imports(item, vendored_libs)
+        elif item.name.endswith('.py'):
+            rewrite_file_imports(item, vendored_libs)
+
+
+def rewrite_file_imports(item, vendored_libs):
+    """Rewrite 'import xxx' and 'from xxx import' for vendored_libs"""
+    text = item.read_text()
+    # Revendor pkg_resources.extern first
+    text = re.sub(r'pkg_resources.extern', r'pip._vendor', text)
+    for lib in vendored_libs:
+        text = re.sub(
+            r'(\n\s*)import %s' % lib,
+            r'\1from pip._vendor import %s' % lib,
+            text,
+        )
+        text = re.sub(
+            r'(\n\s*)from %s' % lib,
+            r'\1from pip._vendor.%s' % lib,
+            text,
+        )
+    item.write_text(text)
+
+
+def apply_patch(ctx, patch_file_path):
+    log('Applying patch %s' % patch_file_path.name)
+    ctx.run('git apply %s' % patch_file_path)
+
+
+def vendor(ctx, vendor_dir):
+    log('Reinstalling vendored libraries')
+    ctx.run(
+        'pip install -t {0} -r {0}/vendor.txt --no-compile'.format(
+            str(vendor_dir),
+        )
+    )
+    remove_all(vendor_dir.glob('*.dist-info'))
+    remove_all(vendor_dir.glob('*.egg-info'))
+
+    # Cleanup setuptools unneeded parts
+    (vendor_dir / 'easy_install.py').unlink()
+    drop_dir(vendor_dir / 'setuptools')
+    drop_dir(vendor_dir / 'pkg_resources' / '_vendor')
+    drop_dir(vendor_dir / 'pkg_resources' / 'extern')
+
+    # Detect the vendored packages/modules
+    vendored_libs = []
+    for item in vendor_dir.iterdir():
+        if item.is_dir():
+            vendored_libs.append(item.name)
+        elif item.name not in FILE_WHITE_LIST:
+            vendored_libs.append(item.name[:-3])
+    log("Detected vendored libraries: %s" % ", ".join(vendored_libs))
+
+    # Global import rewrites
+    log("Rewriting all imports related to vendored libs")
+    for item in vendor_dir.iterdir():
+        if item.is_dir():
+            rewrite_imports(item, vendored_libs)
+        elif item.name not in FILE_WHITE_LIST:
+            rewrite_file_imports(item, vendored_libs)
+
+    # Special cases: apply stored patches
+    log("Apply patches")
+    patch_dir = Path(__file__).parent / 'patches'
+    for patch in patch_dir.glob('*.patch'):
+        apply_patch(ctx, patch)
+
+
+@invoke.task(name=TASK_NAME)
+def main(ctx):
+    git_root = Path(
+        ctx.run('git rev-parse --show-toplevel', hide=True).stdout.strip()
+    )
+    vendor_dir = git_root / 'pip' / '_vendor'
+    log('Using vendor dir: %s' % vendor_dir)
+    clean_vendor(ctx, vendor_dir)
+    vendor(ctx, vendor_dir)
+    log('Revendoring complete')

tasks/vendoring/__init__.py

@@ -0,0 +1,24 @@
+diff --git a/pip/_vendor/cachecontrol/compat.py b/pip/_vendor/cachecontrol/compat.py
+index 54ec211..018e6ac 100644
+--- a/pip/_vendor/cachecontrol/compat.py
++++ b/pip/_vendor/cachecontrol/compat.py
+@@ -10,17 +10,8 @@ except ImportError:
+     import pickle
+ 
+ 
+-# Handle the case where the requests module has been patched to not have
+-# urllib3 bundled as part of its source.
+-try:
+-    from pip._vendor.requests.packages.urllib3.response import HTTPResponse
+-except ImportError:
+-    from urllib3.response import HTTPResponse
+-
+-try:
+-    from pip._vendor.requests.packages.urllib3.util import is_fp_closed
+-except ImportError:
+-    from urllib3.util import is_fp_closed
++from pip._vendor.requests.packages.urllib3.response import HTTPResponse
++from pip._vendor.requests.packages.urllib3.util import is_fp_closed
+ 
+ # Replicate some six behaviour
+ try:

tasks/vendoring/patches/cachecontrol.patch

@@ -0,0 +1,21 @@
+diff --git a/pip/_vendor/distro.py b/pip/_vendor/distro.py
+index 0afa527..9e7daad 100644
+--- a/pip/_vendor/distro.py
++++ b/pip/_vendor/distro.py
+@@ -34,7 +34,6 @@ import sys
+ import json
+ import shlex
+ import logging
+-import argparse
+ import subprocess
+ 
+ 
+@@ -1052,6 +1051,8 @@ _distro = LinuxDistribution()
+ 
+ 
+ def main():
++    import argparse
++
+     logger = logging.getLogger(__name__)
+     logger.setLevel(logging.DEBUG)
+     logger.addHandler(logging.StreamHandler(sys.stdout))

tasks/vendoring/patches/distro.patch

@@ -0,0 +1,48 @@
+diff --git a/pip/_vendor/requests/__init__.py b/pip/_vendor/requests/__init__.py
+index 9c3b769..44f6836 100644
+--- a/pip/_vendor/requests/__init__.py
++++ b/pip/_vendor/requests/__init__.py
+@@ -48,11 +48,13 @@ __license__ = 'Apache 2.0'
+ __copyright__ = 'Copyright 2016 Kenneth Reitz'
+ 
+ # Attempt to enable urllib3's SNI support, if possible
+-try:
+-    from .packages.urllib3.contrib import pyopenssl
+-    pyopenssl.inject_into_urllib3()
+-except ImportError:
+-    pass
++# Note: Patched by pip to prevent using the PyOpenSSL module. On Windows this
++#       prevents upgrading cryptography.
++# try:
++#     from .packages.urllib3.contrib import pyopenssl
++#     pyopenssl.inject_into_urllib3()
++# except ImportError:
++#     pass
+ 
+ import warnings
+ 
+diff --git a/pip/_vendor/requests/compat.py b/pip/_vendor/requests/compat.py
+index eb6530d..353ec29 100644
+--- a/pip/_vendor/requests/compat.py
++++ b/pip/_vendor/requests/compat.py
+@@ -25,12 +25,14 @@ is_py2 = (_ver[0] == 2)
+ #: Python 3.x?
+ is_py3 = (_ver[0] == 3)
+ 
+-try:
+-    import simplejson as json
+-except (ImportError, SyntaxError):
+-    # simplejson does not support Python 3.2, it throws a SyntaxError
+-    # because of u'...' Unicode literals.
+-    import json
++# Note: We've patched out simplejson support in pip because it prevents
++#       upgrading simplejson on Windows.
++# try:
++#     import simplejson as json
++# except (ImportError, SyntaxError):
++#     # simplejson does not support Python 3.2, it throws a SyntaxError
++#     # because of u'...' Unicode literals.
++import json
+ 
+ # ---------
+ # Specifics