unix: install pip and setuptools from wheels

indygreg · indygreg · commit cd2a0ff18b56 · 2021-09-21T19:24:01.000-07:00
This commit refactors the install of pip and setuptools. Previously, we installed both of these by invoking `setup.py` from the host Python. This worked, but it was somewhat old school. And in the case of cross-compiling, it installed the packages to the host Python, not the target Python. After this commit, we download wheels for both pip and setuptools and the install invokes the host Python but runs the pip wheel to invoke pip to install the packages. This is conceptually similar to how the `ensurepip` module works. The end result is that pip is used to install itself and setuptools into the appropriate output directory. This works with both native and cross builds. Aside from minor changes to the directory layout, the end result is the same as far as I can tell. But since cross-compiles now install pip and setuptools correctly, this fixes #84. Because pip now bootstraps self and this doesn't work on musl builds without a patch, we had to move the patching of pip to before it is invoked. We moved the patching of setuptools as well, because it is related. Windows builds were updated to use the wheel based pip and setuptools. However, we don't support cross-compiling on Windows yet, so we continue to use the built python.exe to install these wheels.
diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh
@@ -20,8 +20,74 @@ export LLVM_PROFDATA=${TOOLS_PATH}/${TOOLCHAIN}/bin/llvm-profdata
 find ${TOOLS_PATH}/deps -name '*.so*' -exec rm {} \;
 
 tar -xf Python-${PYTHON_VERSION}.tar.xz
-tar -xf setuptools-${SETUPTOOLS_VERSION}.tar.gz
-tar -xf pip-${PIP_VERSION}.tar.gz
+
+PIP_WHEEL="${ROOT}/pip-${PIP_VERSION}-py3-none-any.whl"
+SETUPTOOLS_WHEEL="${ROOT}/setuptools-${SETUPTOOLS_VERSION}-py3-none-any.whl"
+
+chmod 644 "${PIP_WHEEL}"
+chmod 644 "${SETUPTOOLS_WHEEL}"
+
+# pip and setuptools don't properly handle the case where the current executable
+# isn't dynamic. This is tracked by https://github.com/pypa/pip/issues/6543.
+# We need to patch both.
+#
+# Ideally we'd do this later in the build. However, since we use the pip
+# wheel to bootstrap itself, we need to patch the wheel before it is used.
+#
+# Wheels are zip files. So we simply unzip, patch, and rezip.
+mkdir pip-tmp
+pushd pip-tmp
+unzip "${PIP_WHEEL}"
+
+patch -p1 <<EOF
+diff --git a/pip/_internal/utils/glibc.py b/pip/_internal/utils/glibc.py
+index 819979d80..4ae91e364 100644
+--- a/pip/_internal/utils/glibc.py
++++ b/pip/_internal/utils/glibc.py
+@@ -47,7 +47,10 @@ def glibc_version_string_ctypes():
+     # manpage says, "If filename is NULL, then the returned handle is for the
+     # main program". This way we can let the linker do the work to figure out
+     # which libc our process is actually using.
+-    process_namespace = ctypes.CDLL(None)
++    try:
++        process_namespace = ctypes.CDLL(None)
++    except OSError:
++        return None
+     try:
+         gnu_get_libc_version = process_namespace.gnu_get_libc_version
+     except AttributeError:
+EOF
+
+zip -r "${PIP_WHEEL}" *
+popd
+rm -rf pip-tmp
+
+mkdir setuptools-tmp
+pushd setuptools-tmp
+unzip "${SETUPTOOLS_WHEEL}"
+
+patch -p1 <<EOF
+diff --git a/setuptools/_vendor/packaging/tags.py b/setuptools/_vendor/packaging/tags.py
+index 9064910b..c541e648 100644
+--- a/setuptools/_vendor/packaging/tags.py
++++ b/setuptools/_vendor/packaging/tags.py
+@@ -475,7 +475,10 @@ def _glibc_version_string_ctypes():
+     # which libc our process is actually using.
+     #
+     # Note: typeshed is wrong here so we are ignoring this line.
+-    process_namespace = ctypes.CDLL(None)  # type: ignore
++    try:
++        process_namespace = ctypes.CDLL(None)  # type: ignore
++    except OSError:
++        return None
+     try:
+         gnu_get_libc_version = process_namespace.gnu_get_libc_version
+     except AttributeError:
+EOF
+
+zip -r "${SETUPTOOLS_WHEEL}" *
+popd
+rm -rf setuptools-tmp
 
 # If we are cross-compiling, we need to build a host Python to use during
 # the build.
@@ -50,7 +116,9 @@ if [ "${BUILD_TRIPLE}" != "${TARGET_TRIPLE}" ]; then
       ;;
   esac
 
-  CC="${HOST_CC}" CFLAGS="${EXTRA_HOST_CFLAGS}" CPPFLAGS="${EXTRA_HOST_CFLAGS}" LDFLAGS="${EXTRA_HOST_LDFLAGS}" ./configure --prefix "${TOOLS_PATH}/pyhost"
+  CC="${HOST_CC}" CFLAGS="${EXTRA_HOST_CFLAGS}" CPPFLAGS="${EXTRA_HOST_CFLAGS}" LDFLAGS="${EXTRA_HOST_LDFLAGS}" ./configure \
+    --prefix "${TOOLS_PATH}/pyhost" \
+    --without-ensurepip
 
   make -j "${NUM_CPUS}" install
 
@@ -770,63 +838,17 @@ if [ "${PYBUILD_SHARED}" = "1" ]; then
     fi
 fi
 
-# Install pip so we can patch it to work with non-dynamic executables
-# and work around https://github.com/pypa/pip/issues/6543. But pip's bundled
-# setuptools has the same bug! So we need to install a patched version.
-pushd ${ROOT}/setuptools-${SETUPTOOLS_VERSION}
-patch -p1 <<EOF
-diff --git a/setuptools/_vendor/packaging/tags.py b/setuptools/_vendor/packaging/tags.py
-index 9064910b..c541e648 100644
---- a/setuptools/_vendor/packaging/tags.py
-+++ b/setuptools/_vendor/packaging/tags.py
-@@ -475,7 +475,10 @@ def _glibc_version_string_ctypes():
-     # which libc our process is actually using.
-     #
-     # Note: typeshed is wrong here so we are ignoring this line.
--    process_namespace = ctypes.CDLL(None)  # type: ignore
-+    try:
-+        process_namespace = ctypes.CDLL(None)  # type: ignore
-+    except OSError:
-+        return None
-     try:
-         gnu_get_libc_version = process_namespace.gnu_get_libc_version
-     except AttributeError:
-EOF
-
-${BUILD_PYTHON} setup.py install
-popd
-
-pushd ${ROOT}/pip-${PIP_VERSION}
-
-# pip 21 shipped DOS line endings. https://github.com/pypa/pip/issues/9638.
-# Let's fix that.
-if [ "${PYBUILD_PLATFORM}" = "macos" ]; then
-    find . -name '*.py' -exec perl -i -pe 's/\r\n$/\n/g' {} \;
-else
-    find . -name '*.py' -exec sed -i 's/\r$//g' {} \;
-fi
-
-patch -p1 <<EOF
-diff --git a/src/pip/_internal/utils/glibc.py b/src/pip/_internal/utils/glibc.py
-index 819979d80..4ae91e364 100644
---- a/src/pip/_internal/utils/glibc.py
-+++ b/src/pip/_internal/utils/glibc.py
-@@ -47,7 +47,10 @@ def glibc_version_string_ctypes():
-     # manpage says, "If filename is NULL, then the returned handle is for the
-     # main program". This way we can let the linker do the work to figure out
-     # which libc our process is actually using.
--    process_namespace = ctypes.CDLL(None)
-+    try:
-+        process_namespace = ctypes.CDLL(None)
-+    except OSError:
-+        return None
-     try:
-         gnu_get_libc_version = process_namespace.gnu_get_libc_version
-     except AttributeError:
-EOF
+# Install setuptools and pip as they are common tools that should be in any
+# Python distribution.
+#
+# We disabled ensurepip because we insist on providing our own pip and don't
+# want the final product to possibly be contaminated by another version.
+#
+# It is possible for the Python interpreter to run wheels directly. So we
+# simply use our pip to install self. Kinda crazy, but it works!
 
-${BUILD_PYTHON} setup.py install
-popd
+${BUILD_PYTHON} "${PIP_WHEEL}/pip" install --prefix="${ROOT}/out/python/install" --no-cache-dir --no-index "${PIP_WHEEL}"
+${BUILD_PYTHON} "${PIP_WHEEL}/pip" install --prefix="${ROOT}/out/python/install" --no-cache-dir --no-index "${SETUPTOOLS_WHEEL}"
 
 # Emit metadata to be used in PYTHON.json.
 cat > ${ROOT}/generate_metadata.py << EOF
diff --git a/cpython-unix/build.Dockerfile b/cpython-unix/build.Dockerfile
@@ -23,4 +23,5 @@ RUN apt-get install \
     tar \
     xz-utils \
     unzip \
+    zip \
     zlib1g-dev
diff --git a/cpython-unix/build.cross.Dockerfile b/cpython-unix/build.cross.Dockerfile
@@ -43,6 +43,7 @@ RUN apt-get install \
     tar \
     xz-utils \
     unzip \
+    zip \
     zlib1g-dev
 
 # Cross-building.
diff --git a/cpython-windows/build.py b/cpython-windows/build.py
@@ -1909,8 +1909,8 @@ def build_cpython(
 
     python_version = entry["version"]
 
-    setuptools_archive = download_entry("setuptools", BUILD)
-    pip_archive = download_entry("pip", BUILD)
+    setuptools_wheel = download_entry("setuptools", BUILD)
+    pip_wheel = download_entry("pip", BUILD)
 
     if arch == "amd64":
         build_platform = "x64"
@@ -1930,8 +1930,6 @@ def build_cpython(
                 python_archive,
                 bzip2_archive,
                 openssl_archive,
-                pip_archive,
-                setuptools_archive,
                 sqlite_archive,
                 tk_bin_archive,
                 xz_archive,
@@ -2113,16 +2111,30 @@ def build_cpython(
             args, pcbuild_path, os.environ,
         )
 
-        # Install setuptools and pip.
+        # Install pip and setuptools.
         exec_and_log(
-            [str(install_dir / "python.exe"), "setup.py", "install"],
-            str(td / ("setuptools-%s" % DOWNLOADS["setuptools"]["version"])),
+            [
+                str(install_dir / "python.exe"),
+                str(pip_wheel / "pip"),
+                "install",
+                "--no-cache-dir",
+                "--no-index",
+                str(pip_wheel),
+            ],
+            td,
             os.environ,
         )
 
         exec_and_log(
-            [str(install_dir / "python.exe"), "setup.py", "install"],
-            str(td / ("pip-%s" % DOWNLOADS["pip"]["version"])),
+            [
+                str(install_dir / "python.exe"),
+                str(pip_wheel / "pip"),
+                "install",
+                "--no-cache-dir",
+                "--no-index",
+                str(setuptools_wheel),
+            ],
+            td,
             os.environ,
         )
 
diff --git a/pythonbuild/downloads.py b/pythonbuild/downloads.py
@@ -286,9 +286,9 @@
         "version": "0.12",
     },
     "pip": {
-        "url": "https://files.pythonhosted.org/packages/52/e1/06c018197d8151383f66ebf6979d951995cf495629fc54149491f5d157d0/pip-21.2.4.tar.gz",
-        "size": 1564487,
-        "sha256": "0eb8a1516c3d138ae8689c0c1a60fde7143310832f9dc77e11d8a4bc62de193b",
+        "url": "https://files.pythonhosted.org/packages/ca/31/b88ef447d595963c01060998cb329251648acf4a067721b0452c45527eb8/pip-21.2.4-py3-none-any.whl",
+        "size": 1555100,
+        "sha256": "fa9ebb85d3fd607617c0c44aca302b1b45d87f9c2a1649b46c26167ca4296323",
         "version": "21.2.4",
     },
     "readline": {
@@ -301,9 +301,9 @@
         "license_file": "LICENSE.readline.txt",
     },
     "setuptools": {
-        "url": "https://files.pythonhosted.org/packages/db/e2/c0ced9ccffb61432305665c22842ea120c0f649eec47ecf2a45c596707c4/setuptools-57.4.0.tar.gz",
-        "size": 2141309,
-        "sha256": "6bac238ffdf24e8806c61440e755192470352850f3419a52f26ffe0a1a64f465",
+        "url": "https://files.pythonhosted.org/packages/bd/25/5bdf7f1adeebd4e3fa76b2e2f045ae53ee208e40a4231ad0f0c3007e4353/setuptools-57.4.0-py3-none-any.whl",
+        "size": 819017,
+        "sha256": "a49230977aa6cfb9d933614d2f7b79036e9945c4cdd7583163f4e920b83418d6",
         "version": "57.4.0",
     },
     "sqlite": {
