diff --git a/CHANGELOG.md b/CHANGELOG.md index 6275cba2e..6edc6ba7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ # Master +- For repeat builds, also manage the installed versions of setuptools/wheel, rather than just that of pip (#1007). +- Install an explicit version of wheel rather than the latest release at the time (#1007). +- Output the installed version of pip, setuptools and wheel in the build log (#1007). +- Errors installing pip/setuptools/wheel are now displayed in the build output and fail the build early (#1007). +- Install pip using itself rather than `get-pip.py` (#1007). +- Disable pip's version check + cache when installing pip/setuptools/wheel (#1007). +- Install setuptools from PyPI rather than a vendored copy (#1007). - Reduce the number of environment variables exposed to `bin/{pre,post}_compile` and other subprocesses (#1011) # 173 (2020-07-21) diff --git a/bin/compile b/bin/compile index dd270d802..9a0a2d6f6 100755 --- a/bin/compile +++ b/bin/compile @@ -65,26 +65,6 @@ PYPY36="pypy3.6" # Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)? DEFAULT_PYTHON_STACK="cedar-14" -# If pip doesn't match this version (the version we install), run the installer. -PIP_UPDATE="20.0.2" - -for file in "$BUILD_DIR/runtime.txt" "$CACHE_DIR/.heroku/python-version" ; do - [ -f "$file" ] || continue - - version=$(tr -d '[:space:]' < "$file") - - case "$version" in "$PY34"*) - # Python 3.4 support was dropped in pip >= 19.2. - PIP_UPDATE="19.1.1" - break - ;; - esac -done - -if [[ -f "$BUILD_DIR/Pipfile" ]]; then - # Do not force pipenv users to re-install pipenv locally. - PIP_UPDATE="9.0.2" -fi # Common Problem Warnings: # This section creates a temporary file in which to stick the output of `pip install`. diff --git a/bin/steps/README.MD b/bin/steps/README.MD index 3c23cf78d..6bf9885f7 100644 --- a/bin/steps/README.MD +++ b/bin/steps/README.MD @@ -2,13 +2,6 @@ TODO: Add context on Python install steps, such as why symlinking vs copying -## Installing the Pip tool - -The Python Buildpack uses a tool called `get-pip` to install the pip tool. This -is done in the `python` script. - -This is in part because Python historically did not come with pip by default. - ## Installing Python packages using Pip ### Convention: Use `python` process to invoke Pip diff --git a/bin/steps/python b/bin/steps/python index 068e147cc..34541be58 100755 --- a/bin/steps/python +++ b/bin/steps/python @@ -131,34 +131,52 @@ if [ ! "$SKIP_INSTALL" ]; then # Record for future reference. echo "$PYTHON_VERSION" > .heroku/python-version echo "$STACK" > .heroku/python-stack - FRESH_PYTHON=true hash -r fi -# Heroku uses the get-pip utility maintained by the Python community to vendor Pip. -# https://github.com/pypa/get-pip -GETPIP_URL="https://lang-python.s3.amazonaws.com/etc/get-pip.py" -GETPIP_PY="${TMPDIR:-/tmp}/get-pip.py" +set -e + +PIP_VERSION='20.0.2' +SETUPTOOLS_VERSION='39.0.1' +WHEEL_VERSION='0.34.2' -if ! curl -s "${GETPIP_URL}" -o "$GETPIP_PY" &> /dev/null; then - mcount "failure.python.get-pip" - echo "Failed to pull down get-pip" - exit 1 +if [[ "${PYTHON_VERSION}" == ${PY34}* ]]; then + # Python 3.4 support was dropped in pip 19.2+ and wheel 0.34.0+. + PIP_VERSION='19.1.1' + WHEEL_VERSION='0.33.6' fi -# If a new Python has been installed or Pip isn't up to date: -if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then +# We don't use get-pip.py, since: +# - it uses `--force-reinstall`, which is unnecessary here and slows down repeat builds +# - it means downloading pip twice (once embedded in get-pip.py, and again during +# the install, since get-pip.py can't install the embedded version directly) +# - we would still have to manage several versions of get-pip.py, to support older Pythons. +# Instead, we use the pip wheel to install itself, using the method described here: +# https://github.com/pypa/pip/issues/2351#issuecomment-69994524 +PIP_WHEEL_FILENAME="pip-${PIP_VERSION}-py2.py3-none-any.whl" +PIP_WHEEL_URL="https://lang-python.s3.amazonaws.com/common/${PIP_WHEEL_FILENAME}" +PIP_WHEEL="${TMPDIR:-/tmp}/${PIP_WHEEL_FILENAME}" + +if ! curl -sSf "${PIP_WHEEL_URL}" -o "$PIP_WHEEL"; then + mcount "failure.python.download-pip" + puts-warn "Failed to download pip" + exit 1 +fi - puts-step "Installing pip" +if [[ -f "$BUILD_DIR/Pipfile" ]]; then + # The buildpack is pinned to old pipenv, which requires older pip. + # Pip 9.0.2 doesn't support installing itself from a wheel, so we have to use split + # versions here (ie: installer pip version different from target pip version). + PIP_VERSION='9.0.2' + PIP_TO_INSTALL="pip==${PIP_VERSION}" +else + PIP_TO_INSTALL="${PIP_WHEEL}" +fi - # Remove old installations. - rm -fr /app/.heroku/python/lib/python*/site-packages/pip-* - rm -fr /app/.heroku/python/lib/python*/site-packages/setuptools-* +puts-step "Installing pip ${PIP_VERSION}, setuptools ${SETUPTOOLS_VERSION} and wheel ${WHEEL_VERSION}" - /app/.heroku/python/bin/python "$GETPIP_PY" pip=="$PIP_UPDATE" &> /dev/null - /app/.heroku/python/bin/pip install "$ROOT_DIR/vendor/setuptools-39.0.1-py2.py3-none-any.whl" &> /dev/null -fi +/app/.heroku/python/bin/python "${PIP_WHEEL}/pip" install --quiet --disable-pip-version-check --no-cache \ + "${PIP_TO_INSTALL}" "setuptools==${SETUPTOOLS_VERSION}" "wheel==${WHEEL_VERSION}" -set -e hash -r diff --git a/test/run-features b/test/run-features index c50a8c33d..478ada1ad 100755 --- a/test/run-features +++ b/test/run-features @@ -44,6 +44,7 @@ testStandardRequirements() { testPipenv() { compile "pipenv" + assertCaptured "Installing pip 9.0.2, setuptools 39.0.1 and wheel 0.34.2" assertCapturedSuccess } diff --git a/test/run-versions b/test/run-versions index 75f165b94..09587d809 100755 --- a/test/run-versions +++ b/test/run-versions @@ -9,6 +9,7 @@ testPythonDefault() { compile "pythonDefault" assertCaptured $DEFAULT_PYTHON_VERSION assertNotCaptured "security update" + assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2" assertCaptured "Installing SQLite3" assertCapturedSuccess } @@ -24,6 +25,7 @@ testPython2() { assertNotCaptured "python-2-7-eol-faq"; fi assertNotCaptured "security update" + assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2" assertCaptured "Installing SQLite3" assertCapturedSuccess } @@ -51,6 +53,7 @@ testPython3_4() { compile "python3_4" assertCaptured $LATEST_34 assertNotCaptured "security update" + assertCaptured "Installing pip 19.1.1, setuptools 39.0.1 and wheel 0.33.6" # if cedar 14 and legacy binaries, fail. if cedar 14 and staging, succeed. if [[ ! -n $USE_STAGING_BINARIES ]] && [[ $STACK == "cedar-14" ]]; then assertCapturedError @@ -86,6 +89,7 @@ testPython3_5() { compile "python3_5" assertCaptured $LATEST_35 assertNotCaptured "security update" + assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2" assertCaptured "Installing SQLite3" assertCapturedSuccess } @@ -108,6 +112,7 @@ testPython3_6() { compile "python3_6" assertCaptured $LATEST_36 assertNotCaptured "security update" + assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2" assertCaptured "Installing SQLite3" assertCapturedSuccess } @@ -134,6 +139,7 @@ testPython3_7() { else assertNotCaptured "security update" assertCaptured $LATEST_37 + assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2" assertCaptured "Installing SQLite3" assertCapturedSuccess fi @@ -177,6 +183,7 @@ testPython3_8() { else assertNotCaptured "security update" assertCaptured $LATEST_38 + assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2" assertCaptured "Installing SQLite3" assertCapturedSuccess fi @@ -195,6 +202,7 @@ testPypy3_6() { else assertCaptured "Installing pypy" assertCaptured "$PYPY_36" + assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2" assertCapturedSuccess fi } @@ -206,6 +214,7 @@ testPypy2_7() { else assertCaptured "Installing pypy" assertCaptured "$PYPY_27" + assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2" assertCapturedSuccess fi } diff --git a/vendor/setuptools-39.0.1-py2.py3-none-any.whl b/vendor/setuptools-39.0.1-py2.py3-none-any.whl deleted file mode 100644 index edc3ca2d8..000000000 Binary files a/vendor/setuptools-39.0.1-py2.py3-none-any.whl and /dev/null differ