From 98ed54359d883c9645892e0835569e3c55dc4240 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Mon, 17 Feb 2025 13:53:55 +0200 Subject: [PATCH] [3.12] CI: Use ImageOS (ubuntu24) instead of runner.os (Linux) in cache keys (GH-130200) (cherry picked from commit 3d7a141c2f235b70dd6c2d64308022c4b77df131) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/workflows/build.yml | 89 ++++++++++++--------------- .github/workflows/reusable-macos.yml | 4 +- .github/workflows/reusable-tsan.yml | 10 +-- .github/workflows/reusable-ubuntu.yml | 26 ++++---- 4 files changed, 60 insertions(+), 69 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bfe5c4c63f9432..e0a4fc6bb648f2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,19 +5,11 @@ on: push: branches: - 'main' - - '3.12' - - '3.11' - - '3.10' - - '3.9' - - '3.8' + - '3.*' pull_request: branches: - 'main' - - '3.12' - - '3.11' - - '3.10' - - '3.9' - - '3.8' + - '3.*' permissions: contents: read @@ -115,8 +107,6 @@ jobs: with: fetch-depth: 1 persist-credentials: false - - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV - name: Check Autoconf and aclocal versions run: | grep "Generated by GNU Autoconf 2.71" configure @@ -157,16 +147,17 @@ jobs: with: python-version: '3.x' - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }} - - name: Install Dependencies + # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }} + - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Add ccache to PATH - run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 with: @@ -310,21 +301,21 @@ jobs: with: persist-credentials: false - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.check_source.outputs.config_hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install Dependencies + - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Configure OpenSSL env vars run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl uses: actions/cache@v4 @@ -333,10 +324,10 @@ jobs: key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - name: Add ccache to PATH run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 with: @@ -369,9 +360,9 @@ jobs: run: sudo ./.github/workflows/posix-deps-apt.sh - name: Configure OpenSSL env vars run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl uses: actions/cache@v4 @@ -380,36 +371,36 @@ jobs: key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - name: Add ccache to PATH run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 with: save: false - name: Setup directory envs for out-of-tree builds run: | - echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV - echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" + echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" - name: Create directories for read-only out-of-tree builds - run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR + run: mkdir -p "$CPYTHON_RO_SRCDIR" "$CPYTHON_BUILDDIR" - name: Bind mount sources read-only - run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.check_source.outputs.config_hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | ../cpython-ro-srcdir/configure \ --config-cache \ --with-pydebug \ - --with-openssl=$OPENSSL_DIR + --with-openssl="$OPENSSL_DIR" - name: Build CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make -j4 @@ -418,18 +409,18 @@ jobs: run: make pythoninfo - name: Remount sources writable for tests # some tests write to srcdir, lack of pyc files slows down testing - run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw + run: sudo mount "$CPYTHON_RO_SRCDIR" -oremount,rw - name: Setup directory envs for out-of-tree builds run: | - echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" - name: "Create hypothesis venv" working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | VENV_LOC=$(realpath -m .)/hypovenv VENV_PYTHON=$VENV_LOC/bin/python - echo "HYPOVENV=${VENV_LOC}" >> $GITHUB_ENV - echo "VENV_PYTHON=${VENV_PYTHON}" >> $GITHUB_ENV - ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -r ${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt + echo "HYPOVENV=${VENV_LOC}" >> "$GITHUB_ENV" + echo "VENV_PYTHON=${VENV_PYTHON}" >> "$GITHUB_ENV" + ./python -m venv "$VENV_LOC" && "$VENV_PYTHON" -m pip install -r "${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt" - name: 'Restore Hypothesis database' id: cache-hypothesis-database uses: actions/cache@v4 @@ -483,15 +474,15 @@ jobs: with: persist-credentials: false - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.check_source.outputs.config_hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install Dependencies + - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Set up GCC-10 for ASAN uses: egor-tensin/setup-gcc@v1 @@ -499,9 +490,9 @@ jobs: version: 10 - name: Configure OpenSSL env vars run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl uses: actions/cache@v4 @@ -510,10 +501,10 @@ jobs: key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - name: Add ccache to PATH run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 with: diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index a7ed573df339bf..918f0e8856a4e2 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -34,12 +34,12 @@ jobs: with: persist-credentials: false - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ inputs.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }} - name: Install Homebrew dependencies run: | brew install pkg-config openssl@3.0 xz gdbm tcl-tk@8 diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml index 874d842201a4f8..d53e9d5b38ed00 100644 --- a/.github/workflows/reusable-tsan.yml +++ b/.github/workflows/reusable-tsan.yml @@ -25,19 +25,19 @@ jobs: with: persist-credentials: false - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} - - name: Install Dependencies + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }} + - name: Install dependencies run: | sudo ./.github/workflows/posix-deps-apt.sh sudo apt install -y clang # Reduce ASLR to avoid TSAN crashing sudo sysctl -w vm.mmap_rnd_bits=28 - - name: TSAN Option Setup + - name: TSAN option setup run: | echo "TSAN_OPTIONS=log_path=${GITHUB_WORKSPACE}/tsan_log suppressions=${GITHUB_WORKSPACE}/Tools/tsan/suppressions${{ fromJSON(inputs.free-threading) @@ -48,7 +48,7 @@ jobs: echo "CXX=clang++" >> "$GITHUB_ENV" - name: Add ccache to PATH run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 with: diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 4f6d73cb3729a1..620b3cf0d2830b 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -37,9 +37,9 @@ jobs: run: sudo ./.github/workflows/posix-deps-apt.sh - name: Configure OpenSSL env vars run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl uses: actions/cache@v4 @@ -48,10 +48,10 @@ jobs: key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - name: Add ccache to PATH run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 with: @@ -59,26 +59,26 @@ jobs: max-size: "200M" - name: Setup directory envs for out-of-tree builds run: | - echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV - echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" + echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" - name: Create directories for read-only out-of-tree builds - run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR + run: mkdir -p "$CPYTHON_RO_SRCDIR" "$CPYTHON_BUILDDIR" - name: Bind mount sources read-only - run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache uses: actions/cache@v4 with: path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: >- ../cpython-ro-srcdir/configure --config-cache --with-pydebug - --with-openssl=$OPENSSL_DIR + --with-openssl="$OPENSSL_DIR" ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} - name: Build CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} @@ -88,7 +88,7 @@ jobs: run: make pythoninfo - name: Remount sources writable for tests # some tests write to srcdir, lack of pyc files slows down testing - run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw + run: sudo mount "$CPYTHON_RO_SRCDIR" -oremount,rw - name: Tests working-directory: ${{ env.CPYTHON_BUILDDIR }} run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"