From 157d6701c2d64912de4bde2330814c7b5641436f Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Sat, 4 May 2024 14:28:17 +0000 Subject: [PATCH 01/10] workflows: Add a new job for packaging release sources This job uses the new artifact attestations: https://github.blog/2024-05-02-introducing-artifact-attestations-now-in-public-beta/ This will allow users to verify that the sources came from a specific workflow run in the llvm-project repository. Currently, this job does not automatically upload sources to the release page, but rather it attaches them the workflow run as artifacts. The release manager is expected to download, verify, and sign the sources before uploading them to the release page. We may be able to automatically upload them in the future once we have a process for signing the binaries within the github workflow. Technically, though, the binaries are being signed as part of the attestation process, but the only way to verify the signatures is using the gh command line tool, and I don't think it is best to rely on that, since the tool may not be easily available on all systems. --- .github/workflows/release-sources.yml | 57 +++++++++++++++++++++++++++ .github/workflows/release-tasks.yml | 8 ++++ 2 files changed, 65 insertions(+) create mode 100644 .github/workflows/release-sources.yml diff --git a/.github/workflows/release-sources.yml b/.github/workflows/release-sources.yml new file mode 100644 index 0000000000000..0029078ccb7ee --- /dev/null +++ b/.github/workflows/release-sources.yml @@ -0,0 +1,57 @@ +name: Release Sources + +permissions: + contents: read + +on: + workflow_dispatch: + inputs: + release-version: + description: Release Version + required: true + type: string + workflow_call: + inputs: + release-version: + description: Release Version + required: true + type: string +jobs: + release-sources: + name: Package Release Sources + if: github.repository_owner == 'llvm' + runs-on: ubuntu-latest + permissions: + id-token: write + attestations: write + steps: + - name: Checkout LLVM + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: llvmorg-${{ inputs.release-version }} + fetch-tags: true + - name: Install Dependencies + run: | + pip install -r ./llvm/utils/git/requirements.txt + - name: Check Permissions + env: + GITHUB_TOKEN: ${{ github.token }} + USER_TOKEN: ${{ secrets.RELEASE_TASKS_USER_TOKEN }} + run: | + ./llvm/utils/release/./github-upload-release.py --token "$GITHUB_TOKEN" --user ${{ github.actor }} --user-token "$USER_TOKEN" check-permissions + - name: Create Tarballs + run: | + ./llvm/utils/release/export.sh -release "${{ inputs.release-version }}" -final + - name: Attest Build Provenance + id: provenance + uses: actions/attest-build-provenance@897ed5eab6ed058a474202017ada7f40bfa52940 # v1.0.0 + with: + subject-path: "*.xz" + - name: Create Tarball Artifacts + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 #v4.3.3 + with: + path: | + *.xz + ${{ steps.provenance.outputs.bundle-path }} + + diff --git a/.github/workflows/release-tasks.yml b/.github/workflows/release-tasks.yml index 29049ff014288..b85a8144a9f18 100644 --- a/.github/workflows/release-tasks.yml +++ b/.github/workflows/release-tasks.yml @@ -85,3 +85,11 @@ jobs: with: release-version: ${{ needs.validate-tag.outputs.release-version }} upload: true + + release-sources: + name: Package Release Sources + needs: + - validate-tag + uses: ./.github/workflows/release-sources.yml + with: + release-version: ${{ needs.validate-tag.outputs.release-version }} From 6c564a0f9204e99388306a78142243ea2590c22a Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Sat, 11 May 2024 03:39:56 +0000 Subject: [PATCH 02/10] Fix permissions when called from release-tasks workflow --- .github/workflows/release-tasks.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release-tasks.yml b/.github/workflows/release-tasks.yml index b85a8144a9f18..2ed56dace1d4c 100644 --- a/.github/workflows/release-tasks.yml +++ b/.github/workflows/release-tasks.yml @@ -88,6 +88,9 @@ jobs: release-sources: name: Package Release Sources + permissions: + id-token: write + attestations: write needs: - validate-tag uses: ./.github/workflows/release-sources.yml From adb733de9411e67cb923b0e0ab4dee4dc44b9717 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Sat, 11 May 2024 14:06:11 +0000 Subject: [PATCH 03/10] Fix tarball paths --- .github/workflows/release-sources.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-sources.yml b/.github/workflows/release-sources.yml index 0029078ccb7ee..aa58e04935814 100644 --- a/.github/workflows/release-sources.yml +++ b/.github/workflows/release-sources.yml @@ -47,11 +47,13 @@ jobs: uses: actions/attest-build-provenance@897ed5eab6ed058a474202017ada7f40bfa52940 # v1.0.0 with: subject-path: "*.xz" + - run: | + mv ${{ steps.provenance.outputs.bundle-path }} . - name: Create Tarball Artifacts uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 #v4.3.3 with: path: | *.xz - ${{ steps.provenance.outputs.bundle-path }} + attestation.jsonl From f95e0f3a546c0e72b0be2f9e2ed5ee7040396e5e Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Sat, 11 May 2024 18:17:54 +0000 Subject: [PATCH 04/10] Add documentation --- llvm/docs/HowToReleaseLLVM.rst | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/llvm/docs/HowToReleaseLLVM.rst b/llvm/docs/HowToReleaseLLVM.rst index 51ab6dfd8d8d5..eff5df074910e 100644 --- a/llvm/docs/HowToReleaseLLVM.rst +++ b/llvm/docs/HowToReleaseLLVM.rst @@ -144,8 +144,17 @@ Tag release candidates: $ git tag -sa llvmorg-X.Y.Z-rcN -The Release Manager must supply pre-packaged source tarballs for users. This can -be done with the export.sh script in utils/release. +The pre-packaged source tarballs will be automatically generated via the +"Release Sources" workflow on GitHub. This workflow will create an artifact +containing all the release tarballs and the artifact attestation. The +Release Manager should download the artifact, verify the tarballs, sign them, +and then upload them to the release page. + +:: + + $ unzip artifact.zip + $ gh auth login + $ for f in *.xz; do gh attestation verify --owner llvm $f && gpg -b $f; done Tarballs, release binaries, or any other release artifacts must be uploaded to GitHub. This can be done using the github-upload-release.py script in utils/release. @@ -154,12 +163,6 @@ GitHub. This can be done using the github-upload-release.py script in utils/rel $ github-upload-release.py upload --token --release X.Y.Z-rcN --files -:: - - $ ./export.sh -release X.Y.Z -rc $RC - -This will generate source tarballs for each LLVM project being validated, which -can be uploaded to github for further testing. Build The Binary Distribution ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 58861f7744129084c36d8cd427aac363d918f531 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Fri, 17 May 2024 08:46:42 -0700 Subject: [PATCH 05/10] Require hashes for pip --- .github/workflows/release-sources.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-sources.yml b/.github/workflows/release-sources.yml index aa58e04935814..56556f5b18eaa 100644 --- a/.github/workflows/release-sources.yml +++ b/.github/workflows/release-sources.yml @@ -32,7 +32,8 @@ jobs: fetch-tags: true - name: Install Dependencies run: | - pip install -r ./llvm/utils/git/requirements.txt + pip install --require-hashes -r ./llvm/utils/git/requirements.txt + - name: Check Permissions env: GITHUB_TOKEN: ${{ github.token }} From 8b58db37af0fb5b8deab2df158c38341eb419b81 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Fri, 17 May 2024 12:25:25 -0700 Subject: [PATCH 06/10] Add pull_request trigger --- .github/workflows/release-sources.yml | 42 +++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-sources.yml b/.github/workflows/release-sources.yml index 56556f5b18eaa..af459296a4eaa 100644 --- a/.github/workflows/release-sources.yml +++ b/.github/workflows/release-sources.yml @@ -16,11 +16,49 @@ on: description: Release Version required: true type: string + pull_request: + types: + - opened + - synchronize + - reopened + # When a PR is closed, we still start this workflow, but then skip + # all the jobs, which makes it effectively a no-op. The reason to + # do this is that it allows us to take advantage of concurrency groups + # to cancel in progress CI jobs whenever the PR is closed. + - closed + +concurrency: + group: ${{ github.workflow }}-${{ inputs.release-version || github.event.pull_request.number }} + cancel-in-progress: True + jobs: + inputs: + name: Collect Job Inputs + if: >- + github.repository_owner == 'llvm' && + github.event.action != 'closed' + outputs: + ref: ${{ steps.inputs.outputs.ref }} + export-args: ${{ steps.inputs.outputs.export-args }} + runs-on: ubuntu-latest + steps: + id: inputs + run: | + ref=${{ inputs.release-version || github.sha }} + if [ -n "${{ inputs.release-version }}" ]; then + export_args="-release ${{ inputs.release-version }} -final" + else + export_args="-git-ref ${{ github.sha }}" + fi + echo "ref=$ref" >> $GITHUB_OUTPUT + echo "export-args=$export_args" >> $GITHUB_OUTPUT + release-sources: name: Package Release Sources if: github.repository_owner == 'llvm' runs-on: ubuntu-latest + needs: + - inputs permissions: id-token: write attestations: write @@ -28,7 +66,7 @@ jobs: - name: Checkout LLVM uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: - ref: llvmorg-${{ inputs.release-version }} + ref: ${{ needs.inputs.outputs.ref }} fetch-tags: true - name: Install Dependencies run: | @@ -42,7 +80,7 @@ jobs: ./llvm/utils/release/./github-upload-release.py --token "$GITHUB_TOKEN" --user ${{ github.actor }} --user-token "$USER_TOKEN" check-permissions - name: Create Tarballs run: | - ./llvm/utils/release/export.sh -release "${{ inputs.release-version }}" -final + ./llvm/utils/release/export.sh ${{ needs.inputs.outputs.export-args }} - name: Attest Build Provenance id: provenance uses: actions/attest-build-provenance@897ed5eab6ed058a474202017ada7f40bfa52940 # v1.0.0 From f160515210591d7bd26f7d607b2961933ce3ba55 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Fri, 17 May 2024 12:55:36 -0700 Subject: [PATCH 07/10] Fix typo --- .github/workflows/release-sources.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release-sources.yml b/.github/workflows/release-sources.yml index af459296a4eaa..b4b1672a00f94 100644 --- a/.github/workflows/release-sources.yml +++ b/.github/workflows/release-sources.yml @@ -42,16 +42,16 @@ jobs: export-args: ${{ steps.inputs.outputs.export-args }} runs-on: ubuntu-latest steps: - id: inputs - run: | - ref=${{ inputs.release-version || github.sha }} - if [ -n "${{ inputs.release-version }}" ]; then - export_args="-release ${{ inputs.release-version }} -final" - else - export_args="-git-ref ${{ github.sha }}" - fi - echo "ref=$ref" >> $GITHUB_OUTPUT - echo "export-args=$export_args" >> $GITHUB_OUTPUT + - id: inputs + run: | + ref=${{ inputs.release-version || github.sha }} + if [ -n "${{ inputs.release-version }}" ]; then + export_args="-release ${{ inputs.release-version }} -final" + else + export_args="-git-ref ${{ github.sha }}" + fi + echo "ref=$ref" >> $GITHUB_OUTPUT + echo "export-args=$export_args" >> $GITHUB_OUTPUT release-sources: name: Package Release Sources From 2aee14e46f483753755f18718827c873b6a57476 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Fri, 17 May 2024 13:03:30 -0700 Subject: [PATCH 08/10] Disable permissions check on pull requests --- .github/workflows/release-sources.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-sources.yml b/.github/workflows/release-sources.yml index b4b1672a00f94..3485a84b15a26 100644 --- a/.github/workflows/release-sources.yml +++ b/.github/workflows/release-sources.yml @@ -15,7 +15,8 @@ on: release-version: description: Release Version required: true - type: string + type: stringA + # Run on pull_requests for testing purposes. pull_request: types: - opened @@ -73,6 +74,7 @@ jobs: pip install --require-hashes -r ./llvm/utils/git/requirements.txt - name: Check Permissions + if: github.event_name != 'pull_request' env: GITHUB_TOKEN: ${{ github.token }} USER_TOKEN: ${{ secrets.RELEASE_TASKS_USER_TOKEN }} From 8bbbd02ef95e99028b376d0287eda86f7340e989 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Fri, 17 May 2024 13:08:04 -0700 Subject: [PATCH 09/10] Fixes --- .github/workflows/release-sources.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-sources.yml b/.github/workflows/release-sources.yml index 3485a84b15a26..0fdd6e7dd656a 100644 --- a/.github/workflows/release-sources.yml +++ b/.github/workflows/release-sources.yml @@ -15,9 +15,11 @@ on: release-version: description: Release Version required: true - type: stringA + type: string # Run on pull_requests for testing purposes. pull_request: + paths: + - '.github/workflows/release-sources.yml' types: - opened - synchronize From 97513a4eade8954f045c730e02030978b5bfe315 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Fri, 17 May 2024 13:18:06 -0700 Subject: [PATCH 10/10] Disable some more steps for pull requests --- .github/workflows/release-sources.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-sources.yml b/.github/workflows/release-sources.yml index 0fdd6e7dd656a..9c5b1a9f01709 100644 --- a/.github/workflows/release-sources.yml +++ b/.github/workflows/release-sources.yml @@ -86,11 +86,13 @@ jobs: run: | ./llvm/utils/release/export.sh ${{ needs.inputs.outputs.export-args }} - name: Attest Build Provenance + if: github.event_name != 'pull_request' id: provenance uses: actions/attest-build-provenance@897ed5eab6ed058a474202017ada7f40bfa52940 # v1.0.0 with: subject-path: "*.xz" - - run: | + - if: github.event_name != 'pull_request' + run: | mv ${{ steps.provenance.outputs.bundle-path }} . - name: Create Tarball Artifacts uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 #v4.3.3