From e0075b9486182e281f37f9ead90f3246e86a90e7 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 22 Apr 2024 21:13:19 +0200 Subject: [PATCH 01/11] github-action: provenance --- .github/workflows/snapshot.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/snapshot.yml b/.github/workflows/snapshot.yml index 9a50021824..a8d86fc224 100644 --- a/.github/workflows/snapshot.yml +++ b/.github/workflows/snapshot.yml @@ -48,11 +48,23 @@ jobs: vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} pipeline: apm-agent-java-snapshot - waitFor: false + artifactName: snapshots + artifactPath: "**/target/*" + waitFor: true printBuildLogs: false buildEnvVars: | dry_run=${{ inputs.dry_run || 'false' }} + - uses: actions/download-artifact@v3 + with: + name: snapshots + path: target/ + + - name: generate build provenance + uses: github-early-access/generate-build-provenance@main + with: + subject-path: "${{ github.workspace }}/**/target/*.jar" + - if: ${{ failure() }} uses: elastic/apm-pipeline-library/.github/actions/slack-message@current with: From 331489f8d5eb649f2b9b3bc24741063f8d79764b Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 22 Apr 2024 21:17:13 +0200 Subject: [PATCH 02/11] support a different branch --- .github/workflows/snapshot.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/snapshot.yml b/.github/workflows/snapshot.yml index a8d86fc224..1bca4f3212 100644 --- a/.github/workflows/snapshot.yml +++ b/.github/workflows/snapshot.yml @@ -5,6 +5,7 @@ on: push: branches: - "main" + - "feature/support-provenance-with-github-actions" workflow_dispatch: inputs: dry_run: @@ -48,6 +49,7 @@ jobs: vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} pipeline: apm-agent-java-snapshot + pipelineBranch: ${{ github.ref_name }} artifactName: snapshots artifactPath: "**/target/*" waitFor: true From 4e756e60a165a1b83f2aa9975f112f022e858b3d Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 22 Apr 2024 21:19:50 +0200 Subject: [PATCH 03/11] Revert "support a different branch" This reverts commit 331489f8d5eb649f2b9b3bc24741063f8d79764b. --- .github/workflows/snapshot.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/snapshot.yml b/.github/workflows/snapshot.yml index 1bca4f3212..a8d86fc224 100644 --- a/.github/workflows/snapshot.yml +++ b/.github/workflows/snapshot.yml @@ -5,7 +5,6 @@ on: push: branches: - "main" - - "feature/support-provenance-with-github-actions" workflow_dispatch: inputs: dry_run: @@ -49,7 +48,6 @@ jobs: vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} pipeline: apm-agent-java-snapshot - pipelineBranch: ${{ github.ref_name }} artifactName: snapshots artifactPath: "**/target/*" waitFor: true From ba639d7ac96a1c7695a34b846f1099ae4ffde1c1 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 22 Apr 2024 21:46:38 +0200 Subject: [PATCH 04/11] ci: faster artifact sharing between the two CIs --- .buildkite/snapshot.yml | 1 + .ci/snapshot.sh | 3 +++ .github/workflows/snapshot.yml | 5 +++-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.buildkite/snapshot.yml b/.buildkite/snapshot.yml index c5cf1bcb79..a027aaa2c5 100644 --- a/.buildkite/snapshot.yml +++ b/.buildkite/snapshot.yml @@ -9,6 +9,7 @@ steps: artifact_paths: - "snapshot.txt" - "**/target/*" + - "dist.tar" notify: - slack: "#apm-agent-java" diff --git a/.ci/snapshot.sh b/.ci/snapshot.sh index a11cd4eb9a..6861b5dffa 100755 --- a/.ci/snapshot.sh +++ b/.ci/snapshot.sh @@ -34,3 +34,6 @@ fi echo "--- Deploy the snapshot :package: [./mvnw $GOAL)] $DRY_RUN_MSG" ./mvnw -V -s .ci/settings.xml -Pgpg clean $GOAL -DskipTests --batch-mode | tee snapshot.txt + +echo "--- Archive the target folder with jar files" +find . -type d -name target -exec find {} -name '*.jar' -print0 \; | xargs -0 tar -cvf dist.tar diff --git a/.github/workflows/snapshot.yml b/.github/workflows/snapshot.yml index a8d86fc224..bffaedac82 100644 --- a/.github/workflows/snapshot.yml +++ b/.github/workflows/snapshot.yml @@ -49,7 +49,7 @@ jobs: vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} pipeline: apm-agent-java-snapshot artifactName: snapshots - artifactPath: "**/target/*" + artifactPath: "dist.tar" waitFor: true printBuildLogs: false buildEnvVars: | @@ -58,7 +58,8 @@ jobs: - uses: actions/download-artifact@v3 with: name: snapshots - path: target/ + + - run: tar xvf dist.tar - name: generate build provenance uses: github-early-access/generate-build-provenance@main From 55e29ba2f7d2f7a0e056885d45ae4642d86bfff3 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 22 Apr 2024 21:48:31 +0200 Subject: [PATCH 05/11] enable provenance --- .github/workflows/snapshot.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/snapshot.yml b/.github/workflows/snapshot.yml index bffaedac82..60cda4c763 100644 --- a/.github/workflows/snapshot.yml +++ b/.github/workflows/snapshot.yml @@ -38,6 +38,9 @@ jobs: runs-on: ubuntu-latest needs: - validate + permissions: + contents: write + id-token: write if: ${{ contains(needs.validate.outputs.is-snapshot, 'true') }} steps: - id: buildkite @@ -59,7 +62,8 @@ jobs: with: name: snapshots - - run: tar xvf dist.tar + - name: untar the buildkite tarbal + run: tar xvf dist.tar - name: generate build provenance uses: github-early-access/generate-build-provenance@main From 5bb18bfee3730466776cb604648ae60207f8c164 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 23 Apr 2024 09:18:41 +0200 Subject: [PATCH 06/11] Revert "Revert "support a different branch"" This reverts commit 4e756e60a165a1b83f2aa9975f112f022e858b3d. --- .github/workflows/snapshot.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/snapshot.yml b/.github/workflows/snapshot.yml index 60cda4c763..30647466d6 100644 --- a/.github/workflows/snapshot.yml +++ b/.github/workflows/snapshot.yml @@ -5,6 +5,7 @@ on: push: branches: - "main" + - "feature/support-provenance-with-github-actions" workflow_dispatch: inputs: dry_run: @@ -51,6 +52,7 @@ jobs: vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} pipeline: apm-agent-java-snapshot + pipelineBranch: ${{ github.ref_name }} artifactName: snapshots artifactPath: "dist.tar" waitFor: true From 6a4d6192168ee7f30bbe9e05493611bd0b06ff19 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 23 Apr 2024 09:23:14 +0200 Subject: [PATCH 07/11] support tarball --- .buildkite/snapshot.yml | 5 ++++- .ci/snapshot.sh | 2 +- .github/workflows/snapshot.yml | 9 ++++++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.buildkite/snapshot.yml b/.buildkite/snapshot.yml index a027aaa2c5..9a8544f4b4 100644 --- a/.buildkite/snapshot.yml +++ b/.buildkite/snapshot.yml @@ -2,6 +2,9 @@ agents: provider: "gcp" image: "family/apm-agent-java-ubuntu-2204" +env: + TARBALL_FILE: ${TARBALL_FILE:-dist.tar} + steps: - label: "Run the snapshot" key: "release" @@ -9,7 +12,7 @@ steps: artifact_paths: - "snapshot.txt" - "**/target/*" - - "dist.tar" + - "${TARBALL_FILE}" notify: - slack: "#apm-agent-java" diff --git a/.ci/snapshot.sh b/.ci/snapshot.sh index 6861b5dffa..a6ef0c4424 100755 --- a/.ci/snapshot.sh +++ b/.ci/snapshot.sh @@ -36,4 +36,4 @@ echo "--- Deploy the snapshot :package: [./mvnw $GOAL)] $DRY_RUN_MSG" ./mvnw -V -s .ci/settings.xml -Pgpg clean $GOAL -DskipTests --batch-mode | tee snapshot.txt echo "--- Archive the target folder with jar files" -find . -type d -name target -exec find {} -name '*.jar' -print0 \; | xargs -0 tar -cvf dist.tar +find . -type d -name target -exec find {} -name '*.jar' -print0 \; | xargs -0 tar -cvf "${TARBALL_FILE:-dist.tar}" diff --git a/.github/workflows/snapshot.yml b/.github/workflows/snapshot.yml index 30647466d6..c24c204e93 100644 --- a/.github/workflows/snapshot.yml +++ b/.github/workflows/snapshot.yml @@ -42,6 +42,8 @@ jobs: permissions: contents: write id-token: write + env: + TARBALL_FILE: dist.tar if: ${{ contains(needs.validate.outputs.is-snapshot, 'true') }} steps: - id: buildkite @@ -54,18 +56,19 @@ jobs: pipeline: apm-agent-java-snapshot pipelineBranch: ${{ github.ref_name }} artifactName: snapshots - artifactPath: "dist.tar" + artifactPath: ${{ env.TARBALL_FILE }} waitFor: true printBuildLogs: false buildEnvVars: | dry_run=${{ inputs.dry_run || 'false' }} + TARBALL_FILE=${{ env.TARBALL_FILE }} - uses: actions/download-artifact@v3 with: name: snapshots - - name: untar the buildkite tarbal - run: tar xvf dist.tar + - name: untar the buildkite tarball + run: tar xvf ${{ env.TARBALL_FILE }} - name: generate build provenance uses: github-early-access/generate-build-provenance@main From 2507472f42b57c90c48e5f73e57e40b151164890 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 23 Apr 2024 10:19:36 +0200 Subject: [PATCH 08/11] support releases with provenance --- .buildkite/release.yml | 4 ++++ .ci/release.sh | 3 +++ .github/workflows/release.yml | 20 ++++++++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/.buildkite/release.yml b/.buildkite/release.yml index 347f35358e..a3da6d8d6e 100644 --- a/.buildkite/release.yml +++ b/.buildkite/release.yml @@ -2,6 +2,9 @@ agents: provider: "gcp" image: "family/apm-agent-java-ubuntu-2204" +env: + TARBALL_FILE: ${TARBALL_FILE:-dist.tar} + steps: - label: "Run the release" key: "release" @@ -9,6 +12,7 @@ steps: artifact_paths: - "release.txt" - "**/target/*" + - "${TARBALL_FILE}" notify: - slack: "#apm-agent-java" diff --git a/.ci/release.sh b/.ci/release.sh index b05947ce95..bcc01b4163 100755 --- a/.ci/release.sh +++ b/.ci/release.sh @@ -32,3 +32,6 @@ fi echo "--- Deploy the release :package: [./mvnw $GOAL)] $DRY_RUN_MSG" ./mvnw -V -s .ci/settings.xml -Pgpg clean $GOAL -DskipTests --batch-mode | tee release.txt + +echo "--- Archive the target folder with jar files" +find . -type d -name target -exec find {} -name '*.jar' -print0 \; | xargs -0 tar -cvf "${TARBALL_FILE:-dist.tar}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bc9b9d16e6..775efcc4de 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -74,6 +74,11 @@ jobs: runs-on: ubuntu-latest needs: - validate-tag + permissions: + contents: write + id-token: write + env: + TARBALL_FILE: dist.tar steps: - id: buildkite continue-on-error: true @@ -86,8 +91,23 @@ jobs: pipeline: apm-agent-java-release waitFor: true printBuildLogs: false + artifactName: releases + artifactPath: ${{ env.TARBALL_FILE }} buildEnvVars: | dry_run=${{ inputs.dry_run || 'false' }} + TARBALL_FILE=${{ env.TARBALL_FILE }} + + - uses: actions/download-artifact@v3 + with: + name: releases + + - name: untar the buildkite tarball + run: tar xvf ${{ env.TARBALL_FILE }} + + - name: generate build provenance + uses: github-early-access/generate-build-provenance@main + with: + subject-path: "${{ github.workspace }}/**/target/*.jar" await-maven-central-artifact: name: "Wait for artifacts to be available on maven central" From 971bad1e5059ef72c64322a7fa8996cb92115c01 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 23 Apr 2024 10:27:56 +0200 Subject: [PATCH 09/11] remove test changes --- .github/workflows/snapshot.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/snapshot.yml b/.github/workflows/snapshot.yml index c24c204e93..6c87ce6797 100644 --- a/.github/workflows/snapshot.yml +++ b/.github/workflows/snapshot.yml @@ -5,7 +5,6 @@ on: push: branches: - "main" - - "feature/support-provenance-with-github-actions" workflow_dispatch: inputs: dry_run: From 08e5715488659cc5b58b0774aedb4b836be29a6c Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 24 Apr 2024 18:24:39 +0200 Subject: [PATCH 10/11] fix shebang --- .ci/scripts/published-artifacts-list.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/scripts/published-artifacts-list.sh b/.ci/scripts/published-artifacts-list.sh index 32edfc6d5a..773a1671fe 100755 --- a/.ci/scripts/published-artifacts-list.sh +++ b/.ci/scripts/published-artifacts-list.sh @@ -1,4 +1,4 @@ -#!/bin/env bash +#!/usr/bin/env bash targets="$(find . -type d -name 'target'|grep -v apm-agent-plugins|grep -v integration-tests|sort)" From ce87909b089a09acaa879bfc8fe73eae3ad890ca Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 24 Apr 2024 18:25:58 +0200 Subject: [PATCH 11/11] use script --- .buildkite/release.yml | 2 +- .buildkite/snapshot.yml | 2 +- .ci/release.sh | 5 ++++- .ci/snapshot.sh | 5 ++++- .github/workflows/release.yml | 2 +- .github/workflows/snapshot.yml | 2 +- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.buildkite/release.yml b/.buildkite/release.yml index a3da6d8d6e..f63a7e06dd 100644 --- a/.buildkite/release.yml +++ b/.buildkite/release.yml @@ -3,7 +3,7 @@ agents: image: "family/apm-agent-java-ubuntu-2204" env: - TARBALL_FILE: ${TARBALL_FILE:-dist.tar} + TARBALL_FILE: ${TARBALL_FILE:-artifacts.tar} steps: - label: "Run the release" diff --git a/.buildkite/snapshot.yml b/.buildkite/snapshot.yml index 9a8544f4b4..899d962e7e 100644 --- a/.buildkite/snapshot.yml +++ b/.buildkite/snapshot.yml @@ -3,7 +3,7 @@ agents: image: "family/apm-agent-java-ubuntu-2204" env: - TARBALL_FILE: ${TARBALL_FILE:-dist.tar} + TARBALL_FILE: ${TARBALL_FILE:-artifacts.tar} steps: - label: "Run the snapshot" diff --git a/.ci/release.sh b/.ci/release.sh index bcc01b4163..2f7c4bd832 100755 --- a/.ci/release.sh +++ b/.ci/release.sh @@ -34,4 +34,7 @@ echo "--- Deploy the release :package: [./mvnw $GOAL)] $DRY_RUN_MSG" ./mvnw -V -s .ci/settings.xml -Pgpg clean $GOAL -DskipTests --batch-mode | tee release.txt echo "--- Archive the target folder with jar files" -find . -type d -name target -exec find {} -name '*.jar' -print0 \; | xargs -0 tar -cvf "${TARBALL_FILE:-dist.tar}" +echo 'gather artifacts' +.ci/published-artifacts-list.sh | tee artifacts.list +echo 'create tarbal' +tar -cvf "${TARBALL_FILE:-artifacts.tar}" -T artifacts.list diff --git a/.ci/snapshot.sh b/.ci/snapshot.sh index a6ef0c4424..5fa49b94a6 100755 --- a/.ci/snapshot.sh +++ b/.ci/snapshot.sh @@ -36,4 +36,7 @@ echo "--- Deploy the snapshot :package: [./mvnw $GOAL)] $DRY_RUN_MSG" ./mvnw -V -s .ci/settings.xml -Pgpg clean $GOAL -DskipTests --batch-mode | tee snapshot.txt echo "--- Archive the target folder with jar files" -find . -type d -name target -exec find {} -name '*.jar' -print0 \; | xargs -0 tar -cvf "${TARBALL_FILE:-dist.tar}" +echo 'gather artifacts' +.ci/published-artifacts-list.sh | tee artifacts.list +echo 'create tarbal' +tar -cvf "${TARBALL_FILE:-artifacts.tar}" -T artifacts.list diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 775efcc4de..76d9bcd125 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -78,7 +78,7 @@ jobs: contents: write id-token: write env: - TARBALL_FILE: dist.tar + TARBALL_FILE: artifacts.tar steps: - id: buildkite continue-on-error: true diff --git a/.github/workflows/snapshot.yml b/.github/workflows/snapshot.yml index 6c87ce6797..30388ce869 100644 --- a/.github/workflows/snapshot.yml +++ b/.github/workflows/snapshot.yml @@ -42,7 +42,7 @@ jobs: contents: write id-token: write env: - TARBALL_FILE: dist.tar + TARBALL_FILE: artifacts.tar if: ${{ contains(needs.validate.outputs.is-snapshot, 'true') }} steps: - id: buildkite