From 9f4c56bfc043c3ce2ca479192d93b41397332b13 Mon Sep 17 00:00:00 2001 From: Ivo Anjo Date: Thu, 13 Feb 2025 11:58:02 +0000 Subject: [PATCH 1/4] Add support for skipping slow builds and reusing previous results This PR follows from the discussion in https://github.com/ruby/ruby-dev-builder/pull/13 . It splits the workflow into two ways of working: 1. When skipping slow Ruby builds, it only does ruby-head builds; this means debug and asan builds from a previous release are reused 2. When not skipping slow Ruby builds, this workflow works exactly as before The "main" tricks are: 1. The matrix for the "build" job is now dynamic, allowing us to reduce the number of builds 2. There's a "reuse-slow" job that is run as alternative to download/reupload builds I think the final result is quite simple/maintainable, although I'll admit it took quite a few iterations to get to this point. Note that this PR does not do everything we discussed in https://github.com/ruby/ruby-dev-builder/pull/13 : 1. It doesn't create the stable 3.4 asan variant yet 2. It doesn't change the build cron schedule or use the skip_slow for anything other than manually triggered builds yet My thinking is, I wanted feedback on if this was the right way to go before investing on the latter parts. (But this PR is self-contained to go in as-is) How does it work? Here's an example run: * Without skip_slow: [run](https://github.com/DataDog/ruby-dev-builder/actions/runs/13326349375) / [resulting release](https://github.com/DataDog/ruby-dev-builder/releases/tag/v20250214.093006) * With skip_slow: [run](https://github.com/DataDog/ruby-dev-builder/actions/runs/13326366082) / [resulting release](https://github.com/DataDog/ruby-dev-builder/releases/tag/v20250214.093100) You may spot that 'ubuntu-24.04-arm' is missing: those runners were having a bad day when I was testing (segfaults and whatnot, often not even in Ruby but also in the rust compiler) so I removed them temporarily to be able to do a full green run. --- .github/workflows/build.yml | 84 ++++++++++++++++++++++++++++++------- 1 file changed, 69 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 004cd03..b9f4556 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,6 +1,12 @@ name: CRuby Dev Builds +permissions: + contents: write on: workflow_dispatch: + inputs: + skip_slow: + type: boolean + default: false push: tags: - '*' @@ -11,8 +17,11 @@ jobs: name: Check if the latest ruby commit is already built runs-on: ubuntu-latest outputs: - should_build: ${{ steps.check_commit.outputs.result }} + should_build: ${{ steps.check_commit.outputs.should_build }} commit: ${{ steps.latest_commit.outputs.commit }} + previous_release: ${{ steps.check_commit.outputs.previous_release }} + build_matrix: ${{ steps.matrix.outputs.build_matrix }} + reuse_matrix: ${{ steps.matrix.outputs.reuse_matrix }} steps: - name: Clone ruby uses: actions/checkout@v4 @@ -32,15 +41,32 @@ jobs: const latestDevCommit = "${{ steps.latest_commit.outputs.commit }}" const { owner, repo } = context.repo let { data: release } = await github.rest.repos.getLatestRelease({ owner, repo }) - const latestReleaseCommit = release.body.split('@')[1] + const firstLine = release.body.split('\n')[0] + const latestReleaseCommit = firstLine.split('@')[1] console.log(`Latest release commit: ${latestReleaseCommit}`) console.log(`Latest ruby commit: ${latestDevCommit}`) - if (latestReleaseCommit === latestDevCommit) { - return 'false' - } else { - return 'true' - } - result-encoding: string + core.setOutput('should_build', latestReleaseCommit !== latestDevCommit) + core.setOutput('previous_release', release.tag_name) + - name: Compute build and reuse matrix + uses: actions/github-script@v7 + id: matrix + with: + script: | + const osList = ['ubuntu-20.04', 'ubuntu-22.04', 'ubuntu-24.04', 'ubuntu-22.04-arm', 'ubuntu-24.04-arm', 'macos-13', 'macos-14'] + const skipSlow = "${{ github.event_name == 'workflow_dispatch' && github.event.inputs.skip_slow == 'true' }}" + const buildMatrix = JSON.stringify({ + os: osList, + name: skipSlow === 'false' ? ['head', 'debug'] : ['head'], + ...(skipSlow === 'false' && { include: [{ os: 'ubuntu-24.04', name: 'asan' }] }) + }) + core.setOutput('build_matrix', buildMatrix) + const reuseMatrix = JSON.stringify({ + os: osList, + name: ['debug'], + include: [{ os: 'ubuntu-24.04', name: 'asan' }] + }) + core.setOutput('reuse_matrix', reuseMatrix) + console.log(`build_matrix: ${buildMatrix}, reuse_matrix: ${reuseMatrix}`) release: name: Create GitHub Release @@ -64,6 +90,11 @@ jobs: tag=$(basename "${{ github.ref }}") fi echo "tag=$tag" >> $GITHUB_OUTPUT + - name: Set release description to built hash + run: echo "ruby/ruby@${{ needs.prepare.outputs.commit }}" >> release-description.md + - name: Append note if buils were reused + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.skip_slow == 'true' }} + run: echo "Skipped building and reused builds from ${{ needs.prepare.outputs.previous_release }} for debug and asan rubies" >> release-description.md - name: Create Release env: GH_TOKEN: ${{ github.token }} @@ -71,17 +102,13 @@ jobs: run: | tag="${{ steps.tag.outputs.tag }}" body="ruby/ruby@${{ needs.prepare.outputs.commit }}" - gh release create --draft "$tag" --title "$tag" --notes "$body" + gh release create --draft "$tag" --title "$tag" --notes-file release-description.md build: needs: [prepare, release] strategy: fail-fast: false - matrix: - os: [ ubuntu-20.04, ubuntu-22.04, ubuntu-24.04, ubuntu-22.04-arm, ubuntu-24.04-arm, macos-13, macos-14 ] - name: [ head, debug ] - include: - - { os: ubuntu-24.04, name: asan } + matrix: ${{ fromJson(needs.prepare.outputs.build_matrix) }} runs-on: ${{ matrix.os }} steps: - name: Clone ruby @@ -205,9 +232,36 @@ jobs: GH_REPO: ${{ github.repository }} run: gh release upload "${{ needs.release.outputs.tag }}" "ruby-${{ matrix.name }}-${{ steps.platform.outputs.platform }}.tar.gz" + reuse-slow: + needs: [prepare, release] + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.prepare.outputs.reuse_matrix) }} + runs-on: ${{ matrix.os }} + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + steps: + - name: Set platform + id: platform + run: | + platform=${{ matrix.os }} + platform=${platform/macos-13/macos-latest} + platform=${platform/macos-14/macos-13-arm64} + platform=${platform/%-arm/-arm64} + echo "platform=$platform" >> $GITHUB_OUTPUT + - name: Download binaries from previous release + run: gh release download "${{ needs.prepare.outputs.previous_release }}" --pattern "ruby-${{ matrix.name }}-${{ steps.platform.outputs.platform }}.tar.gz" + - name: Re-upload Binaries + run: gh release upload "${{ needs.release.outputs.tag }}" "ruby-${{ matrix.name }}-${{ steps.platform.outputs.platform }}.tar.gz" + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.skip_slow == 'true' }} + - name: (Empty step for when reuse is not applied) + run: echo "Not reusing binaries. This step is a no-op." # We can't skip the whole job as publish depends on it, but we skip the uploading + if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.skip_slow == 'true') }} + publish: name: Publish Release - needs: [release, build] + needs: [release, build, reuse-slow] runs-on: ubuntu-latest steps: - name: Publish Release From ae399e4d97e3f621eeb9a114002737fbc5328e92 Mon Sep 17 00:00:00 2001 From: Ivo Anjo Date: Mon, 17 Feb 2025 11:17:27 +0000 Subject: [PATCH 2/4] Break out two branches in condition --- .github/workflows/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b9f4556..88d46f1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -54,11 +54,11 @@ jobs: script: | const osList = ['ubuntu-20.04', 'ubuntu-22.04', 'ubuntu-24.04', 'ubuntu-22.04-arm', 'ubuntu-24.04-arm', 'macos-13', 'macos-14'] const skipSlow = "${{ github.event_name == 'workflow_dispatch' && github.event.inputs.skip_slow == 'true' }}" - const buildMatrix = JSON.stringify({ - os: osList, - name: skipSlow === 'false' ? ['head', 'debug'] : ['head'], - ...(skipSlow === 'false' && { include: [{ os: 'ubuntu-24.04', name: 'asan' }] }) - }) + const buildMatrix = JSON.stringify( + skipSlow === 'false' ? + { os: osList, name: ['head', 'debug'], include: [{ os: 'ubuntu-24.04', name: 'asan' }] } : + { os: osList, name: ['head'] } + ) core.setOutput('build_matrix', buildMatrix) const reuseMatrix = JSON.stringify({ os: osList, From 809d82a2150712ac9ed982cd748a5676275f2c83 Mon Sep 17 00:00:00 2001 From: Ivo Anjo Date: Mon, 17 Feb 2025 11:23:26 +0000 Subject: [PATCH 3/4] Only download binaries when we're going to need them --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 88d46f1..2ed62b0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -252,6 +252,7 @@ jobs: echo "platform=$platform" >> $GITHUB_OUTPUT - name: Download binaries from previous release run: gh release download "${{ needs.prepare.outputs.previous_release }}" --pattern "ruby-${{ matrix.name }}-${{ steps.platform.outputs.platform }}.tar.gz" + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.skip_slow == 'true' }} - name: Re-upload Binaries run: gh release upload "${{ needs.release.outputs.tag }}" "ruby-${{ matrix.name }}-${{ steps.platform.outputs.platform }}.tar.gz" if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.skip_slow == 'true' }} From 9efa7556fde1f8bf232282fdb41f06be7cb25094 Mon Sep 17 00:00:00 2001 From: Ivo Anjo Date: Mon, 17 Feb 2025 12:27:36 +0000 Subject: [PATCH 4/4] Add no-op entry for reuse matrix when not skipping slow builds --- .github/workflows/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2ed62b0..d77f429 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -60,11 +60,11 @@ jobs: { os: osList, name: ['head'] } ) core.setOutput('build_matrix', buildMatrix) - const reuseMatrix = JSON.stringify({ - os: osList, - name: ['debug'], - include: [{ os: 'ubuntu-24.04', name: 'asan' }] - }) + const reuseMatrix = JSON.stringify( + skipSlow === 'false' ? + { os: ['ubuntu-latest'], name: ['noop'] } : // GitHub doesn't like having an empty matrix, skips jobs that depend on reuse-slow + { os: osList, name: ['debug'], include: [{ os: 'ubuntu-24.04', name: 'asan' }] } + ) core.setOutput('reuse_matrix', reuseMatrix) console.log(`build_matrix: ${buildMatrix}, reuse_matrix: ${reuseMatrix}`)