Skip to content

Commit 2b6c4de

Browse files
LaurentGoderreLaurentGoderre
committed
Isolate signing step by moving to its own job
1 parent 190d641 commit 2b6c4de

File tree

1 file changed

+193
-87
lines changed

1 file changed

+193
-87
lines changed

‎.github/workflows/build.yml

Lines changed: 193 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,13 @@ on:
2020
- '' # without this, it's technically "required" 🙃
2121
- 2022
2222
- 2019
23+
pruneArtifact:
24+
required: true
25+
type: choice
26+
options:
27+
- "true"
28+
- "false"
29+
default: "true"
2330
run-name: '${{ inputs.bashbrewArch }}: ${{ inputs.firstTag }} (${{ inputs.buildId }})'
2431
permissions:
2532
contents: read
@@ -44,7 +51,9 @@ jobs:
4451
build:
4552
name: Build ${{ inputs.buildId }}
4653
outputs:
47-
shouldSign: ${{ steps.json.outputs.shouldSign }}
54+
buildJson: ${{ steps.json.outputs.json }}
55+
artifactId: ${{ steps.oci.outputs.artifact-id }}
56+
sha256: ${{ steps.checksum.outputs.sha256 }}
4857
runs-on: ${{ inputs.bashbrewArch == 'windows-amd64' && format('windows-{0}', inputs.windowsVersion) || 'ubuntu-latest' }}
4958
steps:
5059

@@ -156,136 +165,233 @@ jobs:
156165
fi
157166
eval "$shell"
158167
159-
# TODO signing prototype (see above where "shouldSign" is populated)
168+
- name: Generate Checksum
169+
id: checksum
170+
run: |
171+
cd build
172+
tar -cvf temp.tar -C temp .
173+
echo "sha256=$(sha256sum temp.tar)" >> "$GITHUB_OUTPUT"
174+
- name: Stage artifact
175+
id: oci
176+
uses: actions/upload-artifact@v4
177+
with:
178+
name: build-oci
179+
path: |
180+
build/temp.tar*
181+
retention-days: 5
182+
183+
sign:
184+
name: Sign
185+
needs: build
186+
if: fromJSON(needs.build.outputs.buildJson).shouldSign
187+
runs-on: ubuntu-latest
188+
permissions:
189+
contents: read
190+
actions: write # for https://github.com/andymckay/cancel-action (see usage below)
191+
id-token: write # for AWS KMS signing (see usage below)
192+
steps:
193+
- uses: actions/checkout@v4
194+
with:
195+
sparse-checkout-cone-mode: 'false'
196+
sparse-checkout: |
197+
.scripts/oci.jq
198+
.scripts/provenance.jq
199+
- name: Download a single artifact
200+
uses: actions/download-artifact@v4
201+
with:
202+
name: build-oci
203+
- name: Verify artifact
204+
run: |
205+
echo "${{ needs.build.outputs.sha256 }}" sha256sum -c
206+
mkdir -p temp
207+
tar -xvf temp.tar -C temp
160208
- name: Configure AWS (for signing)
161-
if: fromJSON(steps.json.outputs.json).shouldSign
162209
# https://github.com/aws-actions/configure-aws-credentials/releases
163210
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
164211
with:
165-
# TODO drop "subset" (github.ref_name == "main" && ... || ...)
166-
aws-region: ${{ contains(fromJSON('["main","subset"]'), github.ref_name) && secrets.AWS_KMS_PROD_REGION || secrets.AWS_KMS_STAGE_REGION }}
167-
role-to-assume: ${{ contains(fromJSON('["main","subset"]'), github.ref_name) && secrets.AWS_KMS_PROD_ROLE_ARN || secrets.AWS_KMS_STAGE_ROLE_ARN }}
168-
# TODO figure out if there's some way we could make our secrets ternaries here more DRY without major headaches 🙈
212+
aws-region: ${{ secrets.AWS_KMS_REGION }}
213+
role-to-assume: ${{ secrets.AWS_KMS_ROLE_ARN }}
214+
- name: Generate Provenance
215+
env:
216+
buildJson: ${{ needs.build.outputs.buildJson }}
217+
GITHUB_CONTEXT: ${{ toJson(github) }}
218+
run: |
219+
image-digest() {
220+
local dir="$1/blobs"
221+
img=$(
222+
grep -R --include "*" '"mediaType":\s"application/vnd.oci.image.layer.' "$dir" \
223+
| head -n 1 \
224+
| cut -d ':' -f1
225+
)
226+
[ "$(cat $img | jq -r '.mediaType')" = "application/vnd.oci.image.manifest.v1+json" ] || exit 1
227+
echo $img | rev | cut -d '/' -f2,1 --output-delimiter ':' | rev
228+
}
229+
230+
digest=$(image-digest temp)
231+
232+
echo $buildJson | jq -L.scripts --argjson github '${{ env.GITHUB_CONTEXT }}' --argjson runner '${{ toJson(runner) }}' --arg digest ${digest} '
233+
include "provenance";
234+
github_actions_provenance($github; $runner; $digest)
235+
' >> provenance.json
169236
- name: Sign
170-
if: fromJSON(steps.json.outputs.json).shouldSign
171237
env:
172-
AWS_KMS_REGION: ${{ contains(fromJSON('["main","subset"]'), github.ref_name) && secrets.AWS_KMS_PROD_REGION || secrets.AWS_KMS_STAGE_REGION }}
173-
AWS_KMS_KEY_ARN: ${{ contains(fromJSON('["main","subset"]'), github.ref_name) && secrets.AWS_KMS_PROD_KEY_ARN || secrets.AWS_KMS_STAGE_KEY_ARN }}
238+
AWS_KMS_REGION: ${{ secrets.AWS_KMS_REGION }}
239+
AWS_KMS_KEY_ARN: ${{ secrets.AWS_KMS_KEY_ARN }}
240+
241+
DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
242+
DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
174243
run: |
175-
cd build
244+
validate-oci-layout() {
245+
local dir="$1"
246+
jq -L.scripts -s '
247+
include "oci";
248+
validate_oci_layout | true
249+
' "$dir/oci-layout" "$dir/index.json" || return "$?"
250+
local manifest
251+
manifest="$dir/blobs/$(jq -r '.manifests[0].digest | sub(":"; "/")' "$dir/index.json")" || return "$?"
252+
jq -L.scripts -s '
253+
include "oci";
254+
if length != 1 then
255+
error("unexpected image index document count: \(length)")
256+
else .[0] end
257+
| validate_oci_index
176258
177-
args=(
259+
# TODO more validation?
260+
' "$manifest" || return "$?"
261+
}
262+
263+
dockerArgs=(
178264
--interactive
179265
--rm
180266
--read-only
181267
--workdir /tmp # see "--tmpfs" below (TODO the signer currently uses PWD as TMPDIR -- something to fix in the future so we can drop this --workdir and only keep --tmpfs perhaps adding --env TMPDIR=/tmp if necessary)
182268
)
183269
if [ -t 0 ] && [ -t 1 ]; then
184-
args+=( --tty )
270+
dockerArgs+=( --tty )
185271
fi
186272
187273
user="$(id -u)"
188-
args+=( --tmpfs "/tmp:uid=$user" )
274+
dockerArgs+=( --tmpfs "/tmp:uid=$user" )
189275
user+=":$(id -g)"
190-
args+=( --user "$user" )
276+
dockerArgs+=( --user "$user" )
191277
192278
awsEnvs=( "${!AWS_@}" )
193-
args+=( "${awsEnvs[@]/#/--env=}" )
194-
195-
# some very light assumption verification (see TODO in --mount below)
196-
validate-oci-layout() {
197-
local dir="$1"
198-
jq -s '
199-
if length != 1 then
200-
error("unexpected 'oci-layout' document count: " + length)
201-
else .[0] end
202-
| if .imageLayoutVersion != "1.0.0" then
203-
error("unsupported imageLayoutVersion: " + .imageLayoutVersion)
204-
else . end
205-
' "$dir/oci-layout" || return "$?"
206-
jq -s '
207-
if length != 1 then
208-
error("unexpected 'index.json' document count: " + length)
209-
else .[0] end
279+
dockerArgs+=( "${awsEnvs[@]/#/--env=}" )
210280
211-
| if .schemaVersion != 2 then
212-
error("unsupported schemaVersion: " + .schemaVersion)
213-
else . end
214-
| if .mediaType != "application/vnd.oci.image.index.v1+json" and .mediaType then # TODO drop the second half of this validation: https://github.com/moby/buildkit/issues/4595
215-
error("unsupported index mediaType: " + .mediaType)
216-
else . end
217-
| if .manifests | length != 1 then
218-
error("expected only one manifests entry, not " + (.manifests | length))
219-
else . end
220-
221-
| .manifests[0] |= (
222-
if .mediaType != "application/vnd.oci.image.index.v1+json" then
223-
error("unsupported descriptor mediaType: " + .mediaType)
224-
else . end
225-
# TODO validate .digest somehow (`crane validate`?) - would also be good to validate all descriptors recursively
226-
| if .size < 0 then
227-
error("invalid descriptor size: " + .size)
228-
else . end
229-
)
230-
' "$dir/index.json" || return "$?"
231-
local manifest
232-
manifest="$dir/blobs/$(jq -r '.manifests[0].digest | sub(":"; "/")' "$dir/index.json")" || return "$?"
233-
jq -s '
234-
if length != 1 then
235-
error("unexpected image index document count: " + length)
236-
else .[0] end
237-
| if .schemaVersion != 2 then
238-
error("unsupported schemaVersion: " + .schemaVersion)
239-
else . end
240-
| if .mediaType != "application/vnd.oci.image.index.v1+json" then
241-
error("unsupported image index mediaType: " + .mediaType)
242-
else . end
243-
244-
# TODO more validation?
245-
' "$manifest" || return "$?"
246-
}
247281
validate-oci-layout temp
248282
249-
mkdir signed
283+
# Login to Docker Hub
284+
export DOCKER_CONFIG="$PWD/.docker"
285+
mkdir "$DOCKER_CONFIG"
286+
trap 'find "$DOCKER_CONFIG" -type f -exec shred -fuvz "{}" + || :; rm -rf "$DOCKER_CONFIG"' EXIT
287+
docker login --username "$DOCKER_HUB_USERNAME" --password-stdin <<<"$DOCKER_HUB_PASSWORD"
288+
unset DOCKER_HUB_USERNAME DOCKER_HUB_PASSWORD
250289
251-
args+=(
252-
--mount "type=bind,src=$PWD/temp,dst=/doi-build/unsigned" # TODO this currently assumes normalized_builder == "buildkit" and !should_use_docker_buildx_driver -- we need to factor that in later (although this signs the attestations, not the image, so buildkit/buildx is the only builder whose output we *can* sign right now)
253-
--mount "type=bind,src=$PWD/signed,dst=/doi-build/signed"
290+
# Create signatures
291+
dockerArgs+=(
292+
--mount "type=bind,src=$PWD/temp,dst=/doi-build/image" # TODO this currently assumes normalized_builder == "buildkit" and !should_use_docker_buildx_driver -- we need to factor that in later (although this signs the attestations, not the image, so buildkit/buildx is the only builder whose output we *can* sign right now)
293+
--mount "type=bind,src=$PWD/provenance.json,dst=/doi-build/provenance.json"
294+
--mount "type=bind,src=$PWD/.docker,dst=/.docker"
254295
255296
# https://explore.ggcr.dev/?repo=docker/image-signer-verifier
256-
docker/image-signer-verifier:0.3.3@sha256:a5351e6495596429bacea85fbf8f41a77ce7237c26c74fd7c3b94c3e6d409c82
257-
258-
sign
297+
"$IMAGE_SIGNER"
298+
)
259299
260-
--envelope-style oci-content-descriptor
300+
kmsArg=(
301+
# kms key used to sign attestation artifacts
302+
--kms="AWS"
303+
--kms-region="$AWS_KMS_REGION"
304+
--kms-key-ref="$AWS_KMS_KEY_ARN"
261305
262-
--aws_region "$AWS_KMS_REGION"
263-
--aws_arn "awskms:///$AWS_KMS_KEY_ARN"
306+
--referrers-dest="$REFERRERS_REPO" # repo to store attestation artifacts and provenance
307+
)
264308
265-
--input oci:///doi-build/unsigned
266-
--output oci:///doi-build/signed
309+
# Sign buildkit statements
310+
signArgs=(
311+
${kmsArg[@]}
312+
--input=oci:///doi-build/image
313+
--keep=true # keep preserves the unsigned attestations generated by buildkit
267314
)
268315
269-
docker run "${args[@]}"
316+
docker run "${dockerArgs[@]}" sign "${signArgs[@]}"
270317
271-
validate-oci-layout signed
318+
# Attach and sign provenance
319+
provArgs=(
320+
${kmsArg[@]}
321+
--image=oci:///doi-build/image
322+
--statement="/doi-build/provenance.json"
323+
)
324+
docker run "${dockerArgs[@]}" attest "${provArgs[@]}"
325+
326+
push:
327+
name: Push
328+
needs:
329+
- build
330+
- sign
331+
# - verify
332+
if: ${{ always() }}
333+
runs-on: ubuntu-latest
334+
steps:
335+
- run: ${{ (needs.sign.result == 'skipped' && needs.build.result == 'success') || needs.verify.result == 'success' || 'exit 1' }}
336+
- name: Download a single artifact
337+
uses: actions/download-artifact@v4
338+
with:
339+
name: build-oci
340+
- name: Verify artifact
341+
run: |
342+
echo "${{ needs.build.outputs.sha256 }}" sha256sum -c
343+
mkdir -p temp
344+
tar -xvf temp.tar -C temp
345+
- name: Tools
346+
run: |
347+
mkdir .gha-bin
348+
echo "$PWD/.gha-bin" >> "$GITHUB_PATH"
272349
273-
# TODO validate that "signed" still has all the original layer blobs from "temp" (ie, that the attestation manifest *just* has some new layers and everything else is unchanged)
350+
case "${RUNNER_ARCH}" in \
351+
X64) ARCH='amd64';; \
352+
esac
274353
275-
rm -rf temp
276-
mv signed temp
354+
_download() {
355+
local target="$1"; shift
356+
local url="$1"; shift
357+
wget --timeout=5 -O "$target" "$url" --progress=dot:giga
358+
}
277359
360+
# https://doi-janky.infosiftr.net/job/wip/job/crane
361+
_download ".gha-bin/crane" "https://doi-janky.infosiftr.net/job/wip/job/crane/lastSuccessfulBuild/artifact/crane-$ARCH"
362+
# TODO checksum verification ("checksums.txt")
363+
chmod +x ".gha-bin/crane"
364+
".gha-bin/crane" version
278365
- name: Push
279366
env:
280367
DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
281368
DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
369+
buildJson: ${{ needs.build.outputs.buildJson }}
282370
run: |
283371
export DOCKER_CONFIG="$PWD/.docker"
284372
mkdir "$DOCKER_CONFIG"
285373
trap 'find "$DOCKER_CONFIG" -type f -exec shred -fuvz "{}" + || :; rm -rf "$DOCKER_CONFIG"' EXIT
286374
docker login --username "$DOCKER_HUB_USERNAME" --password-stdin <<<"$DOCKER_HUB_PASSWORD"
287375
unset DOCKER_HUB_USERNAME DOCKER_HUB_PASSWORD
288376
289-
cd build
290-
shell="$(jq <<<"$json" -r '.commands.push')"
377+
shell="$(jq <<<"$buildJson" -r '.commands.push')"
291378
eval "$shell"
379+
380+
clean:
381+
name: Cleanup
382+
needs:
383+
- build
384+
# - verify
385+
- push
386+
if: ${{ always() }}
387+
runs-on: ubuntu-latest
388+
steps:
389+
- name: Clean Up Artifact
390+
if: ${{ inputs.pruneArtifact == 'true' }}
391+
run: |
392+
curl -L \
393+
-X DELETE \
394+
-H "Accept: application/vnd.github+json" \
395+
-H "Authorization: Bearer ${{ github.token }}" \
396+
-H "X-GitHub-Api-Version: 2022-11-28" \
397+
https://api.github.com/repos/${{ github.repository }}/actions/artifacts/${{ needs.build.outputs.artifactId }}

0 commit comments

Comments
 (0)