Skip to content

Commit fd32d2b

Browse files
LaurentGoderreLaurentGoderre
committed
Isolate signing step by moving to its own job
1 parent e8fa03d commit fd32d2b

File tree

1 file changed

+196
-82
lines changed

1 file changed

+196
-82
lines changed

‎.github/workflows/build.yml‎

Lines changed: 196 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@ on:
2020
- '' # without this, it's technically "required" 🙃
2121
- 2022
2222
- 2019
23+
pruneArtifact:
24+
required: true
25+
type: boolean
26+
default: true
2327
run-name: '${{ inputs.bashbrewArch }}: ${{ inputs.firstTag }} (${{ inputs.buildId }})'
2428
permissions:
2529
contents: read
@@ -36,15 +40,17 @@ env:
3640

3741
# the image we'll run to access the signing tool
3842
# https://explore.ggcr.dev/?repo=docker/image-signer-verifier
39-
IMAGE_SIGNER: 'docker/image-signer-verifier:0.6.3@sha256:d7930e03b48064b6c2d9f9c0421f65a5dacc6aba7d91b9d2e320d2976becfeac'
43+
IMAGE_SIGNER: 'docker/image-signer-verifier:0.6.9@sha256:e38be2b9e6f010cf3432a2772b0e800feee572f7733c6df81e21293cb3e977e0'
4044

4145
# Docker Hub repository we'll push the (signed) attestation artifacts to
4246
REFERRERS_REPO: oisupport/referrers
4347
jobs:
4448
build:
4549
name: Build ${{ inputs.buildId }}
4650
outputs:
47-
shouldSign: ${{ steps.json.outputs.shouldSign }}
51+
json: ${{ steps.json.outputs.json }}
52+
artifactUrl: ${{ steps.oci.outputs.artifact-url }}
53+
artifactSha256: ${{ steps.sha256.outputs.sha256 }}
4854
runs-on: ${{ inputs.bashbrewArch == 'windows-amd64' && format('windows-{0}', inputs.windowsVersion) || 'ubuntu-latest' }}
4955
steps:
5056

@@ -156,135 +162,243 @@ jobs:
156162
fi
157163
eval "$shell"
158164
159-
# TODO signing prototype (see above where "shouldSign" is populated)
165+
# save the build as an "artifact" so we can sign it in a separate step (and minimize the exposure of the signing credentials); also save a checksum so we can be sure it transmits between the steps accurately
166+
# TODO what do we do here for "classic" builds like Windows? (no "temp" OCI layout currently)
167+
- name: Generate Artifact
168+
id: sha256
169+
run: |
170+
tar -cvf temp.tar -C build temp
171+
sha256="$(sha256sum temp.tar | cut -d' ' -f1)"
172+
echo "sha256=$sha256" >> "$GITHUB_OUTPUT"
173+
- name: Upload Artifact
174+
id: oci
175+
uses: actions/upload-artifact@v4
176+
with:
177+
name: build-oci
178+
path: |
179+
temp.tar
180+
retention-days: 5
181+
182+
sign:
183+
name: Sign
184+
needs: build
185+
if: fromJSON(needs.build.outputs.json).shouldSign
186+
runs-on: ubuntu-latest
187+
permissions:
188+
contents: read
189+
id-token: write # for AWS KMS signing (see usage below)
190+
steps:
191+
- uses: actions/checkout@v4
192+
with:
193+
sparse-checkout-cone-mode: 'false'
194+
sparse-checkout: |
195+
.scripts/oci.jq
196+
.scripts/provenance.jq
197+
- name: Download Artifact
198+
uses: actions/download-artifact@v4
199+
with:
200+
name: build-oci
201+
- name: Extract Artifact
202+
env:
203+
sha256: ${{ needs.build.outputs.artifactSha256 }}
204+
run: |
205+
sha256sum <<<"$sha256 *temp.tar" --strict --check -
206+
tar -xvf temp.tar
207+
[ -d temp ] # basic "valid JSON" check
160208
- name: Configure AWS (for signing)
161-
if: fromJSON(steps.json.outputs.json).shouldSign
162209
# https://github.com/aws-actions/configure-aws-credentials/releases
163210
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
164211
with:
165212
aws-region: ${{ github.ref_name == 'main' && secrets.AWS_KMS_PROD_REGION || secrets.AWS_KMS_STAGE_REGION }}
166213
role-to-assume: ${{ github.ref_name == 'main' && secrets.AWS_KMS_PROD_ROLE_ARN || secrets.AWS_KMS_STAGE_ROLE_ARN }}
167-
# TODO figure out if there's some way we could make our secrets ternaries here more DRY without major headaches 🙈
214+
- name: Generate Provenance
215+
env:
216+
json: ${{ needs.build.outputs.json }}
217+
GITHUB_CONTEXT: ${{ toJson(github) }}
218+
run: |
219+
image-digest() {
220+
local dir="$1/blobs"
221+
img=$(
222+
grep -R --include "*" '"mediaType":\s"application/vnd.oci.image.layer.' "$dir" \
223+
| head -n 1 \
224+
| cut -d ':' -f1
225+
)
226+
[ "$(cat $img | jq -r '.mediaType')" = "application/vnd.oci.image.manifest.v1+json" ] || exit 1
227+
echo $img | rev | cut -d '/' -f2,1 --output-delimiter ':' | rev
228+
}
229+
230+
digest=$(image-digest temp)
231+
232+
echo $json | jq -L.scripts --argjson github '${{ env.GITHUB_CONTEXT }}' --argjson runner '${{ toJson(runner) }}' --arg digest ${digest} '
233+
include "provenance";
234+
github_actions_provenance($github; $runner; $digest)
235+
' >> provenance.json
168236
- name: Sign
169-
if: fromJSON(steps.json.outputs.json).shouldSign
170237
env:
171238
AWS_KMS_REGION: ${{ github.ref_name == 'main' && secrets.AWS_KMS_PROD_REGION || secrets.AWS_KMS_STAGE_REGION }}
172239
AWS_KMS_KEY_ARN: ${{ github.ref_name == 'main' && secrets.AWS_KMS_PROD_KEY_ARN || secrets.AWS_KMS_STAGE_KEY_ARN }}
240+
241+
DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_SIGNING_USERNAME }}
242+
DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_SIGNING_PASSWORD }}
173243
run: |
174-
cd build
244+
validate-oci-layout() {
245+
local dir="$1"
246+
jq -L.scripts -s '
247+
include "oci";
248+
validate_oci_layout | true
249+
' "$dir/oci-layout" "$dir/index.json" || return "$?"
250+
local manifest
251+
manifest="$dir/blobs/$(jq -r '.manifests[0].digest | sub(":"; "/")' "$dir/index.json")" || return "$?"
252+
jq -L.scripts -s '
253+
include "oci";
254+
if length != 1 then
255+
error("unexpected image index document count: \(length)")
256+
else .[0] end
257+
| validate_oci_index
175258
176-
args=(
259+
# TODO more validation?
260+
' "$manifest" || return "$?"
261+
}
262+
263+
dockerArgs=(
177264
--interactive
178265
--rm
179266
--read-only
180267
--workdir /tmp # see "--tmpfs" below (TODO the signer currently uses PWD as TMPDIR -- something to fix in the future so we can drop this --workdir and only keep --tmpfs perhaps adding --env TMPDIR=/tmp if necessary)
181268
)
182269
if [ -t 0 ] && [ -t 1 ]; then
183-
args+=( --tty )
270+
dockerArgs+=( --tty )
184271
fi
185272
186273
user="$(id -u)"
187-
args+=( --tmpfs "/tmp:uid=$user" )
274+
dockerArgs+=( --tmpfs "/tmp:uid=$user" )
188275
user+=":$(id -g)"
189-
args+=( --user "$user" )
276+
dockerArgs+=( --user "$user" )
190277
191278
awsEnvs=( "${!AWS_@}" )
192-
args+=( "${awsEnvs[@]/#/--env=}" )
193-
194-
# some very light assumption verification (see TODO in --mount below)
195-
validate-oci-layout() {
196-
local dir="$1"
197-
jq -s '
198-
if length != 1 then
199-
error("unexpected 'oci-layout' document count: " + length)
200-
else .[0] end
201-
| if .imageLayoutVersion != "1.0.0" then
202-
error("unsupported imageLayoutVersion: " + .imageLayoutVersion)
203-
else . end
204-
' "$dir/oci-layout" || return "$?"
205-
jq -s '
206-
if length != 1 then
207-
error("unexpected 'index.json' document count: " + length)
208-
else .[0] end
209-
210-
| if .schemaVersion != 2 then
211-
error("unsupported schemaVersion: " + .schemaVersion)
212-
else . end
213-
| if .mediaType != "application/vnd.oci.image.index.v1+json" and .mediaType then # TODO drop the second half of this validation: https://github.com/moby/buildkit/issues/4595
214-
error("unsupported index mediaType: " + .mediaType)
215-
else . end
216-
| if .manifests | length != 1 then
217-
error("expected only one manifests entry, not " + (.manifests | length))
218-
else . end
219-
220-
| .manifests[0] |= (
221-
if .mediaType != "application/vnd.oci.image.index.v1+json" then
222-
error("unsupported descriptor mediaType: " + .mediaType)
223-
else . end
224-
# TODO validate .digest somehow (`crane validate`?) - would also be good to validate all descriptors recursively
225-
| if .size < 0 then
226-
error("invalid descriptor size: " + .size)
227-
else . end
228-
)
229-
' "$dir/index.json" || return "$?"
230-
local manifest
231-
manifest="$dir/blobs/$(jq -r '.manifests[0].digest | sub(":"; "/")' "$dir/index.json")" || return "$?"
232-
jq -s '
233-
if length != 1 then
234-
error("unexpected image index document count: " + length)
235-
else .[0] end
236-
| if .schemaVersion != 2 then
237-
error("unsupported schemaVersion: " + .schemaVersion)
238-
else . end
239-
| if .mediaType != "application/vnd.oci.image.index.v1+json" then
240-
error("unsupported image index mediaType: " + .mediaType)
241-
else . end
279+
dockerArgs+=( "${awsEnvs[@]/#/--env=}" )
242280
243-
# TODO more validation?
244-
' "$manifest" || return "$?"
245-
}
246281
validate-oci-layout temp
247282
248-
mkdir signed
283+
# Login to Docker Hub
284+
export DOCKER_CONFIG="$PWD/.docker"
285+
mkdir "$DOCKER_CONFIG"
286+
trap 'find "$DOCKER_CONFIG" -type f -exec shred -fuvz "{}" + || :; rm -rf "$DOCKER_CONFIG"' EXIT
287+
docker login --username "$DOCKER_HUB_USERNAME" --password-stdin <<<"$DOCKER_HUB_PASSWORD"
288+
unset DOCKER_HUB_USERNAME DOCKER_HUB_PASSWORD
249289
250-
args+=(
251-
--mount "type=bind,src=$PWD/temp,dst=/doi-build/unsigned" # TODO this currently assumes normalized_builder == "buildkit" and !should_use_docker_buildx_driver -- we need to factor that in later (although this signs the attestations, not the image, so buildkit/buildx is the only builder whose output we *can* sign right now)
252-
--mount "type=bind,src=$PWD/signed,dst=/doi-build/signed"
290+
# Create signatures
291+
dockerArgs+=(
292+
--mount "type=bind,src=$PWD/temp,dst=/doi-build/image,ro" # TODO this currently assumes normalized_builder == "buildkit" and !should_use_docker_buildx_driver -- we need to factor that in later (although this signs the attestations, not the image, so buildkit/buildx is the only builder whose output we *can* sign right now)
293+
--mount "type=bind,src=$PWD/provenance.json,dst=/doi-build/provenance.json,ro"
294+
--mount "type=bind,src=$DOCKER_CONFIG,dst=/docker-config,ro"
295+
--env DOCKER_CONFIG=/docker-config # TODO verify that image-signer supports this environment variable correctly
253296
254297
# https://explore.ggcr.dev/?repo=docker/image-signer-verifier
255-
docker/image-signer-verifier:0.3.3@sha256:a5351e6495596429bacea85fbf8f41a77ce7237c26c74fd7c3b94c3e6d409c82
256-
257-
sign
298+
"$IMAGE_SIGNER"
299+
)
258300
259-
--envelope-style oci-content-descriptor
301+
kmsArg=(
302+
# kms key used to sign attestation artifacts
303+
--kms="AWS"
304+
--kms-region="$AWS_KMS_REGION"
305+
--kms-key-ref="$AWS_KMS_KEY_ARN"
260306
261-
--aws_region "$AWS_KMS_REGION"
262-
--aws_arn "awskms:///$AWS_KMS_KEY_ARN"
307+
--referrers-dest="$REFERRERS_REPO" # repo to store attestation artifacts and provenance
308+
)
263309
264-
--input oci:///doi-build/unsigned
265-
--output oci:///doi-build/signed
310+
# Sign buildkit statements
311+
signArgs=(
312+
"${kmsArg[@]}"
313+
--input=oci:///doi-build/image
314+
--keep=true # keep preserves the unsigned attestations generated by buildkit
266315
)
267316
268-
docker run "${args[@]}"
317+
docker run "${dockerArgs[@]}" sign "${signArgs[@]}"
269318
270-
validate-oci-layout signed
319+
# Attach and sign provenance
320+
provArgs=(
321+
"${kmsArg[@]}"
322+
--image=oci:///doi-build/image
323+
--statement="/doi-build/provenance.json"
324+
)
325+
docker run "${dockerArgs[@]}" attest "${provArgs[@]}"
326+
327+
push:
328+
name: Push
329+
needs:
330+
- build
331+
- sign
332+
# - verify
333+
if: ${{ always() }}
334+
runs-on: ubuntu-latest
335+
steps:
336+
- run: ${{ (needs.sign.result == 'skipped' && needs.build.result == 'success') || needs.verify.result == 'success' || 'exit 1' }}
337+
- name: Download Artifact
338+
uses: actions/download-artifact@v4
339+
with:
340+
name: build-oci
341+
- name: Extract Artifact
342+
env:
343+
sha256: ${{ needs.build.outputs.artifactSha256 }}
344+
run: |
345+
sha256sum <<<"$sha256 *temp.tar" --strict --check -
346+
tar -xvf temp.tar
347+
[ -d temp ] # basic "valid JSON" check
348+
- name: Tools
349+
run: |
350+
mkdir .gha-bin
351+
echo "$PWD/.gha-bin" >> "$GITHUB_PATH"
271352
272-
# TODO validate that "signed" still has all the original layer blobs from "temp" (ie, that the attestation manifest *just* has some new layers and everything else is unchanged)
353+
case "${RUNNER_ARCH}" in \
354+
X64) ARCH='amd64';; \
355+
esac
273356
274-
rm -rf temp
275-
mv signed temp
357+
_download() {
358+
local target="$1"; shift
359+
local url="$1"; shift
360+
wget --timeout=5 -O "$target" "$url" --progress=dot:giga
361+
}
276362
363+
# https://doi-janky.infosiftr.net/job/wip/job/crane
364+
_download ".gha-bin/crane" "https://doi-janky.infosiftr.net/job/wip/job/crane/lastSuccessfulBuild/artifact/crane-$ARCH"
365+
# TODO checksum verification ("checksums.txt")
366+
chmod +x ".gha-bin/crane"
367+
".gha-bin/crane" version
277368
- name: Push
278369
env:
279370
DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
280371
DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
372+
json: ${{ needs.build.outputs.json }}
281373
run: |
282374
export DOCKER_CONFIG="$PWD/.docker"
283375
mkdir "$DOCKER_CONFIG"
284376
trap 'find "$DOCKER_CONFIG" -type f -exec shred -fuvz "{}" + || :; rm -rf "$DOCKER_CONFIG"' EXIT
285377
docker login --username "$DOCKER_HUB_USERNAME" --password-stdin <<<"$DOCKER_HUB_PASSWORD"
286378
unset DOCKER_HUB_USERNAME DOCKER_HUB_PASSWORD
287379
288-
cd build
289380
shell="$(jq <<<"$json" -r '.commands.push')"
290381
eval "$shell"
382+
383+
clean:
384+
name: Cleanup
385+
needs:
386+
- build
387+
# - verify
388+
- push
389+
if: ${{ inputs.pruneArtifact }}
390+
runs-on: ubuntu-latest
391+
steps:
392+
- name: Clean Up Artifact
393+
env:
394+
ARTIFACT_URL: ${{ needs.build.outputs.artifactUrl }}
395+
TOKEN: ${{ github.token }}
396+
run: |
397+
url="${ARTIFACT_URL/\/\/github\.com///api.github.com/repos}"
398+
url="${url/runs\/*\/artifacts/artifacts}" # Translate web URL to API url
399+
curl -L \
400+
-X DELETE \
401+
-H "Accept: application/vnd.github+json" \
402+
-H "Authorization: Bearer $TOKEN" \
403+
-H "X-GitHub-Api-Version: 2022-11-28" \
404+
"$url"

0 commit comments

Comments
 (0)