Adjust scripts/tests to generate Go code coverage data

With Go 1.20+, we can generate coverage information from binaries (not just tests), so our "canonical output" tests can *also* generate appropriate coverage information so we can see more clearly how we're doing and where we can improve.

I decided *not* to hook this up to Codecov for now, because I have not been happy with their service, but I did leave a `TODO` comment in the appropriate place for where/how we could implement similar functionality (like their "change in coverage" comments).

Author: tianon

.github/workflows/ci.yml

@@ -31,4 +31,10 @@ jobs:
           .bin/bashbrew --version
           echo "$PWD/.bin" >> "$GITHUB_PATH"
       - run: .test/test.sh
+      - uses: actions/upload-artifact@v4
+        with:
+          name: coverage
+          path: .test/coverage**
+          if-no-files-found: error
       - run: git diff --exit-code
+      # TODO download latest coverage artifacts from HEAD / PR target to emulate Codecov but without another flaky third-party service that's begging for write-access to all our repositories via a GitHub App? 👀

.gitignore

@@ -8,8 +8,8 @@ user="$(id -u):$(id -g)"
 args=(
 	--interactive --rm --init
 	--user "$user"
-	--mount "type=bind,src=$dir,dst=/app"
-	--workdir /app
+	--mount "type=bind,src=$dir,dst=$dir"
+	--workdir "$dir"
 	--tmpfs /tmp,exec
 	--env HOME=/tmp
 
@@ -20,6 +20,7 @@ args=(
 
 	--env "CGO_ENABLED=${CGO_ENABLED-0}"
 	--env "GOTOOLCHAIN=${GOTOOLCHAIN-local}"
+	--env GOCOVERDIR # https://go.dev/doc/build-cover
 	--env GODEBUG
 	--env GOFLAGS
 	--env GOOS --env GOARCH

.go-env.sh

@@ -0,0 +1 @@
+coverage**

.test/.gitignore

@@ -0,0 +1,79 @@
+[
+  {
+    "schemaVersion": 2,
+    "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json",
+    "manifests": [
+      {
+        "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+        "digest": "sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23",
+        "size": 946,
+        "annotations": {
+          "com.docker.official-images.bashbrew.arch": "windows-amd64",
+          "org.opencontainers.image.ref.name": "docker.io/tianon/test@sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23"
+        },
+        "platform": {
+          "architecture": "amd64",
+          "os": "windows",
+          "os.version": "10.0.20348.2340"
+        }
+      }
+    ],
+    "annotations": {
+      "org.opencontainers.image.ref.name": "docker.io/tianon/test@sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23"
+    }
+  },
+  {
+    "schemaVersion": 2,
+    "mediaType": "application/vnd.oci.image.index.v1+json",
+    "manifests": [
+      {
+        "mediaType": "application/vnd.oci.image.manifest.v1+json",
+        "digest": "sha256:e2fc4e5012d16e7fe466f5291c476431beaa1f9b90a5c2125b493ed28e2aba57",
+        "size": 861,
+        "annotations": {
+          "com.docker.official-images.bashbrew.arch": "amd64",
+          "org.opencontainers.image.ref.name": "docker.io/tianon/test@sha256:e2fc4e5012d16e7fe466f5291c476431beaa1f9b90a5c2125b493ed28e2aba57",
+          "org.opencontainers.image.revision": "3fb6ebca4163bf5b9cc496ac3e8f11cb1e754aee",
+          "org.opencontainers.image.source": "https://github.com/docker-library/hello-world.git#3fb6ebca4163bf5b9cc496ac3e8f11cb1e754aee:amd64/hello-world",
+          "org.opencontainers.image.url": "https://hub.docker.com/_/hello-world",
+          "org.opencontainers.image.version": "linux"
+        },
+        "platform": {
+          "architecture": "amd64",
+          "os": "linux"
+        }
+      },
+      {
+        "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+        "digest": "sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23",
+        "size": 946,
+        "annotations": {
+          "com.docker.official-images.bashbrew.arch": "windows-amd64",
+          "org.opencontainers.image.ref.name": "docker.io/tianon/test@sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23"
+        },
+        "platform": {
+          "architecture": "amd64",
+          "os": "windows",
+          "os.version": "10.0.20348.2340"
+        }
+      },
+      {
+        "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+        "digest": "sha256:3a0bd0fb5ad6dd6528dc78726b3df78e980b39b379e99c5a508904ec17cfafe5",
+        "size": 946,
+        "annotations": {
+          "com.docker.official-images.bashbrew.arch": "windows-amd64",
+          "org.opencontainers.image.ref.name": "docker.io/tianon/test@sha256:3a0bd0fb5ad6dd6528dc78726b3df78e980b39b379e99c5a508904ec17cfafe5"
+        },
+        "platform": {
+          "architecture": "amd64",
+          "os": "windows",
+          "os.version": "10.0.17763.5576"
+        }
+      }
+    ],
+    "annotations": {
+      "org.opencontainers.image.ref.name": "docker.io/tianon/test@sha256:347290ddd775c1b85a3e381b09edde95242478eb65153e9b17225356f4c072ac"
+    }
+  }
+]

.test/lookup-test.json

@@ -28,8 +28,40 @@ time bashbrew fetch "$@"
 
 time "$dir/../sources.sh" "$@" > "$dir/sources.json"
 
+rm -rf "$dir/coverage"
+mkdir -p "$dir/coverage"
+export GOCOVERDIR="${GOCOVERDIR:-"$dir/coverage"}"
+
+rm -f "$dir/../bin/builds" # make sure we build with -cover for sure
 time "$dir/../builds.sh" --cache "$dir/cache-builds.json" "$dir/sources.json" > "$dir/builds.json"
 
+# test again, but with "--cache=..." instead of "--cache ..." (which also lets us delete the cache and get slightly better coverage reports at the expense of speed / Hub requests)
+time "$dir/../builds.sh" --cache="$dir/cache-builds.json" "$dir/sources.json" > "$dir/builds.json"
+
+# test "lookup" code for more edge cases
+"$dir/../.go-env.sh" go build -cover -trimpath -o "$dir/../bin/lookup" ./cmd/lookup
+lookup=(
+	# force a config blob lookup for platform object creation (and top-level Docker media type!)
+	'tianon/test@sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23'
+	# (this is the first Windows manifest of "tianon/test:index-no-platform-smaller" referenced below)
+
+	# tianon/test:index-no-platform-smaller - a "broken" index with *zero* platform objects in it (so every manifest requires a platform lookup)
+	'tianon/test@sha256:347290ddd775c1b85a3e381b09edde95242478eb65153e9b17225356f4c072ac'
+	# (doing these in the same run means the manifest from above should be cached and exercise more codepaths for better coverage)
+)
+"$dir/../bin/lookup" "${lookup[@]}" | jq -s > "$dir/lookup-test.json"
+
+# don't leave around the "-cover" versions of these binaries
+rm -f "$dir/../bin/builds" "$dir/../bin/lookup"
+
+# Go tests
+"$dir/../.go-env.sh" go test -cover ./... -args -test.gocoverdir="$GOCOVERDIR"
+
+# combine the coverage data into the "legacy" coverage format (understood by "go tool cover") and pre-generate HTML for easier digestion of the data
+"$dir/../.go-env.sh" go tool covdata textfmt -i "$GOCOVERDIR" -o "$dir/coverage.txt"
+"$dir/../.go-env.sh" go tool cover -html "$dir/coverage.txt" -o "$dir/coverage.html"
+"$dir/../.go-env.sh" go tool cover -func "$dir/coverage.txt"
+
 # generate an "example commands" file so that changes to generated commands are easier to review
 SOURCE_DATE_EPOCH=0 jq -r -L "$dir/.." '
 	include "meta";

.test/test.sh

@@ -0,0 +1,2 @@
+**
+!.gitignore

bin/.gitignore

@@ -7,13 +7,14 @@ export BASHBREW_STAGING_TEMPLATE
 
 dir="$(dirname "$BASH_SOURCE")"
 dir="$(readlink -ve "$dir")"
-if ( cd "$dir" && ./.any-go-nt.sh builds ); then
+bin="$dir/bin/builds"
+if ( cd "$dir" && ./.any-go-nt.sh "$bin" ); then
 	{
-		echo "building '$dir/builds' from 'builds.go'"
-		"$dir/.go-env.sh" go build -v -o builds builds.go
-		ls -l "$dir/builds"
+		echo "building '$bin'"
+		"$dir/.go-env.sh" go build ${GOCOVERDIR:+-cover} -v -trimpath -o "$bin" ./cmd/builds
+		ls -l "$bin"
 	} >&2
 fi
-[ -x "$dir/builds" ]
+[ -x "$bin" ]
 
-"$dir/builds" "$@" | jq .
+"$bin" "$@" | jq .

builds.sh

@@ -15,21 +15,21 @@ func main() {
 	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
 	defer stop()
 
-	img := os.Args[1]
-
-	ref, err := registry.ParseRefNormalized(img)
-	if err != nil {
-		panic(err)
-	}
-
-	index, err := registry.SynthesizeIndex(ctx, ref)
-	if err != nil {
-		panic(err)
-	}
-
-	e := json.NewEncoder(os.Stdout)
-	e.SetIndent("", "\t")
-	if err := e.Encode(index); err != nil {
-		panic(err)
+	for _, img := range os.Args[1:] {
+		ref, err := registry.ParseRefNormalized(img)
+		if err != nil {
+			panic(err)
+		}
+
+		index, err := registry.SynthesizeIndex(ctx, ref)
+		if err != nil {
+			panic(err)
+		}
+
+		e := json.NewEncoder(os.Stdout)
+		e.SetIndent("", "\t")
+		if err := e.Encode(index); err != nil {
+			panic(err)
+		}
 	}
 }