Add test reports to PRs

This patch updates GitHub actions to provide more information about
tests executed as part of pull requests.

Author: akutz

.github/workflows/ci.yml

@@ -6,7 +6,7 @@ env:
   # sometimes we need to explicitly override the Go version, ex. CVEs,
   # when it is not possible to update the go.mod version yet, ex. the
   # internal builds do not yet support that version of Go.
-  GO_VERSION:
+  GO_VERSION: ""
 
 on:
   pull_request:
@@ -239,17 +239,21 @@ jobs:
         indicators: true
         output: both
         thresholds: '79 89' # really '80 90', but the values are not inclusive
-    - name: Save pull request ID
+    - name: Save commit and pull request ID
       if: github.event_name == 'pull_request'
       env:
         PR_NUMBER: ${{ github.event.number }}
       run: |
+        echo "${GITHUB_SHA}" >commit_id
         echo "${PR_NUMBER}" >pull_request_id
-    - name: Store code coverage results
+    - name: Store pull request reports
       if: github.event_name == 'pull_request'
       uses: actions/upload-artifact@v4
       with:
-        name: code-coverage-results
+        name: pull-request-reports
         path: |
           code-coverage-results.md
+          commit_id
           pull_request_id
+          test-results.json
+          test-results.xml

.github/workflows/post-coverage-to-pr.yml renamed to .github/workflows/pr-reports.yml

@@ -1,4 +1,4 @@
-name: post-coverage-to-pr
+name: pr-reports
 
 on:
   workflow_run:
@@ -12,9 +12,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Fetch code coverage results
+    - name: Fetch pull request reports
       uses: actions/github-script@v7
-      id: fetch-code-coverage-results
+      id: fetch-pr-reports
       with:
         script: |
           let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
@@ -24,7 +24,7 @@ jobs:
           });
           let is_pull_request = 'false';
           let matchedArtifacts = allArtifacts.data.artifacts.filter((artifact) => {
-            return artifact.name == "code-coverage-results"
+            return artifact.name == "pull-request-reports"
           });
           if (matchedArtifacts.length > 0) {
             let download = await github.rest.actions.downloadArtifact({
@@ -35,22 +35,40 @@ jobs:
             });
             let fs = require('fs');
             fs.writeFileSync(
-              `${process.env.GITHUB_WORKSPACE}/code-coverage-results.zip`,
+              `${process.env.GITHUB_WORKSPACE}/pull-request-reports.zip`,
               Buffer.from(download.data)
             );
             is_pull_request = 'true';
           }
           core.info('Setting output: is_pull_request=' + is_pull_request);
           core.setOutput('is_pull_request', is_pull_request);
 
-    - name: Unzip code coverage results
-      if: steps.fetch-code-coverage-results.outputs.is_pull_request == 'true'
-      run: unzip code-coverage-results.zip
+    - name: Unzip pull request reports
+      if: steps.fetch-pr-reports.outputs.is_pull_request == 'true'
+      run: unzip pull-request-reports.zip
+
+    - name: Load commit ID
+      uses: actions/github-script@v7
+      id: load-commit-id
+      if: steps.fetch-pr-reports.outputs.is_pull_request == 'true'
+      with:
+        script: |
+          let fs = require('fs');
+          let pull_request_id = fs.readFileSync(
+            `${process.env.GITHUB_WORKSPACE}/commit_id`
+          );
+          core.info('commit_id=' + commit_id);
+          if (commit_id == '') {
+            core.error('commit_id is empty');
+          } else {
+            core.info('Setting output: commit_id=' + commit_id);
+            core.setOutput('commit_id', Number(commit_id));
+          }
 
     - name: Load pull request ID
       uses: actions/github-script@v7
       id: load-pull-request-id
-      if: steps.fetch-code-coverage-results.outputs.is_pull_request == 'true'
+      if: steps.fetch-pr-reports.outputs.is_pull_request == 'true'
       with:
         script: |
           let fs = require('fs');
@@ -65,10 +83,21 @@ jobs:
             core.setOutput('pull_request_id', Number(pull_request_id));
           }
 
-    - name: Update PR with code coverage results
+    - name: Update pull request with test report
+      uses: mikepenz/action-junit-report@v5
+      if: steps.fetch-pr-reports.outputs.is_pull_request == 'true'
+      with:
+        report_paths: test-report.xml
+        commit: ${{ steps.load-commit-id.outputs.commit_id }}
+        pr_id: ${{ steps.load-pull-request-id.outputs.pull_request_id }}
+        comment: true
+        updateComment: true
+        include_time_in_summary: true
+
+    - name: Update pull request with code coverage
       uses: marocchino/sticky-pull-request-comment@v2
-      if: steps.fetch-code-coverage-results.outputs.is_pull_request == 'true'
+      if: steps.fetch-pr-reports.outputs.is_pull_request == 'true'
       with:
         number: ${{ steps.load-pull-request-id.outputs.pull_request_id }}
         recreate: true
-        path: code-coverage-results.md
+        path: pull-request-reports.md

.gitignore

@@ -1,3 +1,8 @@
+# Ignore the test reports.
+junit*.xml
+test-results.json
+test-results.xml
+
 # Only vendor the net-op CRD for networkinterfaces.
 config/crd/external-crds/netoperator.vmware.com_*
 !config/crd/external-crds/netoperator.vmware.com_networkinterfaces.yaml

Makefile

@@ -105,9 +105,12 @@ IMAGE ?= vmoperator-controller
 IMAGE_TAG ?= latest
 IMG ?= ${IMAGE}:${IMAGE_TAG}
 
-# Code coverage files
+# Code coverage files.
 COVERAGE_FILE := cover.out
 
+# Test result file name (sans extension).
+TEST_RESULTS_FILE := test-results
+
 # Gather a set of root packages that have at least one file that matches
 # the pattern *_test.go as a child or descendent in that directory.
 # However, given this is not a cheap operation, only gather these packages if
@@ -217,7 +220,7 @@ help: ## Display this help
 .PHONY: test-api
 test-api: | $(GINKGO)
 test-api: ## Run API tests
-	COVERAGE_FILE="" hack/test.sh ./api
+	COVERAGE_FILE="" TEST_RESULTS_FILE="" hack/test.sh ./api
 
 .PHONY: test-nocover
 test-nocover: | $(GINKGO)
@@ -228,8 +231,8 @@ test-nocover: ## Run tests sans coverage
 .PHONY: test
 test: | $(GINKGO) $(ETCD) $(KUBE_APISERVER)
 test: ## Run tests
-	@rm -f "$(COVERAGE_FILE)"
-	COVERAGE_FILE="$(COVERAGE_FILE)" $(MAKE) test-nocover
+	@rm -f "$(COVERAGE_FILE)" "$(TEST_RESULTS_FILE).*"
+	COVERAGE_FILE="$(COVERAGE_FILE)" TEST_RESULTS_FILE="$(TEST_RESULTS_FILE)" $(MAKE) test-nocover
 
 .PHONY: coverage-xml
 coverage-xml: $(GOCOV) $(GOCOV_XML)

hack/test.sh

@@ -32,6 +32,18 @@ if [ -n "${COVERAGE_FILE:-}" ]; then
   GO_TEST_FLAGS+=("--coverprofile=${COVERAGE_FILE:-}")
 fi
 
+# Create test report(s).
+if [ -n "${TEST_RESULTS_FILE:-}" ]; then
+  GO_TEST_FLAGS+=("--output-dir='$(pwd)'")
+  GO_TEST_FLAGS+=("--junit-report=${TEST_RESULTS_FILE}.xml")
+  GO_TEST_FLAGS+=("--json-report=${TEST_RESULTS_FILE}.json")
+fi
+
+# If run on GitHub then use the Ginkgo GitHub flag.
+if [ -n "${GITHUB_RUN_ID:-}" ]; then
+  GO_TEST_FLAGS+=("--github-output")
+fi
+
 # Run the tests.
 # shellcheck disable=SC2086
 ginkgo "${GO_TEST_FLAGS[@]+"${GO_TEST_FLAGS[@]}"}" "${@:-}" || TEST_CMD_EXIT_CODE="${?}"