Skip to content

Commit 2ed7d4a

Browse files
lucacomelucacome
authored
Add NGF build information to test results (#2353)
Problem: We don't add the commit that was tested to the test results and we don't have a way to confirm that the correct version was tested. Solution: Add build info to the tests results
1 parent caf413c commit 2ed7d4a

18 files changed

+95
-55
lines changed

‎.github/workflows/functional.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -118,19 +118,19 @@ jobs:
118118
run: |
119119
ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
120120
ngf_tag=${{ steps.ngf-meta.outputs.version }}
121-
make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=telemetry GW_SERVICE_TYPE=LoadBalancer
121+
make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=telemetry GW_SERVICE_TYPE=LoadBalancer CI=true
122122
working-directory: ./tests
123123

124124
- name: Run functional graceful-recovery tests
125125
run: |
126126
ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
127127
ngf_tag=${{ steps.ngf-meta.outputs.version }}
128-
make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=graceful-recovery GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }}
128+
make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=graceful-recovery GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true
129129
working-directory: ./tests
130130

131131
- name: Run functional tests
132132
run: |
133133
ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
134134
ngf_tag=${{ steps.ngf-meta.outputs.version }}
135-
make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GW_SERVICE_TYPE=LoadBalancer
135+
make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GW_SERVICE_TYPE=LoadBalancer CI=true
136136
working-directory: ./tests

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,8 @@ endif
4545

4646
.PHONY: help
4747
help: Makefile ## Display this help
48-
@grep -hE '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "; printf "Usage:\n\n make \033[36m<target>\033[0m [VARIABLE=value...]\n\nTargets:\n\n"}; {printf " \033[36m%-30s\033[0m %s\n", $$1, $$2}'
49-
@grep -hE '^(override )?[a-zA-Z_-]+ \??\+?= .*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = " \\??\\+?= .*?## "; printf "\nVariables:\n\n"}; {gsub(/override /, "", $$1); printf " \033[36m%-30s\033[0m %s\n", $$1, $$2}'
48+
@grep -hE '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "; printf "Usage:\n\n make \033[36m<target>\033[0m [VARIABLE=value...]\n\nTargets:\n\n"}; {printf " \033[36m%-30s\033[0m %s\n", $$1, $$2}'
49+
@grep -hE '^(override )?[a-zA-Z_-]+ \??\+?= .*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = " \\??\\+?= .*?## "; printf "\nVariables:\n\n"}; {gsub(/override /, "", $$1); printf " \033[36m%-30s\033[0m %s\n", $$1, $$2}'
5050

5151
.PHONY: build-prod-images
5252
build-prod-images: build-prod-ngf-image build-prod-nginx-image ## Build the NGF and nginx docker images for production

‎tests/Makefile‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
1-
CI=false
1+
CI ?= false
22
CLUSTER_NAME ?= kind
33
CONFORMANCE_PREFIX = conformance-test-runner## Prefix for the conformance test runner image
44
CONFORMANCE_TAG = latest## Tag for the conformance test runner image
55
GATEWAY_CLASS = nginx## Gateway class to use
66
GINKGO_FLAGS =
77
GINKGO_LABEL =
8+
GITHUB_OUTPUT =
89
GW_API_VERSION ?= $(shell sed -n 's/.*ref=v\(.*\)/\1/p' ../config/crd/gateway-api/standard/kustomization.yaml)## Supported Gateway API version from current NGF
910
GW_API_PREV_VERSION ?= 1.1.0## Supported Gateway API version from previous NGF release
1011
GW_SERVICE_TYPE = NodePort## Service type to use for the gateway
@@ -15,7 +16,11 @@ PROVISIONER_MANIFEST = conformance/provisioner/provisioner.yaml
1516
SUPPORTED_FEATURES = HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect,HTTPRouteHostRewrite,HTTPRoutePathRewrite,GatewayPort8080,HTTPRouteResponseHeaderModification,GRPCExactMethodMatching,GRPCRouteListenerHostnameMatching,GRPCRouteHeaderMatching
1617

1718
ifneq ($(GINKGO_LABEL),)
18-
override GINKGO_FLAGS += --label-filter "$(GINKGO_LABEL)"
19+
override GINKGO_FLAGS += --label-filter "$(GINKGO_LABEL)"
20+
endif
21+
22+
ifeq ($(CI),true)
23+
GITHUB_OUTPUT := --github-output
1924
endif
2025

2126
.PHONY: update-go-modules
@@ -33,9 +38,9 @@ run-conformance-tests: ## Run conformance tests
3338
kubectl apply -f conformance/conformance-rbac.yaml
3439
kubectl run -i conformance \
3540
--image=$(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) --image-pull-policy=Never \
36-
--overrides='{ "spec": { "serviceAccountName": "conformance" } }' \
41+
--overrides='{ "spec": { "serviceAccountName": "conformance" } }' \
3742
--restart=Never -- sh -c "go test -v . -tags conformance,experimental -args --gateway-class=$(GATEWAY_CLASS) \
38-
--supported-features=$(SUPPORTED_FEATURES) --version=$(NGF_VERSION) \
43+
--supported-features=$(SUPPORTED_FEATURES) --version=$(NGF_VERSION) \
3944
--report-output=output.txt; cat output.txt" | tee output.txt
4045
./scripts/check-pod-exit-code.sh
4146
sed -e '1,/CONFORMANCE PROFILE/d' output.txt > conformance-profile.yaml
@@ -65,8 +70,8 @@ reset-go-modules: ## Reset the go modules changes
6570

6671
# Check if PLUS_ENABLED is true
6772
ifeq ($(PLUS_ENABLED),true)
68-
# If true, set NGINX_PREFIX to $NGINX_PLUS_PREFIX
69-
NGINX_PREFIX := $(NGINX_PLUS_PREFIX)
73+
# If true, set NGINX_PREFIX to $NGINX_PLUS_PREFIX
74+
NGINX_PREFIX := $(NGINX_PLUS_PREFIX)
7075
endif
7176

7277
.PHONY: setup-gcp-and-run-tests
@@ -107,18 +112,19 @@ start-longevity-test: nfr-test ## Start the longevity test to run for 4 days in
107112
stop-longevity-test: export STOP_LONGEVITY=true
108113
stop-longevity-test: nfr-test ## Stop the longevity test and collects results
109114

115+
110116
.PHONY: .vm-nfr-test
111117
.vm-nfr-test: ## Runs the NFR tests on the GCP VM (called by `nfr-test`)
112-
go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v --force-newlines $(ifeq $(CI),true,--github-output) \
113-
--label-filter "nfr" $(GINKGO_FLAGS) --timeout 3h ./suite -- --gateway-api-version=$(GW_API_VERSION) \
118+
go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) \
119+
--label-filter "nfr" $(GINKGO_FLAGS) --timeout 3h ./suite -- --gateway-api-version=$(GW_API_VERSION) \
114120
--gateway-api-prev-version=$(GW_API_PREV_VERSION) --image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \
115121
--plus-enabled=$(PLUS_ENABLED) --ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \
116122
--pull-policy=$(PULL_POLICY) --service-type=$(GW_SERVICE_TYPE) \
117123
--is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL)
118124

119125
.PHONY: test
120126
test: ## Runs the functional tests on your default k8s cluster
121-
go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v --force-newlines \
127+
go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) \
122128
--label-filter "functional" $(GINKGO_FLAGS) ./suite -- \
123129
--gateway-api-version=$(GW_API_VERSION) --gateway-api-prev-version=$(GW_API_PREV_VERSION) \
124130
--image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \

‎tests/framework/info.go‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,13 @@ package framework
22

33
import (
44
"fmt"
5+
"runtime/debug"
56

67
core "k8s.io/api/core/v1"
78
"sigs.k8s.io/controller-runtime/pkg/client"
89
)
910

11+
// GetLogs returns the logs for all containers in all pods for a release.
1012
func GetLogs(rm ResourceManager, namespace string, releaseName string) string {
1113
var returnLogs string
1214
pods, err := rm.GetPods(namespace, client.MatchingLabels{
@@ -32,6 +34,7 @@ func GetLogs(rm ResourceManager, namespace string, releaseName string) string {
3234
return returnLogs
3335
}
3436

37+
// GetEvents returns the events for a namespace.
3538
func GetEvents(rm ResourceManager, namespace string) string {
3639
var returnEvents string
3740
events, err := rm.GetEvents(namespace)
@@ -53,3 +56,28 @@ func GetEvents(rm ResourceManager, namespace string) string {
5356
}
5457
return returnEvents
5558
}
59+
60+
// GetBuildInfo returns the build information.
61+
func GetBuildInfo() (commitHash string, commitTime string, dirtyBuild string) {
62+
commitHash = "unknown"
63+
commitTime = "unknown"
64+
dirtyBuild = "unknown"
65+
66+
info, ok := debug.ReadBuildInfo()
67+
if !ok {
68+
return
69+
}
70+
71+
for _, kv := range info.Settings {
72+
switch kv.Key {
73+
case "vcs.revision":
74+
commitHash = kv.Value
75+
case "vcs.time":
76+
commitTime = kv.Value
77+
case "vcs.modified":
78+
dirtyBuild = kv.Value
79+
}
80+
}
81+
82+
return
83+
}

‎tests/framework/results.go‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,10 +49,13 @@ func WriteSystemInfoToFile(file *os.File, ci ClusterInfo, plus bool) error {
4949
if ci.IsGKE {
5050
clusterType = "GKE"
5151
}
52+
53+
commit, date, dirty := GetBuildInfo()
54+
55+
//nolint:lll
5256
text := fmt.Sprintf(
53-
//nolint:lll
54-
"# Results\n\n## Test environment\n\nNGINX Plus: %v\n\n%s Cluster:\n\n- Node count: %d\n- k8s version: %s\n- vCPUs per node: %d\n- RAM per node: %s\n- Max pods per node: %d\n",
55-
plus, clusterType, ci.NodeCount, ci.K8sVersion, ci.CPUCountPerNode, ci.MemoryPerNode, ci.MaxPodsPerNode,
57+
"# Results\n\n## Test environment\n\nNGINX Plus: %v\n\nNGINX Gateway Fabric:\n\n- Commit: %s\n- Date: %s\n- Dirty: %v\n\n%s Cluster:\n\n- Node count: %d\n- k8s version: %s\n- vCPUs per node: %d\n- RAM per node: %s\n- Max pods per node: %d\n",
58+
plus, commit, date, dirty, clusterType, ci.NodeCount, ci.K8sVersion, ci.CPUCountPerNode, ci.MemoryPerNode, ci.MaxPodsPerNode,
5659
)
5760
if _, err := fmt.Fprint(file, text); err != nil {
5861
return err

‎tests/scripts/remote-scripts/run-nfr-tests.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ elif [ "${STOP_LONGEVITY}" == "true" ]; then
1010
GINKGO_LABEL="longevity-teardown"
1111
fi
1212

13-
cd nginx-gateway-fabric/tests && make .vm-nfr-test TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION="${NGF_VERSION}"
13+
cd nginx-gateway-fabric/tests && make .vm-nfr-test CI=${CI} TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION="${NGF_VERSION}"
1414

1515
if [ "${START_LONGEVITY}" == "true" ]; then
1616
suite/scripts/longevity-wrk.sh

‎tests/scripts/remote-scripts/run-tests.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,4 @@ set -e
44

55
source "${HOME}"/vars.env
66

7-
cd nginx-gateway-fabric/tests && make test TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL="${GINKGO_LABEL}" GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION="${NGF_VERSION}"
7+
cd nginx-gateway-fabric/tests && make test CI=${CI} TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL="${GINKGO_LABEL}" GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION="${NGF_VERSION}"

‎tests/scripts/run-tests-gcp-vm.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ gcloud compute scp --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" "${SC
1616
gcloud compute ssh --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" username@"${RESOURCE_NAME}" \
1717
--command="export START_LONGEVITY=${START_LONGEVITY} &&\
1818
export STOP_LONGEVITY=${STOP_LONGEVITY} &&\
19+
export CI=${CI} &&\
1920
bash -s" <"${SCRIPT_DIR}"/remote-scripts/${SCRIPT}
2021
retcode=$?
2122

‎tests/suite/client_settings_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package suite
1+
package main
22

33
import (
44
"bytes"

‎tests/suite/dataplane_perf_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package suite
1+
package main
22

33
import (
44
"fmt"

0 commit comments

Comments
 (0)