Skip to content

Commit b944373

Browse files
miledxzmiledxz
committed
Run NFR tests on 1st and 15th of every month (nginx#2249)
Problem: We want to catch problems early Solution: Run NFR tests periodically
1 parent b18326f commit b944373

File tree

2 files changed

+142
-99
lines changed

2 files changed

+142
-99
lines changed

‎.github/workflows/nfr.yml‎

Lines changed: 141 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,14 @@ on:
1717
description: Tag of the NGF and NGINX Docker images
1818
required: true
1919
default: edge
20-
nginx_plus:
21-
description: Run tests with NGINX Plus
22-
required: false
23-
default: false
24-
type: boolean
20+
type:
21+
description: Type of NGINX image to test
22+
required: true
23+
default: both
24+
type: choice
25+
options: [oss, plus, both]
26+
schedule:
27+
- cron: "0 16 1,15 * *" # Run on the 1st and 15th of every month at 16:00 UTC
2528

2629
defaults:
2730
run:
@@ -35,104 +38,143 @@ permissions:
3538
contents: read
3639

3740
jobs:
41+
vars:
42+
name: Set up vars
43+
runs-on: ubuntu-22.04
44+
outputs:
45+
test_label: ${{ github.event.inputs.test_label || 'all' }}
46+
version: ${{ github.event.inputs.version || 'edge' }}
47+
image_tag: ${{ github.event.inputs.image_tag || 'edge' }}
48+
types: ${{ steps.var.outputs.types }}
49+
permissions:
50+
contents: read
51+
steps:
52+
- name: Set vars
53+
id: var
54+
run: |
55+
if ${{ github.event.inputs.type == 'both' || github.event_name == 'schedule' }}; then
56+
echo 'types=["oss","plus"]' >> $GITHUB_OUTPUT
57+
else
58+
echo 'types=["${{ github.event.inputs.type }}"]' >> $GITHUB_OUTPUT
59+
fi
60+
3861
setup-and-run-tests:
3962
name: Setup and Run NFR Tests
4063
runs-on: ubuntu-22.04
4164
permissions:
42-
contents: write # needed for opening PR with the results files
43-
pull-requests: write # needed for opening PR with the results files
65+
contents: read
4466
id-token: write # needed for authenticating to GCP
67+
needs: vars
68+
strategy:
69+
matrix:
70+
type: ${{ fromJson(needs.vars.outputs.types) }}
71+
steps:
72+
- name: Checkout Repository
73+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
74+
75+
- name: Authenticate to Google Cloud
76+
id: auth
77+
uses: google-github-actions/auth@71fee32a0bb7e97b4d33d548e7d957010649d8fa # v2.1.3
78+
with:
79+
token_format: access_token
80+
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY }}
81+
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
82+
83+
- name: Set up Cloud SDK
84+
uses: google-github-actions/setup-gcloud@98ddc00a17442e89a24bbf282954a3b65ce6d200 # v2.1.0
85+
with:
86+
project_id: ${{ secrets.GCP_PROJECT_ID }}
87+
install_components: kubectl
88+
89+
- name: Setup dotenv file
90+
working-directory: ./tests/scripts
91+
run: |
92+
echo "RESOURCE_NAME=nfr-tests-${{ github.run_id }}" >> vars.env
93+
echo "TAG=${{ needs.vars.outputs.image_tag }}" >> vars.env
94+
echo "PREFIX=ghcr.io/nginxinc/nginx-gateway-fabric" >> vars.env
95+
echo "NGINX_PREFIX=ghcr.io/nginxinc/nginx-gateway-fabric/nginx" >> vars.env
96+
echo "NGINX_PLUS_PREFIX=us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/nginx-gateway-fabric/nginx-plus" >> vars.env
97+
echo "GKE_CLUSTER_NAME=nfr-tests-${{ github.run_id }}" >> vars.env
98+
echo "GKE_CLUSTER_ZONE=us-east1-b" >> vars.env
99+
echo "GKE_CLUSTER_REGION=us-east1" >> vars.env
100+
echo "GKE_PROJECT=${{ secrets.GCP_PROJECT_ID }}" >> vars.env
101+
echo "GKE_SVC_ACCOUNT=${{ secrets.GCP_SERVICE_ACCOUNT }}" >> vars.env
102+
echo "GKE_NODES_SERVICE_ACCOUNT=${{ secrets.GKE_NODES_SERVICE_ACCOUNT }}" >> vars.env
103+
echo "IMAGE=projects/debian-cloud/global/images/debian-11-bullseye-v20240213" >> vars.env
104+
echo "NETWORK_TAGS=nfr-tests-${{ github.run_id }}" >> vars.env
105+
echo "NGF_REPO=nginxinc" >> vars.env
106+
echo "NGF_BRANCH=${{ github.ref_name }}" >> vars.env
107+
echo "SOURCE_IP_RANGE=$(curl -sS -4 icanhazip.com)/32" >> vars.env
108+
echo "ADD_VM_IP_AUTH_NETWORKS=true" >> vars.env
109+
echo "PLUS_ENABLED=${{ matrix.type == 'plus' }}" >> vars.env
110+
echo "GINKGO_LABEL=" >> vars.env
111+
echo "NGF_VERSION=${{ needs.vars.outputs.version }}" >> vars.env
112+
echo "GKE_NUM_NODES=12" >> vars.env
113+
echo "GKE_MACHINE_TYPE=n2d-standard-16" >> vars.env
45114
115+
- name: Create GKE cluster
116+
working-directory: ./tests
117+
run: make create-gke-cluster CI=true
118+
119+
- name: Create and setup VM
120+
working-directory: ./tests
121+
run: make create-and-setup-vm
122+
123+
- name: Run Tests
124+
working-directory: ./tests
125+
run: |
126+
if ${{ needs.vars.outputs.test_label != 'all' }}; then
127+
sed -i '/^GINKGO_LABEL=/s/=.*/="${{ needs.vars.outputs.test_label }}"/' "scripts/vars.env" && make nfr-test;
128+
else
129+
make nfr-test;
130+
fi
131+
132+
- name: Upload Artifacts
133+
uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
134+
with:
135+
name: results-${{ matrix.type }}
136+
path: tests/results/
137+
138+
- name: Cleanup
139+
working-directory: ./tests
140+
if: always()
141+
run: |
142+
bash scripts/cleanup-vm.sh true
143+
make delete-gke-cluster
144+
rm -rf scripts/vars.env
145+
146+
pr-results:
147+
name: Open PR with results
148+
runs-on: ubuntu-22.04
149+
permissions:
150+
contents: write # needed for opening PR with the results files
151+
pull-requests: write # needed for opening PR with the results files
152+
needs: [vars, setup-and-run-tests]
46153
steps:
47-
- name: Checkout Repository
48-
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
49-
50-
- name: Authenticate to Google Cloud
51-
id: auth
52-
uses: google-github-actions/auth@71fee32a0bb7e97b4d33d548e7d957010649d8fa # v2.1.3
53-
with:
54-
token_format: access_token
55-
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY }}
56-
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
57-
58-
- name: Set up Cloud SDK
59-
uses: google-github-actions/setup-gcloud@98ddc00a17442e89a24bbf282954a3b65ce6d200 # v2.1.0
60-
with:
61-
project_id: ${{ secrets.GCP_PROJECT_ID }}
62-
install_components: kubectl
63-
64-
- name: Setup dotenv file
65-
working-directory: ./tests/scripts
66-
run: |
67-
echo "RESOURCE_NAME=nfr-tests-${{ github.run_id }}" >> vars.env
68-
echo "TAG=${{ inputs.image_tag }}" >> vars.env
69-
echo "PREFIX=ghcr.io/nginxinc/nginx-gateway-fabric" >> vars.env
70-
echo "NGINX_PREFIX=ghcr.io/nginxinc/nginx-gateway-fabric/nginx" >> vars.env
71-
echo "NGINX_PLUS_PREFIX=us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/nginx-gateway-fabric/nginx-plus" >> vars.env
72-
echo "GKE_CLUSTER_NAME=nfr-tests-${{ github.run_id }}" >> vars.env
73-
echo "GKE_CLUSTER_ZONE=us-east1-b" >> vars.env
74-
echo "GKE_CLUSTER_REGION=us-east1" >> vars.env
75-
echo "GKE_PROJECT=${{ secrets.GCP_PROJECT_ID }}" >> vars.env
76-
echo "GKE_SVC_ACCOUNT=${{ secrets.GCP_SERVICE_ACCOUNT }}" >> vars.env
77-
echo "GKE_NODES_SERVICE_ACCOUNT=${{ secrets.GKE_NODES_SERVICE_ACCOUNT }}" >> vars.env
78-
echo "IMAGE=projects/debian-cloud/global/images/debian-11-bullseye-v20240213" >> vars.env
79-
echo "NETWORK_TAGS=nfr-tests-${{ github.run_id }}" >> vars.env
80-
echo "NGF_REPO=nginxinc" >> vars.env
81-
echo "NGF_BRANCH=${{ github.ref_name }}" >> vars.env
82-
echo "SOURCE_IP_RANGE=$(curl -sS -4 icanhazip.com)/32" >> vars.env
83-
echo "ADD_VM_IP_AUTH_NETWORKS=true" >> vars.env
84-
echo "PLUS_ENABLED=${{ inputs.nginx_plus }}" >> vars.env
85-
echo "GINKGO_LABEL=" >> vars.env
86-
echo "NGF_VERSION=${{ inputs.version }}" >> vars.env
87-
echo "GKE_NUM_NODES=12" >> vars.env
88-
echo "GKE_MACHINE_TYPE=n2d-standard-16" >> vars.env
89-
90-
- name: Create GKE cluster
91-
working-directory: ./tests
92-
run:
93-
make create-gke-cluster CI=true
94-
95-
- name: Create and setup VM
96-
working-directory: ./tests
97-
run:
98-
make create-and-setup-vm
99-
100-
- name: Run Tests
101-
working-directory: ./tests
102-
run: |
103-
if ${{ inputs.test_label != 'all' }}; then
104-
sed -i '/^GINKGO_LABEL=/s/=.*/="${{ inputs.test_label }}"/' "scripts/vars.env" && make nfr-test;
105-
else
106-
make nfr-test;
107-
fi
108-
109-
- name: Cleanup
110-
working-directory: ./tests
111-
if: always()
112-
run: |
113-
bash scripts/cleanup-vm.sh true
114-
make delete-gke-cluster
115-
rm -rf scripts/vars.env
116-
117-
- name: Open a PR with the results
118-
uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6.1.0
119-
with:
120-
token: ${{ secrets.NGINX_PAT }}
121-
commit-message: NFR Test Results for NGF version ${{ inputs.version }} ${{ inputs.nginx_plus == true && '(Plus)' || ''}}
122-
author: nginx-bot <[email protected]>
123-
committer: nginx-bot <[email protected]>
124-
branch: tests/nfr-tests-${{ inputs.version }}${{ inputs.nginx_plus == true && '-plus' || ''}}
125-
delete-branch: true
126-
title: NFR Test Results for NGF version ${{ inputs.version }} ${{ inputs.nginx_plus == true && '(Plus)' || ''}}
127-
add-paths: |
128-
tests/results/
129-
body: |
130-
Update with NFR test results for NGF version ${{ inputs.version }} ${{ inputs.nginx_plus == true && '(Plus)' || ''}}
131-
- Auto-generated by the NFR tests workflow run ${{ github.run_id }}
132-
- Tests ran using Docker image tag ${{ inputs.image_tag }}
133-
- ${{ inputs.test_label }} test(s) ran
134-
- NGINX Plus enabled: ${{ inputs.nginx_plus }}
135-
labels: |
136-
tests
137-
assignees: ${{ github.actor }}
138-
draft: true
154+
- name: Checkout Repository
155+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
156+
157+
- name: Download Artifacts
158+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
159+
with:
160+
path: tests/results/
161+
162+
- name: Open a PR with the results
163+
uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6.1.0
164+
with:
165+
token: ${{ secrets.NGINX_PAT }}
166+
commit-message: NFR Test Results for NGF version ${{ needs.vars.outputs.version }}
167+
author: nginx-bot <[email protected]>
168+
committer: nginx-bot <[email protected]>
169+
branch: tests/nfr-tests-${{ needs.vars.outputs.version }}
170+
delete-branch: true
171+
title: NFR Test Results for NGF version ${{ needs.vars.outputs.version }}
172+
add-paths: |
173+
tests/results/
174+
body: |
175+
Update with NFR test results for NGF version ${{ needs.vars.outputs.version }} ${{ needs.vars.outputs.types }}
176+
- Auto-generated by the NFR tests workflow run ${{ github.run_id }}
177+
- Tests ran using Docker image tag ${{ needs.vars.outputs.image_tag }}
178+
- ${{ needs.vars.outputs.test_label }} test(s) ran
179+
assignees: ${{ github.actor }}
180+
draft: ${{ github.event_name != 'schedule' }}

‎.github/workflows/release-pr.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ jobs:
9595
commit-message: Release ${{ inputs.version }}
9696
title: Release ${{ inputs.version }}
9797
draft: true
98+
delete-branch: true
9899
branch: docs/release-${{ inputs.version }}
99100
author: nginx-bot <[email protected]>
100101
committer: nginx-bot <[email protected]>

0 commit comments

Comments
 (0)