From a8880d510ee8ab9922bce1cf2208ae805fa0a706 Mon Sep 17 00:00:00 2001 From: Camryn Lee Date: Mon, 28 Apr 2025 20:20:48 +0000 Subject: [PATCH 1/9] init commit: create script for mtu check --- hack/manifests/nginx.yaml | 22 +++++++ hack/scripts/cilium-mtu-validation.sh | 86 +++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 hack/manifests/nginx.yaml create mode 100755 hack/scripts/cilium-mtu-validation.sh diff --git a/hack/manifests/nginx.yaml b/hack/manifests/nginx.yaml new file mode 100644 index 0000000000..919d551cd5 --- /dev/null +++ b/hack/manifests/nginx.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx + labels: + app: nginx + namespace: kube-system +spec: + replicas: 2 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:latest + ports: + - containerPort: 80 \ No newline at end of file diff --git a/hack/scripts/cilium-mtu-validation.sh b/hack/scripts/cilium-mtu-validation.sh new file mode 100755 index 0000000000..364d9dbe0e --- /dev/null +++ b/hack/scripts/cilium-mtu-validation.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +NAMESPACE="kube-system" + +# Deploy nginx pods for MTU testing +kubectl apply -f hack/manifests/nginx.yaml +kubectl wait --for=condition=available --timeout=60s -n $NAMESPACE deployment/nginx + +echo "Checking MTU for pods in namespace: $NAMESPACE using Cilium agent and nginx MTU" + +# Get all nodes +nodes=$(kubectl get nodes -o jsonpath='{.items[*].metadata.name}') + +for node in $nodes; do + echo "Checking node: $node" + + # Get the Cilium agent pod running on this node + cilium_pod=$(kubectl get pods -n $NAMESPACE -o wide --field-selector spec.nodeName=$node -l k8s-app=cilium -o jsonpath='{.items[0].metadata.name}') + + if [ -z "$cilium_pod" ]; then + echo "Failed to find Cilium agent pod on node $node" + continue + fi + + # Get the MTU of eth0 in the Cilium agent pod + cilium_mtu=$(kubectl exec -n $NAMESPACE $cilium_pod -- cat /sys/class/net/eth0/mtu 2>/dev/null) + + if [ -z "$cilium_mtu" ]; then + echo "Failed to get MTU from Cilium agent pod on node $node" + continue + fi + + echo "Cilium agent eth0 MTU: $cilium_mtu" + + # Get the nginx pod running on this node + nginx_pod=$(kubectl get pods -n $NAMESPACE -o wide --field-selector spec.nodeName=$node -l app=nginx -o jsonpath='{.items[0].metadata.name}') + if [ -z "$nginx_pod" ]; then + echo "Failed to find nginx pod on node $node" + continue + fi + # Get the MTU of eth0 in the nginx pod + nginx_mtu=$(kubectl exec -n $NAMESPACE $nginx_pod -- cat /sys/class/net/eth0/mtu 2>/dev/null) + if [ -z "$nginx_mtu" ]; then + echo "Failed to get MTU from nginx pod on node $node" + continue + fi + echo "Nginx pod eth0 MTU: $nginx_mtu" + + # Get the node's eth0 MTU + node_mtu=$(kubectl debug node/$node -it --image=busybox -- sh -c "cat /sys/class/net/eth0/mtu" 2>/dev/null | tail -n 1) + + if [ -z "$node_mtu" ]; then + echo "Failed to get MTU from node $node" + continue + fi + echo "Node eth0 MTU: $node_mtu" + + # Check if the MTUs match + if [ "$cilium_mtu" -eq "$nginx_mtu" ] && [ "$nginx_mtu" -eq "$node_mtu" ]; then + echo "MTU validation passed for node $node" + else + echo "MTU validation failed for node $node" + echo "Cilium agent MTU: $cilium_mtu, Nginx pod MTU: $nginx_mtu, Node MTU: $node_mtu" + echo "##[error]MTU validation failed. MTUs do not match." + exit 1 + fi + + echo "----------------------------------------" + +done + +# Clean up +kubectl delete deployment nginx -n $NAMESPACE +echo "Cleaned up nginx deployment" + +# Clean up the debug pod +debug_pod=$(kubectl get pods -o name | grep "node-debugger") +if [ -n "$debug_pod" ]; then + kubectl delete $debug_pod + kubectl wait --for=delete $debug_pod --timeout=60s + if [ $? -ne 0 ]; then + echo "Failed to clean up debug pod $debug_pod" + fi +else + echo "No debug pod found" +fi \ No newline at end of file From 5e157ad9c6e0f13bc03e8b1f16290b5bf4f4b823 Mon Sep 17 00:00:00 2001 From: Camryn Lee Date: Thu, 1 May 2025 18:06:09 +0000 Subject: [PATCH 2/9] ci: cilium e2e run mtu check --- .../cilium/cilium-overlay-load-test-template.yaml | 12 ++++++++++-- .../cilium-dualstackoverlay-e2e-step-template.yaml | 7 +++++++ .../cilium-dualstackoverlay-e2e.steps.yaml | 8 ++++++++ .../cilium-overlay-e2e-step-template.yaml | 7 +++++++ .../cilium-overlay-e2e.steps.yaml | 8 ++++++++ .../cilium-overlay-e2e-step-template.yaml | 7 +++++++ .../cilium-overlay/cilium-overlay-e2e.steps.yaml | 1 + .pipelines/templates/cilium-tests.yaml | 7 +++++++ hack/manifests/nginx.yaml | 2 +- hack/scripts/cilium-mtu-validation.sh | 4 ++-- 10 files changed, 58 insertions(+), 5 deletions(-) diff --git a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml index 98a2305c46..50ff6de5a4 100644 --- a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml +++ b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml @@ -78,7 +78,8 @@ stages: addSpnToEnvironment: true inlineScript: | set -ex - az extension add --name aks-preview + az extension remove --name aks-preview || true + az extension add --name aks-preview --version 14.0.0b3 --yes make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) ls -lah pwd @@ -151,7 +152,8 @@ stages: addSpnToEnvironment: true inlineScript: | set -ex - az extension add --name aks-preview + az extension remove --name aks-preview || true + az extension add --name aks-preview --version 14.0.0b3 --yes make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) ls -lah pwd @@ -266,6 +268,12 @@ stages: fi name: "testAsyncDelete" displayName: "Verify Async Delete when CNS is down" + - script: | + cd hack/scripts + chmod +x cilium-mtu-validation.sh + ./cilium-mtu-validation.sh + name: "CiliumMTUValidation" + displayName: "Run Cilium MTU Validation" - template: ../k8s-e2e/k8s-e2e-job-template.yaml parameters: sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) diff --git a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml index 9337ef48d5..2d08a9d37b 100644 --- a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml @@ -161,3 +161,10 @@ steps: fi name: "testAsyncDelete" displayName: "Verify Async Delete when CNS is down" + + - script: | + cd hack/scripts + chmod +x cilium-mtu-validation.sh + ./cilium-mtu-validation.sh + name: "CiliumMTUValidation" + displayName: "Run Cilium MTU Validation" diff --git a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.steps.yaml b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.steps.yaml index dc49686b13..0875184ab1 100644 --- a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.steps.yaml +++ b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.steps.yaml @@ -170,3 +170,11 @@ steps: fi name: "testAsyncDelete" displayName: "Verify Async Delete when CNS is down" + + - script: | + cd hack/scripts + chmod +x cilium-mtu-validation.sh + ./cilium-mtu-validation.sh + workingDirectory: $(ACN_DIR) + name: "CiliumMTUValidation" + displayName: "Run Cilium MTU Validation" diff --git a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml index 7a81685b85..3b6fd72eb3 100644 --- a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml @@ -198,6 +198,13 @@ steps: fi name: "testAsyncDelete" displayName: "Verify Async Delete when CNS is down" + + - script: | + cd hack/scripts + chmod +x cilium-mtu-validation.sh + ./cilium-mtu-validation.sh + name: "CiliumMTUValidation" + displayName: "Run Cilium MTU Validation" - script: | ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/ diff --git a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.steps.yaml b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.steps.yaml index 6856847c1e..a789012b7f 100644 --- a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.steps.yaml +++ b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.steps.yaml @@ -196,6 +196,14 @@ steps: fi name: "testAsyncDelete" displayName: "Verify Async Delete when CNS is down" + + - script: | + cd hack/scripts + chmod +x cilium-mtu-validation.sh + ./cilium-mtu-validation.sh + workingDirectory: $(ACN_DIR) + name: "CiliumMTUValidation" + displayName: "Run Cilium MTU Validation" - script: | ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/ diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml index 7444ef1d98..e96101f75a 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml @@ -248,3 +248,10 @@ steps: fi name: "testAsyncDelete" displayName: "Verify Async Delete when CNS is down" + + - script: | + cd hack/scripts + chmod +x cilium-mtu-validation.sh + ./cilium-mtu-validation.sh + name: "CiliumMTUValidation" + displayName: "Run Cilium MTU Validation" diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.steps.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.steps.yaml index c82bafb9a8..acc97dca84 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.steps.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.steps.yaml @@ -246,3 +246,4 @@ steps: fi name: "testAsyncDelete" displayName: "Verify Async Delete when CNS is down" + diff --git a/.pipelines/templates/cilium-tests.yaml b/.pipelines/templates/cilium-tests.yaml index 6821e4ec56..c645d6ac80 100644 --- a/.pipelines/templates/cilium-tests.yaml +++ b/.pipelines/templates/cilium-tests.yaml @@ -85,3 +85,10 @@ steps: fi name: "testAsyncDelete" displayName: "Verify Async Delete when CNS is down" + + - script: | + cd hack/scripts + chmod +x cilium-mtu-validation.sh + ./cilium-mtu-validation.sh + name: "CiliumMTUValidation" + displayName: "Run Cilium MTU Validation" diff --git a/hack/manifests/nginx.yaml b/hack/manifests/nginx.yaml index 919d551cd5..5736a3a022 100644 --- a/hack/manifests/nginx.yaml +++ b/hack/manifests/nginx.yaml @@ -6,7 +6,7 @@ metadata: app: nginx namespace: kube-system spec: - replicas: 2 + replicas: 4 selector: matchLabels: app: nginx diff --git a/hack/scripts/cilium-mtu-validation.sh b/hack/scripts/cilium-mtu-validation.sh index 364d9dbe0e..89c67ba86c 100755 --- a/hack/scripts/cilium-mtu-validation.sh +++ b/hack/scripts/cilium-mtu-validation.sh @@ -3,7 +3,7 @@ NAMESPACE="kube-system" # Deploy nginx pods for MTU testing -kubectl apply -f hack/manifests/nginx.yaml +kubectl apply -f ../manifests/nginx.yaml kubectl wait --for=condition=available --timeout=60s -n $NAMESPACE deployment/nginx echo "Checking MTU for pods in namespace: $NAMESPACE using Cilium agent and nginx MTU" @@ -32,7 +32,7 @@ for node in $nodes; do echo "Cilium agent eth0 MTU: $cilium_mtu" - # Get the nginx pod running on this node + # Get an nginx pod running on this node nginx_pod=$(kubectl get pods -n $NAMESPACE -o wide --field-selector spec.nodeName=$node -l app=nginx -o jsonpath='{.items[0].metadata.name}') if [ -z "$nginx_pod" ]; then echo "Failed to find nginx pod on node $node" From bb037a76e9695d9e60c3ec19979e6462cccd2ca6 Mon Sep 17 00:00:00 2001 From: Camryn Lee Date: Thu, 1 May 2025 22:13:46 +0000 Subject: [PATCH 3/9] ci: scale pod deployment in release tests --- .../cni/cilium/cilium-overlay-load-test-template.yaml | 10 ++++++---- .../cilium-dualstackoverlay-e2e-step-template.yaml | 3 +++ .../cilium-dualstackoverlay-e2e.steps.yaml | 3 +++ .../cilium-overlay-e2e-step-template.yaml | 3 +++ .../cilium-overlay-e2e.steps.yaml | 3 +++ .../cilium-overlay-e2e-step-template.yaml | 3 +++ .../cilium-overlay/cilium-overlay-e2e.steps.yaml | 1 + .pipelines/templates/cilium-tests.yaml | 3 +++ hack/scripts/cilium-mtu-validation.sh | 3 --- 9 files changed, 25 insertions(+), 7 deletions(-) diff --git a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml index 50ff6de5a4..191ea2415e 100644 --- a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml +++ b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml @@ -78,8 +78,7 @@ stages: addSpnToEnvironment: true inlineScript: | set -ex - az extension remove --name aks-preview || true - az extension add --name aks-preview --version 14.0.0b3 --yes + az extension add --name aks-preview make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) ls -lah pwd @@ -152,8 +151,7 @@ stages: addSpnToEnvironment: true inlineScript: | set -ex - az extension remove --name aks-preview || true - az extension add --name aks-preview --version 14.0.0b3 --yes + az extension add --name aks-preview make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) ls -lah pwd @@ -269,6 +267,10 @@ stages: name: "testAsyncDelete" displayName: "Verify Async Delete when CNS is down" - script: | + echo "Deploy nginx pods for MTU testing" + kubectl apply -f hack/manifests/nginx.yaml + kubectl wait --for=condition=available --timeout=60s -n kube-system deployment/nginx + kubectl scale deployment nginx --replicas=25 -n kube-system cd hack/scripts chmod +x cilium-mtu-validation.sh ./cilium-mtu-validation.sh diff --git a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml index 2d08a9d37b..485bb71e73 100644 --- a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml @@ -163,6 +163,9 @@ steps: displayName: "Verify Async Delete when CNS is down" - script: | + echo "Deploy nginx pods for MTU testing" + kubectl apply -f hack/manifests/nginx.yaml + kubectl wait --for=condition=available --timeout=60s -n kube-system deployment/nginx cd hack/scripts chmod +x cilium-mtu-validation.sh ./cilium-mtu-validation.sh diff --git a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.steps.yaml b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.steps.yaml index 0875184ab1..90cc52190e 100644 --- a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.steps.yaml +++ b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.steps.yaml @@ -172,6 +172,9 @@ steps: displayName: "Verify Async Delete when CNS is down" - script: | + echo "Deploy nginx pods for MTU testing" + kubectl apply -f hack/manifests/nginx.yaml + kubectl wait --for=condition=available --timeout=60s -n kube-system deployment/nginx cd hack/scripts chmod +x cilium-mtu-validation.sh ./cilium-mtu-validation.sh diff --git a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml index 3b6fd72eb3..2ac2eda3d0 100644 --- a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml @@ -200,6 +200,9 @@ steps: displayName: "Verify Async Delete when CNS is down" - script: | + echo "Deploy nginx pods for MTU testing" + kubectl apply -f hack/manifests/nginx.yaml + kubectl wait --for=condition=available --timeout=60s -n kube-system deployment/nginx cd hack/scripts chmod +x cilium-mtu-validation.sh ./cilium-mtu-validation.sh diff --git a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.steps.yaml b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.steps.yaml index a789012b7f..7af8472488 100644 --- a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.steps.yaml +++ b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.steps.yaml @@ -198,6 +198,9 @@ steps: displayName: "Verify Async Delete when CNS is down" - script: | + echo "Deploy nginx pods for MTU testing" + kubectl apply -f hack/manifests/nginx.yaml + kubectl wait --for=condition=available --timeout=60s -n kube-system deployment/nginx cd hack/scripts chmod +x cilium-mtu-validation.sh ./cilium-mtu-validation.sh diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml index e96101f75a..c221a554e8 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml @@ -250,6 +250,9 @@ steps: displayName: "Verify Async Delete when CNS is down" - script: | + echo "Deploy nginx pods for MTU testing" + kubectl apply -f hack/manifests/nginx.yaml + kubectl wait --for=condition=available --timeout=60s -n kube-system deployment/nginx cd hack/scripts chmod +x cilium-mtu-validation.sh ./cilium-mtu-validation.sh diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.steps.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.steps.yaml index acc97dca84..f6533a86d1 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.steps.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.steps.yaml @@ -246,4 +246,5 @@ steps: fi name: "testAsyncDelete" displayName: "Verify Async Delete when CNS is down" + diff --git a/.pipelines/templates/cilium-tests.yaml b/.pipelines/templates/cilium-tests.yaml index c645d6ac80..4f013544bd 100644 --- a/.pipelines/templates/cilium-tests.yaml +++ b/.pipelines/templates/cilium-tests.yaml @@ -87,6 +87,9 @@ steps: displayName: "Verify Async Delete when CNS is down" - script: | + echo "Deploy nginx pods for MTU testing" + kubectl apply -f hack/manifests/nginx.yaml + kubectl wait --for=condition=available --timeout=60s -n kube-system deployment/nginx cd hack/scripts chmod +x cilium-mtu-validation.sh ./cilium-mtu-validation.sh diff --git a/hack/scripts/cilium-mtu-validation.sh b/hack/scripts/cilium-mtu-validation.sh index 89c67ba86c..a1d721c0b4 100755 --- a/hack/scripts/cilium-mtu-validation.sh +++ b/hack/scripts/cilium-mtu-validation.sh @@ -2,9 +2,6 @@ NAMESPACE="kube-system" -# Deploy nginx pods for MTU testing -kubectl apply -f ../manifests/nginx.yaml -kubectl wait --for=condition=available --timeout=60s -n $NAMESPACE deployment/nginx echo "Checking MTU for pods in namespace: $NAMESPACE using Cilium agent and nginx MTU" From 70959a2b9c86564d8664e6c1f7d7453ec6b0aace Mon Sep 17 00:00:00 2001 From: Camryn Lee Date: Fri, 2 May 2025 20:25:18 +0000 Subject: [PATCH 4/9] ci: rollout status --- .pipelines/cni/cilium/cilium-overlay-load-test-template.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml index 191ea2415e..5a394b035d 100644 --- a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml +++ b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml @@ -271,6 +271,7 @@ stages: kubectl apply -f hack/manifests/nginx.yaml kubectl wait --for=condition=available --timeout=60s -n kube-system deployment/nginx kubectl scale deployment nginx --replicas=25 -n kube-system + kubectl rollout status deployment nginx -n kube-system cd hack/scripts chmod +x cilium-mtu-validation.sh ./cilium-mtu-validation.sh From 82f6e036f9e1740976ec80c6445e7043b571bc3d Mon Sep 17 00:00:00 2001 From: Camryn Lee Date: Mon, 5 May 2025 19:03:05 +0000 Subject: [PATCH 5/9] test: template call --- .../cilium/cilium-overlay-load-test-template.yaml | 12 +----------- ...cilium-dualstackoverlay-e2e-step-template.yaml | 10 +--------- .../cilium-dualstackoverlay-e2e.steps.yaml | 11 +---------- .../cilium-overlay-e2e.steps.yaml | 11 +---------- .pipelines/templates/cilium-mtu-check.yaml | 10 ++++++++++ .pipelines/templates/cilium-tests.yaml | 10 +--------- hack/manifests/nginx.yaml | 2 +- hack/scripts/cilium-mtu-validation.sh | 15 +++++++++++++++ 8 files changed, 31 insertions(+), 50 deletions(-) create mode 100644 .pipelines/templates/cilium-mtu-check.yaml diff --git a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml index 5a394b035d..a58bd03dbf 100644 --- a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml +++ b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml @@ -266,17 +266,7 @@ stages: fi name: "testAsyncDelete" displayName: "Verify Async Delete when CNS is down" - - script: | - echo "Deploy nginx pods for MTU testing" - kubectl apply -f hack/manifests/nginx.yaml - kubectl wait --for=condition=available --timeout=60s -n kube-system deployment/nginx - kubectl scale deployment nginx --replicas=25 -n kube-system - kubectl rollout status deployment nginx -n kube-system - cd hack/scripts - chmod +x cilium-mtu-validation.sh - ./cilium-mtu-validation.sh - name: "CiliumMTUValidation" - displayName: "Run Cilium MTU Validation" + - template: ../../templates/cilium-mtu-check.yaml - template: ../k8s-e2e/k8s-e2e-job-template.yaml parameters: sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) diff --git a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml index 485bb71e73..7c1ec21f33 100644 --- a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml @@ -162,12 +162,4 @@ steps: name: "testAsyncDelete" displayName: "Verify Async Delete when CNS is down" - - script: | - echo "Deploy nginx pods for MTU testing" - kubectl apply -f hack/manifests/nginx.yaml - kubectl wait --for=condition=available --timeout=60s -n kube-system deployment/nginx - cd hack/scripts - chmod +x cilium-mtu-validation.sh - ./cilium-mtu-validation.sh - name: "CiliumMTUValidation" - displayName: "Run Cilium MTU Validation" + - template: ../../templates/cilium-mtu-check.yaml diff --git a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.steps.yaml b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.steps.yaml index 90cc52190e..7f00f3e124 100644 --- a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.steps.yaml +++ b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.steps.yaml @@ -171,13 +171,4 @@ steps: name: "testAsyncDelete" displayName: "Verify Async Delete when CNS is down" - - script: | - echo "Deploy nginx pods for MTU testing" - kubectl apply -f hack/manifests/nginx.yaml - kubectl wait --for=condition=available --timeout=60s -n kube-system deployment/nginx - cd hack/scripts - chmod +x cilium-mtu-validation.sh - ./cilium-mtu-validation.sh - workingDirectory: $(ACN_DIR) - name: "CiliumMTUValidation" - displayName: "Run Cilium MTU Validation" + - template: ../../templates/cilium-mtu-check.yaml diff --git a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.steps.yaml b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.steps.yaml index 7af8472488..bdedaa7901 100644 --- a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.steps.yaml +++ b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.steps.yaml @@ -197,16 +197,7 @@ steps: name: "testAsyncDelete" displayName: "Verify Async Delete when CNS is down" - - script: | - echo "Deploy nginx pods for MTU testing" - kubectl apply -f hack/manifests/nginx.yaml - kubectl wait --for=condition=available --timeout=60s -n kube-system deployment/nginx - cd hack/scripts - chmod +x cilium-mtu-validation.sh - ./cilium-mtu-validation.sh - workingDirectory: $(ACN_DIR) - name: "CiliumMTUValidation" - displayName: "Run Cilium MTU Validation" + - template: ../../templates/cilium-mtu-check.yaml - script: | ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/ diff --git a/.pipelines/templates/cilium-mtu-check.yaml b/.pipelines/templates/cilium-mtu-check.yaml new file mode 100644 index 0000000000..efd3ecf007 --- /dev/null +++ b/.pipelines/templates/cilium-mtu-check.yaml @@ -0,0 +1,10 @@ +steps: + - script: | + echo "Deploy nginx pods for MTU testing" + kubectl apply -f hack/manifests/nginx.yaml + kubectl wait --for=condition=available --timeout=60s -n kube-system deployment/nginx + cd hack/scripts + chmod +x cilium-mtu-validation.sh + ./cilium-mtu-validation.sh + name: "CiliumMTUValidation" + displayName: "Run Cilium MTU Validation" \ No newline at end of file diff --git a/.pipelines/templates/cilium-tests.yaml b/.pipelines/templates/cilium-tests.yaml index 4f013544bd..e3d091bd3f 100644 --- a/.pipelines/templates/cilium-tests.yaml +++ b/.pipelines/templates/cilium-tests.yaml @@ -86,12 +86,4 @@ steps: name: "testAsyncDelete" displayName: "Verify Async Delete when CNS is down" - - script: | - echo "Deploy nginx pods for MTU testing" - kubectl apply -f hack/manifests/nginx.yaml - kubectl wait --for=condition=available --timeout=60s -n kube-system deployment/nginx - cd hack/scripts - chmod +x cilium-mtu-validation.sh - ./cilium-mtu-validation.sh - name: "CiliumMTUValidation" - displayName: "Run Cilium MTU Validation" + - template: ./cilium-mtu-check.yaml diff --git a/hack/manifests/nginx.yaml b/hack/manifests/nginx.yaml index 5736a3a022..0ef6b902bd 100644 --- a/hack/manifests/nginx.yaml +++ b/hack/manifests/nginx.yaml @@ -17,6 +17,6 @@ spec: spec: containers: - name: nginx - image: nginx:latest + image: mcr.microsoft.com/azurelinux/base/nginx:1 ports: - containerPort: 80 \ No newline at end of file diff --git a/hack/scripts/cilium-mtu-validation.sh b/hack/scripts/cilium-mtu-validation.sh index a1d721c0b4..0d82ef5045 100755 --- a/hack/scripts/cilium-mtu-validation.sh +++ b/hack/scripts/cilium-mtu-validation.sh @@ -1,5 +1,20 @@ #!/bin/bash +echo "Deploy nginx pods for MTU testing" +kubectl apply -f ../manifests/nginx.yaml +kubectl wait --for=condition=available --timeout=60s -n kube-system deployment/nginx + +# Check node count +node_count=$(kubectl get nodes --no-headers | wc -l) + +# in CNI release test scenario scale deployments to 3 * node count to get replicas on each node +if [ "$node_count" -gt 1 ]; then + echo "Scaling nginx deployment to $((3 * node_count)) replicas" + kubectl scale deployment nginx --replicas=$((2 * node_count)) -n kube-system +fi +# Wait for nginx pods to be ready +kubectl wait --for=condition=available --timeout=60s -n kube-system deployment/nginx + NAMESPACE="kube-system" From d25b1c6da9731521724662a3f10738e9386f3dcf Mon Sep 17 00:00:00 2001 From: Camryn Lee Date: Thu, 8 May 2025 16:25:56 +0000 Subject: [PATCH 6/9] addressing comments, fix script scale deployment --- .pipelines/templates/cilium-mtu-check.yaml | 3 --- hack/scripts/cilium-mtu-validation.sh | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.pipelines/templates/cilium-mtu-check.yaml b/.pipelines/templates/cilium-mtu-check.yaml index efd3ecf007..6cf1967493 100644 --- a/.pipelines/templates/cilium-mtu-check.yaml +++ b/.pipelines/templates/cilium-mtu-check.yaml @@ -1,8 +1,5 @@ steps: - script: | - echo "Deploy nginx pods for MTU testing" - kubectl apply -f hack/manifests/nginx.yaml - kubectl wait --for=condition=available --timeout=60s -n kube-system deployment/nginx cd hack/scripts chmod +x cilium-mtu-validation.sh ./cilium-mtu-validation.sh diff --git a/hack/scripts/cilium-mtu-validation.sh b/hack/scripts/cilium-mtu-validation.sh index 0d82ef5045..5ef87a8d72 100755 --- a/hack/scripts/cilium-mtu-validation.sh +++ b/hack/scripts/cilium-mtu-validation.sh @@ -10,7 +10,7 @@ node_count=$(kubectl get nodes --no-headers | wc -l) # in CNI release test scenario scale deployments to 3 * node count to get replicas on each node if [ "$node_count" -gt 1 ]; then echo "Scaling nginx deployment to $((3 * node_count)) replicas" - kubectl scale deployment nginx --replicas=$((2 * node_count)) -n kube-system + kubectl scale deployment nginx --replicas=$((3 * node_count)) -n kube-system fi # Wait for nginx pods to be ready kubectl wait --for=condition=available --timeout=60s -n kube-system deployment/nginx From 931260433cd4e314f82cc80ba50fed0f92d24879 Mon Sep 17 00:00:00 2001 From: Camryn Lee Date: Thu, 8 May 2025 18:49:41 +0000 Subject: [PATCH 7/9] address nits, test maxSkew, add exit on errors --- .../cilium-overlay-e2e-step-template.yaml | 12 ++-------- hack/manifests/nginx.yaml | 9 +++++++- hack/scripts/cilium-mtu-validation.sh | 23 +++++++++++-------- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml index 2ac2eda3d0..a9f024ae15 100644 --- a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml @@ -198,16 +198,8 @@ steps: fi name: "testAsyncDelete" displayName: "Verify Async Delete when CNS is down" - - - script: | - echo "Deploy nginx pods for MTU testing" - kubectl apply -f hack/manifests/nginx.yaml - kubectl wait --for=condition=available --timeout=60s -n kube-system deployment/nginx - cd hack/scripts - chmod +x cilium-mtu-validation.sh - ./cilium-mtu-validation.sh - name: "CiliumMTUValidation" - displayName: "Run Cilium MTU Validation" + + - template: ../../templates/cilium-mtu-check.yaml - script: | ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/ diff --git a/hack/manifests/nginx.yaml b/hack/manifests/nginx.yaml index 0ef6b902bd..2ecfc95a72 100644 --- a/hack/manifests/nginx.yaml +++ b/hack/manifests/nginx.yaml @@ -19,4 +19,11 @@ spec: - name: nginx image: mcr.microsoft.com/azurelinux/base/nginx:1 ports: - - containerPort: 80 \ No newline at end of file + - containerPort: 80 + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname # KV: Key is hostname, value is each unique nodename + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app: nginx \ No newline at end of file diff --git a/hack/scripts/cilium-mtu-validation.sh b/hack/scripts/cilium-mtu-validation.sh index 5ef87a8d72..9e8386b849 100755 --- a/hack/scripts/cilium-mtu-validation.sh +++ b/hack/scripts/cilium-mtu-validation.sh @@ -1,8 +1,9 @@ #!/bin/bash +NAMESPACE="kube-system" echo "Deploy nginx pods for MTU testing" kubectl apply -f ../manifests/nginx.yaml -kubectl wait --for=condition=available --timeout=60s -n kube-system deployment/nginx +kubectl wait --for=condition=available --timeout=60s -n $NAMESPACE deployment/nginx # Check node count node_count=$(kubectl get nodes --no-headers | wc -l) @@ -10,12 +11,11 @@ node_count=$(kubectl get nodes --no-headers | wc -l) # in CNI release test scenario scale deployments to 3 * node count to get replicas on each node if [ "$node_count" -gt 1 ]; then echo "Scaling nginx deployment to $((3 * node_count)) replicas" - kubectl scale deployment nginx --replicas=$((3 * node_count)) -n kube-system + kubectl scale deployment nginx --replicas=$((3 * node_count)) -n $NAMESPACE fi # Wait for nginx pods to be ready -kubectl wait --for=condition=available --timeout=60s -n kube-system deployment/nginx +kubectl wait --for=condition=available --timeout=60s -n $NAMESPACE deployment/nginx -NAMESPACE="kube-system" echo "Checking MTU for pods in namespace: $NAMESPACE using Cilium agent and nginx MTU" @@ -31,7 +31,8 @@ for node in $nodes; do if [ -z "$cilium_pod" ]; then echo "Failed to find Cilium agent pod on node $node" - continue + echo "##[error]Failed to find Cilium agent pod on node $node" + exit 1 fi # Get the MTU of eth0 in the Cilium agent pod @@ -39,7 +40,8 @@ for node in $nodes; do if [ -z "$cilium_mtu" ]; then echo "Failed to get MTU from Cilium agent pod on node $node" - continue + echo "##[error]Failed to get MTU from Cilium agent pod on node $node" + exit 1 fi echo "Cilium agent eth0 MTU: $cilium_mtu" @@ -48,13 +50,15 @@ for node in $nodes; do nginx_pod=$(kubectl get pods -n $NAMESPACE -o wide --field-selector spec.nodeName=$node -l app=nginx -o jsonpath='{.items[0].metadata.name}') if [ -z "$nginx_pod" ]; then echo "Failed to find nginx pod on node $node" - continue + echo "##[error]Failed to find nginx pod on node $node" + exit 1 fi # Get the MTU of eth0 in the nginx pod nginx_mtu=$(kubectl exec -n $NAMESPACE $nginx_pod -- cat /sys/class/net/eth0/mtu 2>/dev/null) if [ -z "$nginx_mtu" ]; then echo "Failed to get MTU from nginx pod on node $node" - continue + echo "##[error]Failed to get MTU from nginx pod on node $node" + exit 1 fi echo "Nginx pod eth0 MTU: $nginx_mtu" @@ -63,7 +67,8 @@ for node in $nodes; do if [ -z "$node_mtu" ]; then echo "Failed to get MTU from node $node" - continue + echo "##[error]Failed to get MTU from node $node" + exit 1 fi echo "Node eth0 MTU: $node_mtu" From 225d09ae28956317a1355922e9086ea86a8f5432 Mon Sep 17 00:00:00 2001 From: Camryn Lee Date: Mon, 12 May 2025 21:15:13 +0000 Subject: [PATCH 8/9] adding missing template calls --- .../cilium-overlay-e2e-step-template.yaml | 12 ++---------- .../cilium-overlay/cilium-overlay-e2e.steps.yaml | 1 + 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml index c221a554e8..04730a16f0 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml @@ -248,13 +248,5 @@ steps: fi name: "testAsyncDelete" displayName: "Verify Async Delete when CNS is down" - - - script: | - echo "Deploy nginx pods for MTU testing" - kubectl apply -f hack/manifests/nginx.yaml - kubectl wait --for=condition=available --timeout=60s -n kube-system deployment/nginx - cd hack/scripts - chmod +x cilium-mtu-validation.sh - ./cilium-mtu-validation.sh - name: "CiliumMTUValidation" - displayName: "Run Cilium MTU Validation" + + - template: ../../templates/cilium-mtu-check.yaml \ No newline at end of file diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.steps.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.steps.yaml index f6533a86d1..4674d4e3cb 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.steps.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.steps.yaml @@ -247,4 +247,5 @@ steps: name: "testAsyncDelete" displayName: "Verify Async Delete when CNS is down" + - template: ../../templates/cilium-mtu-check.yaml From 64fce27f74055e63eaa2decb318934710fa63564 Mon Sep 17 00:00:00 2001 From: Camryn Lee Date: Thu, 15 May 2025 23:02:59 +0000 Subject: [PATCH 9/9] fix cilium-overlay e2e --- .../singletenancy/cilium-overlay/cilium-overlay-e2e.steps.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.steps.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.steps.yaml index 4674d4e3cb..39c2cae2a3 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.steps.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.steps.yaml @@ -248,4 +248,5 @@ steps: displayName: "Verify Async Delete when CNS is down" - template: ../../templates/cilium-mtu-check.yaml +