Skip to content

Commit 89697b8

Browse files
camilamacedo86camilamacedo86
committed
(cleanup): (cleanup): Refactor metrics endpoint tests by extracting shared helpers
Refactored the metrics endpoint tests by introducing reusable helper functions. This reduces duplication and improves maintainability across both tests.
1 parent 591c73c commit 89697b8

File tree

2 files changed

+127
-162
lines changed

2 files changed

+127
-162
lines changed

‎test/e2e/metrics_test.go

Lines changed: 104 additions & 162 deletions
Original file line numberDiff line numberDiff line change
@@ -1,76 +1,110 @@
1+
// nolint:gosec
2+
// Package e2e contains end-to-end tests to verify that the metrics endpoints
3+
// for both components. Metrics are exported and accessible by authorized users through
4+
// RBAC and ServiceAccount tokens.
5+
//
6+
// These tests perform the following steps:
7+
// 1. Create a ClusterRoleBinding to grant necessary permissions for accessing metrics.
8+
// 2. Generate a ServiceAccount token for authentication.
9+
// 3. Deploy a curl pod to interact with the metrics endpoint.
10+
// 4. Wait for the curl pod to become ready.
11+
// 5. Execute a curl command from the pod to validate the metrics endpoint.
12+
// 6. Clean up all resources created during the test, such as the ClusterRoleBinding and curl pod.
113
package e2e
214

315
import (
416
"bytes"
517
"io"
618
"os/exec"
719
"testing"
20+
"time"
821

922
"github.com/stretchr/testify/require"
23+
24+
"github.com/operator-framework/operator-controller/test/utils"
1025
)
1126

12-
// nolint:gosec
1327
// TestOperatorControllerMetricsExportedEndpoint verifies that the metrics endpoint for the operator controller
14-
// is exported correctly and accessible by authorized users through RBAC and a ServiceAccount token.
15-
// The test performs the following steps:
16-
// 1. Creates a ClusterRoleBinding to grant necessary permissions for accessing metrics.
17-
// 2. Generates a ServiceAccount token for authentication.
18-
// 3. Deploys a curl pod to interact with the metrics endpoint.
19-
// 4. Waits for the curl pod to become ready.
20-
// 5. Executes a curl command from the pod to validate the metrics endpoint.
21-
// 6. Cleans up all resources created during the test, such as the ClusterRoleBinding and curl pod.
2228
func TestOperatorControllerMetricsExportedEndpoint(t *testing.T) {
23-
var (
24-
token string
25-
curlPod = "curl-metrics"
26-
client = ""
27-
clients = []string{"kubectl", "oc"}
29+
client := utils.FindK8sClient(t)
30+
namespace := getNamespace(t, client, "control-plane=operator-controller-controller-manager")
31+
32+
createClusterRoleBinding(
33+
t, client, "operator-controller-metrics-binding",
34+
"operator-controller-metrics-reader", namespace,
35+
"operator-controller-controller-manager",
2836
)
2937

30-
t.Log("Looking for k8s client")
31-
for _, c := range clients {
32-
// Would prefer to use `command -v`, but even that may not be installed!
33-
err := exec.Command(c, "version", "--client").Run()
34-
if err == nil {
35-
client = c
36-
break
37-
}
38-
}
39-
if client == "" {
40-
t.Fatal("k8s client not found")
41-
}
42-
t.Logf("Using %q as k8s client", client)
38+
token := createServiceAccountToken(t, client, namespace, "operator-controller-controller-manager")
39+
40+
createCurlPod(t, client, namespace, "oper-curl-metrics", "operator-controller-controller-manager")
41+
waitForPodReady(t, client, namespace, "oper-curl-metrics")
42+
43+
validateMetricsEndpoint(
44+
t, client, namespace, "oper-curl-metrics", token,
45+
"https://operator-controller-service."+namespace+".svc.cluster.local:8443/metrics",
46+
)
47+
}
48+
49+
// TestCatalogdMetricsExportedEndpoint verifies that the metrics endpoint for catalogd
50+
func TestCatalogdMetricsExportedEndpoint(t *testing.T) {
51+
client := utils.FindK8sClient(t)
52+
namespace := getNamespace(t, client, "control-plane=catalogd-controller-manager")
53+
54+
createClusterRoleBinding(
55+
t, client, "catalogd-metrics-binding",
56+
"catalogd-metrics-reader", namespace,
57+
"catalogd-controller-manager",
58+
)
59+
60+
token := createServiceAccountToken(t, client, namespace, "catalogd-controller-manager")
61+
62+
createCurlPod(t, client, namespace, "catalogd-curl-metrics", "catalogd-controller-manager")
63+
waitForPodReady(t, client, namespace, "catalogd-curl-metrics")
4364

44-
t.Log("Determining operator-controller namespace")
45-
cmd := exec.Command(client, "get", "pods", "--all-namespaces", "--selector=control-plane=operator-controller-controller-manager", "--output=jsonpath={.items[0].metadata.namespace}")
65+
validateMetricsEndpoint(
66+
t, client, namespace, "catalogd-curl-metrics", token,
67+
"https://catalogd-service."+namespace+".svc.cluster.local:7443/metrics",
68+
)
69+
}
70+
71+
func getNamespace(t *testing.T, client, selector string) string {
72+
cmd := exec.Command(client, "get", "pods", "--all-namespaces", "--selector="+selector, "--output=jsonpath={.items[0].metadata.namespace}")
4673
output, err := cmd.CombinedOutput()
47-
require.NoError(t, err, "Error creating determining operator-controller namespace: %s", string(output))
48-
namespace := string(output)
74+
require.NoError(t, err, "Error determining namespace: %s", string(output))
75+
76+
namespace := string(bytes.TrimSpace(output))
4977
if namespace == "" {
50-
t.Fatal("No operator-controller namespace found")
78+
t.Fatal("No namespace found for selector " + selector)
5179
}
52-
t.Logf("Using %q as operator-controller namespace", namespace)
80+
return namespace
81+
}
5382

54-
t.Log("Creating ClusterRoleBinding for operator controller metrics")
55-
cmd = exec.Command(client, "create", "clusterrolebinding", "operator-controller-metrics-binding",
56-
"--clusterrole=operator-controller-metrics-reader",
57-
"--serviceaccount="+namespace+":operator-controller-controller-manager")
58-
output, err = cmd.CombinedOutput()
83+
func createClusterRoleBinding(t *testing.T, client, name, clusterRole, namespace, serviceAccount string) {
84+
t.Logf("Creating ClusterRoleBinding %s", name)
85+
cmd := exec.Command(client, "create", "clusterrolebinding", name,
86+
"--clusterrole="+clusterRole,
87+
"--serviceaccount="+namespace+":"+serviceAccount)
88+
output, err := cmd.CombinedOutput()
5989
require.NoError(t, err, "Error creating ClusterRoleBinding: %s", string(output))
6090

6191
defer func() {
62-
t.Log("Cleaning up ClusterRoleBinding")
63-
_ = exec.Command(client, "delete", "clusterrolebinding", "operator-controller-metrics-binding", "--ignore-not-found=true").Run()
92+
t.Logf("Cleaning up ClusterRoleBinding %s", name)
93+
_ = exec.Command(client, "delete", "clusterrolebinding", name, "--ignore-not-found=true").Run()
6494
}()
95+
}
6596

97+
func createServiceAccountToken(t *testing.T, client, namespace, serviceAccount string) string {
6698
t.Log("Generating ServiceAccount token")
67-
tokenCmd := exec.Command(client, "create", "token", "operator-controller-controller-manager", "-n", namespace)
68-
tokenOutput, tokenCombinedOutput, err := stdoutAndCombined(tokenCmd)
99+
cmd := exec.Command(client, "create", "token", serviceAccount, "-n", namespace)
100+
tokenOutput, tokenCombinedOutput, err := stdoutAndCombined(cmd)
69101
require.NoError(t, err, "Error creating token: %s", string(tokenCombinedOutput))
70-
token = string(bytes.TrimSpace(tokenOutput))
102+
return string(bytes.TrimSpace(tokenOutput))
103+
}
71104

105+
func createCurlPod(t *testing.T, client, namespace, podName, serviceAccount string) {
72106
t.Log("Creating curl pod to validate the metrics endpoint")
73-
cmd = exec.Command(client, "run", curlPod,
107+
cmd := exec.Command(client, "run", podName,
74108
"--image=curlimages/curl:7.87.0", "-n", namespace,
75109
"--restart=Never",
76110
"--overrides", `{
@@ -81,154 +115,62 @@ func TestOperatorControllerMetricsExportedEndpoint(t *testing.T) {
81115
"command": ["sh", "-c", "sleep 3600"],
82116
"securityContext": {
83117
"allowPrivilegeEscalation": false,
84-
"capabilities": {
85-
"drop": ["ALL"]
86-
},
118+
"capabilities": {"drop": ["ALL"]},
87119
"runAsNonRoot": true,
88120
"runAsUser": 1000,
89-
"seccompProfile": {
90-
"type": "RuntimeDefault"
91-
}
121+
"seccompProfile": {"type": "RuntimeDefault"}
92122
}
93123
}],
94-
"serviceAccountName": "operator-controller-controller-manager"
124+
"serviceAccountName": "`+serviceAccount+`"
95125
}
96126
}`)
97-
output, err = cmd.CombinedOutput()
127+
output, err := cmd.CombinedOutput()
98128
require.NoError(t, err, "Error creating curl pod: %s", string(output))
99129

100130
defer func() {
101131
t.Log("Cleaning up curl pod")
102-
_ = exec.Command(client, "delete", "pod", curlPod, "-n", namespace, "--ignore-not-found=true").Run()
132+
_ = exec.Command(client, "delete", "pod", podName, "-n", namespace, "--ignore-not-found=true").Run()
103133
}()
104-
105-
t.Log("Waiting for the curl pod to be ready")
106-
waitCmd := exec.Command(client, "wait", "--for=condition=Ready", "pod", curlPod, "-n", namespace, "--timeout=60s")
107-
waitOutput, waitErr := waitCmd.CombinedOutput()
108-
require.NoError(t, waitErr, "Error waiting for curl pod to be ready: %s", string(waitOutput))
109-
110-
t.Log("Validating the metrics endpoint")
111-
metricsURL := "https://operator-controller-service." + namespace + ".svc.cluster.local:8443/metrics"
112-
curlCmd := exec.Command(client, "exec", curlPod, "-n", namespace, "--",
113-
"curl", "-v", "-k", "-H", "Authorization: Bearer "+token, metricsURL)
114-
output, err = curlCmd.CombinedOutput()
115-
require.NoError(t, err, "Error calling metrics endpoint: %s", string(output))
116-
require.Contains(t, string(output), "200 OK", "Metrics endpoint did not return 200 OK")
117134
}
118135

119-
// nolint:gosec
120-
// TestCatalogdMetricsExportedEndpoint verifies that the metrics endpoint for the catalogd
121-
// is exported correctly and accessible by authorized users through RBAC and a ServiceAccount token.
122-
// The test performs the following steps:
123-
// 1. Creates a ClusterRoleBinding to grant necessary permissions for accessing metrics.
124-
// 2. Generates a ServiceAccount token for authentication.
125-
// 3. Deploys a curl pod to interact with the metrics endpoint.
126-
// 4. Waits for the curl pod to become ready.
127-
// 5. Executes a curl command from the pod to validate the metrics endpoint.
128-
// 6. Cleans up all resources created during the test, such as the ClusterRoleBinding and curl pod.
129-
func TestCatalogdMetricsExportedEndpoint(t *testing.T) {
130-
var (
131-
token string
132-
curlPod = "curl-metrics"
133-
client = ""
134-
clients = []string{"kubectl", "oc"}
135-
)
136+
func waitForPodReady(t *testing.T, client, namespace, podName string) {
137+
t.Logf("Waiting for pod %s to appear before checking readiness...", podName)
138+
maxRetries := 10
139+
delay := time.Second * 3
136140

137-
t.Log("Looking for k8s client")
138-
for _, c := range clients {
139-
// Would prefer to use `command -v`, but even that may not be installed!
140-
err := exec.Command(c, "version", "--client").Run()
141+
// Ensure the pod exists before waiting for readiness
142+
for i := 0; i < maxRetries; i++ {
143+
cmd := exec.Command(client, "get", "pod", podName, "-n", namespace)
144+
_, err := cmd.CombinedOutput()
141145
if err == nil {
142-
client = c
146+
t.Logf("Pod %s found, proceeding to wait for readiness...", podName)
143147
break
144148
}
149+
t.Logf("Pod %s not found yet, retrying... (%d/%d)", podName, i+1, maxRetries)
150+
time.Sleep(delay)
145151
}
146-
if client == "" {
147-
t.Fatal("k8s client not found")
148-
}
149-
t.Logf("Using %q as k8s client", client)
150152

151-
t.Log("Determining catalogd namespace")
152-
cmd := exec.Command(client, "get", "pods", "--all-namespaces", "--selector=control-plane=catalogd-controller-manager", "--output=jsonpath={.items[0].metadata.namespace}")
153+
// Now wait for the pod to become ready
154+
t.Logf("Waiting for pod %s to be ready...", podName)
155+
cmd := exec.Command(client, "wait", "--for=condition=Ready", "pod", podName, "-n", namespace, "--timeout=120s")
153156
output, err := cmd.CombinedOutput()
154-
require.NoError(t, err, "Error creating determining catalogd namespace: %s", string(output))
155-
namespace := string(output)
156-
if namespace == "" {
157-
t.Fatal("No catalogd namespace found")
158-
}
159-
t.Logf("Using %q as catalogd namespace", namespace)
160-
161-
t.Log("Creating ClusterRoleBinding for metrics access")
162-
cmd = exec.Command(client, "create", "clusterrolebinding", "catalogd-metrics-binding",
163-
"--clusterrole=catalogd-metrics-reader",
164-
"--serviceaccount="+namespace+":catalogd-controller-manager")
165-
output, err = cmd.CombinedOutput()
166-
require.NoError(t, err, "Error creating ClusterRoleBinding: %s", string(output))
167-
168-
defer func() {
169-
t.Log("Cleaning up ClusterRoleBinding")
170-
_ = exec.Command(client, "delete", "clusterrolebinding", "catalogd-metrics-binding", "--ignore-not-found=true").Run()
171-
}()
172-
173-
t.Log("Creating service account token for authentication")
174-
tokenCmd := exec.Command(client, "create", "token", "catalogd-controller-manager", "-n", namespace)
175-
tokenOutput, tokenCombinedOutput, err := stdoutAndCombined(tokenCmd)
176-
require.NoError(t, err, "Error creating token: %s", string(tokenCombinedOutput))
177-
token = string(bytes.TrimSpace(tokenOutput))
178-
179-
t.Log("Creating a pod to run curl commands")
180-
cmd = exec.Command(client, "run", curlPod,
181-
"--image=curlimages/curl:7.87.0", "-n", namespace,
182-
"--restart=Never",
183-
"--overrides", `{
184-
"spec": {
185-
"containers": [{
186-
"name": "curl",
187-
"image": "curlimages/curl:7.87.0",
188-
"command": ["sh", "-c", "sleep 3600"],
189-
"securityContext": {
190-
"allowPrivilegeEscalation": false,
191-
"capabilities": {
192-
"drop": ["ALL"]
193-
},
194-
"runAsNonRoot": true,
195-
"runAsUser": 1000,
196-
"seccompProfile": {
197-
"type": "RuntimeDefault"
198-
}
199-
}
200-
}],
201-
"serviceAccountName": "catalogd-controller-manager"
202-
}
203-
}`)
204-
output, err = cmd.CombinedOutput()
205-
require.NoError(t, err, "Error creating curl pod: %s", string(output))
206-
207-
defer func() {
208-
t.Log("Cleaning up curl pod")
209-
_ = exec.Command(client, "delete", "pod", curlPod, "-n", namespace, "--ignore-not-found=true").Run()
210-
}()
211-
212-
t.Log("Waiting for the curl pod to become ready")
213-
waitCmd := exec.Command(client, "wait", "--for=condition=Ready", "pod", curlPod, "-n", namespace, "--timeout=60s")
214-
waitOutput, waitErr := waitCmd.CombinedOutput()
215-
require.NoError(t, waitErr, "Error waiting for curl pod to be ready: %s", string(waitOutput))
157+
require.NoError(t, err, "Error waiting for pod to be ready: %s", string(output))
158+
}
216159

160+
func validateMetricsEndpoint(t *testing.T, client, namespace, podName, token, metricsURL string) {
217161
t.Log("Validating the metrics endpoint")
218-
metricsURL := "https://catalogd-service." + namespace + ".svc.cluster.local:7443/metrics"
219-
curlCmd := exec.Command(client, "exec", curlPod, "-n", namespace, "--",
162+
cmd := exec.Command(client, "exec", podName, "-n", namespace, "--",
220163
"curl", "-v", "-k", "-H", "Authorization: Bearer "+token, metricsURL)
221-
output, err = curlCmd.CombinedOutput()
164+
output, err := cmd.CombinedOutput()
222165
require.NoError(t, err, "Error calling metrics endpoint: %s", string(output))
223166
require.Contains(t, string(output), "200 OK", "Metrics endpoint did not return 200 OK")
224167
}
225168

226169
func stdoutAndCombined(cmd *exec.Cmd) ([]byte, []byte, error) {
227-
var outOnly bytes.Buffer
228-
var outAndErr bytes.Buffer
170+
var outOnly, outAndErr bytes.Buffer
229171
allWriter := io.MultiWriter(&outOnly, &outAndErr)
230-
cmd.Stderr = &outAndErr
231172
cmd.Stdout = allWriter
173+
cmd.Stderr = &outAndErr
232174
err := cmd.Run()
233175
return outOnly.Bytes(), outAndErr.Bytes(), err
234176
}

‎test/utils/utils.go

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
package utils
2+
3+
import (
4+
"os/exec"
5+
"testing"
6+
)
7+
8+
// FindK8sClient returns the first available Kubernetes CLI client from the system,
9+
// It checks for the existence of each client by running `version --client`.
10+
// If no suitable client is found, the function terminates the test with a failure.
11+
func FindK8sClient(t *testing.T) string {
12+
t.Logf("Finding kubectl client")
13+
clients := []string{"kubectl", "oc"}
14+
for _, c := range clients {
15+
// Would prefer to use `command -v`, but even that may not be installed!
16+
if err := exec.Command(c, "version", "--client").Run(); err == nil {
17+
t.Logf("Using %q as k8s client", c)
18+
return c
19+
}
20+
}
21+
t.Fatal("k8s client not found")
22+
return ""
23+
}

0 commit comments

Comments
 (0)