Skip to content

OCPBUGS-60564: [OTE] Add webhook to validate openshift-service-ca certificate rotation #450

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Aug 21, 2025
Merged

OCPBUGS-60564: [OTE] Add webhook to validate openshift-service-ca certificate rotation #450

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dbad83e
UPSTREAM: <carry>: [OTE] Add webhook to validate openshift-service-ca…
camilamacedo86 Aug 13, 2025
f3a2561
UPSTREAM: <carry>: Adds ResourceVersion checks to the tls secret dele…
camilamacedo86 Aug 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions openshift/tests-extension/.openshift-tests-extension/openshift_payload_olmv1.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,16 @@
"lifecycle": "blocking",
"environmentSelector": {}
},
{
"name": "[sig-olmv1][OCPFeatureGate:NewOLMWebhookProviderOpenshiftServiceCA][Skipped:Disconnected][Serial] OLMv1 operator with webhooks should be tolerant to openshift-service-ca certificate rotation",
"labels": {},
"resources": {
"isolation": {}
},
"source": "openshift:payload:olmv1",
"lifecycle": "blocking",
"environmentSelector": {}
},
{
"name": "[sig-olmv1][OCPFeatureGate:NewOLMWebhookProviderOpenshiftServiceCA][Skipped:Disconnected][Serial] OLMv1 operator with webhooks should be tolerant to tls secret deletion",
"labels": {},
Expand Down
90 changes: 83 additions & 7 deletions openshift/tests-extension/test/webhooks.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,12 @@ import (
)

const (
webhookCatalogName = "webhook-operator-catalog"
webhookOperatorPackageName = "webhook-operator"
webhookOperatorCRDName = "webhooktests.webhook.operators.coreos.io"
webhookServiceCert = "webhook-operator-webhook-service-cert"
openshiftServiceCANamespace = "openshift-service-ca"
openshiftServiceCASigningKeySecretName = "signing-key"
webhookCatalogName = "webhook-operator-catalog"
webhookOperatorPackageName = "webhook-operator"
webhookOperatorCRDName = "webhooktests.webhook.operators.coreos.io"
webhookServiceCert = "webhook-operator-webhook-service-cert"
)

var _ = Describe("[sig-olmv1][OCPFeatureGate:NewOLMWebhookProviderOpenshiftServiceCA][Skipped:Disconnected][Serial] OLMv1 operator with webhooks",
Expand Down Expand Up @@ -84,7 +86,6 @@ var _ = Describe("[sig-olmv1][OCPFeatureGate:NewOLMWebhookProviderOpenshiftServi
helpers.DescribeAllClusterCatalogs(ctx)
helpers.DescribeAllClusterExtensions(ctx, webhookOperatorInstallNamespace)
By("dumping webhook diagnostics")
// Additional diagnostics specific for this test
helpers.RunAndPrint(ctx, "get", "mutatingwebhookconfigurations.admissionregistration.k8s.io", "-oyaml")
helpers.RunAndPrint(ctx, "get", "validatingwebhookconfigurations.admissionregistration.k8s.io", "-oyaml")
}
Expand Down Expand Up @@ -167,14 +168,88 @@ var _ = Describe("[sig-olmv1][OCPFeatureGate:NewOLMWebhookProviderOpenshiftServi
}))
})

Copy link
Contributor Author

@camilamacedo86 camilamacedo86 Aug 20, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Orginal that does not work well was:

https://github.com/openshift/origin/pull/30059/files#diff-6dd6fa78ac85235012d3c9910f8510bc1640c830a83888681bd5922cec0dffcbR149-R164

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What was the problem?

It("should be tolerant to openshift-service-ca certificate rotation", func(ctx SpecContext) {
certificateSecretName := webhookServiceCert
var oldSecretResourceVersion string

By("ensuring the webhook operator's service certificate secret exists and getting its ResourceVersion")
Eventually(func(g Gomega) {
secret := &corev1.Secret{}
err := k8sClient.Get(ctx, client.ObjectKey{Name: certificateSecretName, Namespace: webhookOperatorInstallNamespace}, secret)
g.Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("failed to get webhook service certificate secret %s/%s", webhookOperatorInstallNamespace, certificateSecretName))
g.Expect(secret.Data).ToNot(BeEmpty(), "expected webhook service certificate secret data to not be empty")
oldSecretResourceVersion = secret.ResourceVersion
g.Expect(oldSecretResourceVersion).ToNot(BeEmpty(), "expected secret ResourceVersion to not be empty")
}).WithTimeout(3 * time.Minute).WithPolling(5 * time.Second).Should(Succeed())

By("deleting the openshift-service-ca signing-key secret")
signingKeySecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: openshiftServiceCASigningKeySecretName,
Namespace: openshiftServiceCANamespace,
},
}
err := k8sClient.Delete(ctx, signingKeySecret, client.PropagationPolicy(metav1.DeletePropagationBackground))
Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred())

By("waiting for the webhook operator's service certificate secret to be recreated with a new ResourceVersion")
Eventually(func(g Gomega) {
secret := &corev1.Secret{}
err := k8sClient.Get(ctx, client.ObjectKey{Name: certificateSecretName, Namespace: webhookOperatorInstallNamespace}, secret)
if apierrors.IsNotFound(err) {
GinkgoLogr.Info(fmt.Sprintf("Secret %s/%s not found yet (still polling for recreation)", webhookOperatorInstallNamespace, certificateSecretName))
return
}
g.Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("failed to get webhook service certificate secret %s/%s: %v", webhookOperatorInstallNamespace, certificateSecretName, err))
g.Expect(secret.ResourceVersion).ToNot(Equal(oldSecretResourceVersion), "expected secret ResourceVersion to be different from the old one")
g.Expect(secret.Data).ToNot(BeEmpty(), "expected webhook service certificate secret data to not be empty after recreation")
}).WithTimeout(5*time.Minute).WithPolling(10*time.Second).Should(Succeed(), "webhook service certificate secret did not get recreated with a new ResourceVersion and populated within timeout")

By("checking webhook is responsive through cert rotation")
Eventually(func(g Gomega) {
resourceName := fmt.Sprintf("cert-rotation-test-%s", rand.String(5))
resource := newWebhookTest(resourceName, webhookOperatorInstallNamespace, true)

_, err := dynamicClient.Resource(webhookTestV1).Namespace(webhookOperatorInstallNamespace).Create(ctx, resource, metav1.CreateOptions{})
g.Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("failed to create test resource %s: %v", resourceName, err))

err = dynamicClient.Resource(webhookTestV1).Namespace(webhookOperatorInstallNamespace).Delete(ctx, resource.GetName(), metav1.DeleteOptions{})
g.Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred(), fmt.Sprintf("failed to delete test resource %s: %v", resourceName, err))
}).WithTimeout(5 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())

DeferCleanup(func() {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add debug to validate the cert in case of failures
It should us identify the issue better if we need to do so.

// Specific check for this test
if CurrentSpecReport().Failed() {
By("dumping certificate details for debugging")
secret := &corev1.Secret{}
if err := k8sClient.Get(ctx, client.ObjectKey{
Name: webhookServiceCert,
Namespace: webhookOperatorInstallNamespace,
}, secret); err == nil {
if crt, ok := secret.Data["tls.crt"]; ok && len(crt) > 0 {
printTLSCertInfo(crt)
} else {
_, _ = GinkgoWriter.Write([]byte("[diag] tls.crt key not found or empty in secret\n"))
}
} else {
fmt.Fprintf(GinkgoWriter, "[diag] failed to get secret for cert dump: %v\n", err)
}
}
})
})

It("should be tolerant to tls secret deletion", func(ctx SpecContext) {
certificateSecretName := webhookServiceCert
By("ensuring secret exists before deletion attempt")
var oldSecretResourceVersion string

By("ensuring secret exists before deletion attempt and getting its ResourceVersion")
Eventually(func(g Gomega) {
secret := &corev1.Secret{}
err := k8sClient.Get(ctx, client.ObjectKey{Name: certificateSecretName, Namespace: webhookOperatorInstallNamespace}, secret)
g.Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("failed to get secret %s/%s", webhookOperatorInstallNamespace, certificateSecretName))
}).WithTimeout(1 * time.Minute).WithPolling(5 * time.Second).Should(Succeed())
oldSecretResourceVersion = secret.ResourceVersion
g.Expect(oldSecretResourceVersion).ToNot(BeEmpty(), "expected secret ResourceVersion to not be empty")
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@perdasilva I did your suggestion here as well

}).WithTimeout(5 * time.Minute).WithPolling(5 * time.Second).Should(Succeed())

By("checking webhook is responsive through secret recreation after manual deletion")
tlsSecret := &corev1.Secret{
Expand Down Expand Up @@ -215,6 +290,7 @@ var _ = Describe("[sig-olmv1][OCPFeatureGate:NewOLMWebhookProviderOpenshiftServi
return
}
g.Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("failed to get webhook service certificate secret %s/%s: %v", webhookOperatorInstallNamespace, certificateSecretName, err))
g.Expect(secret.ResourceVersion).ToNot(Equal(oldSecretResourceVersion), "expected secret ResourceVersion to be different from the old one")
g.Expect(secret.Data).ToNot(BeEmpty(), "expected webhook service certificate secret data to not be empty after recreation")
}).WithTimeout(5*time.Minute).WithPolling(10*time.Second).Should(Succeed(), "webhook service certificate secret did not get recreated and populated within timeout")

Expand Down