MON-3701: clean-up trusted CA bundle for Alertmanager

CMO needed to inject the hashed version of the trusted CA bundle
ConfigMap into the Alertmanager resource when OAuth proxy was used
because it couldn't detect updates to the bundle and reload it.

The trusted CA bundle is still required to be mounted into the pod for
the Alertmanager container but it is declared directly in the static
manifest instead of being injected at runtime by CMO since Alertmanager
reloads the CA whenever it changes.

Signed-off-by: Simon Pasquier <[email protected]>

Author: simonpasquier

assets/alertmanager/alertmanager.yaml

@@ -143,10 +143,19 @@ spec:
     runAsUser: 65534
   serviceAccountName: alertmanager-main
   version: 0.26.0
+  volumeMounts:
+  - mountPath: /etc/pki/ca-trust/extracted/pem/
+    name: alertmanager-trusted-ca-bundle
   volumes:
   - configMap:
       name: metrics-client-ca
     name: metrics-client-ca
+  - configMap:
+      items:
+      - key: ca-bundle.crt
+        path: tls-ca-bundle.pem
+      name: alertmanager-trusted-ca-bundle
+    name: alertmanager-trusted-ca-bundle
   web:
     httpConfig:
       headers:

jsonnet/components/alertmanager.libsonnet

@@ -16,7 +16,6 @@ function(params)
     trustedCaBundle: generateCertInjection.trustedCNOCaBundleCM(cfg.namespace, 'alertmanager-trusted-ca-bundle'),
 
     // OpenShift route to access the Alertmanager UI.
-
     route: {
       apiVersion: 'v1',
       kind: 'Route',
@@ -411,6 +410,22 @@ function(params)
               name: 'metrics-client-ca',
             },
           },
+          {
+            name: $.trustedCaBundle.metadata.name,
+            configMap: {
+              name: $.trustedCaBundle.metadata.name,
+              items: [{
+                key: 'ca-bundle.crt',
+                path: 'tls-ca-bundle.pem',
+              }],
+            },
+          },
+        ],
+        volumeMounts+: [
+          {
+            name: $.trustedCaBundle.metadata.name,
+            mountPath: '/etc/pki/ca-trust/extracted/pem/',
+          },
         ],
       },
     },

jsonnet/utils/generate-certificate-injection.libsonnet

@@ -15,6 +15,7 @@
     },
     data: {},
   },
+
   // This CA bundle is injected by the service-ca-operator.
   SCOCaBundleCM(cfgNamespace, cfgName):: {
     apiVersion: 'v1',
@@ -28,6 +29,7 @@
     },
     data: {},
   },
+
   SCOCaBundleVolume(volName):: {
     name: volName,
     configmap: {

pkg/manifests/manifests.go

@@ -586,7 +586,7 @@ func setupStartupProbe(a *monv1.Alertmanager) {
 	)
 }
 
-func (f *Factory) AlertmanagerMain(trustedCABundleCM *v1.ConfigMap) (*monv1.Alertmanager, error) {
+func (f *Factory) AlertmanagerMain() (*monv1.Alertmanager, error) {
 	a, err := f.NewAlertmanager(f.assets.MustNewAssetSlice(AlertmanagerMain))
 	if err != nil {
 		return nil, err
@@ -651,18 +651,6 @@ func (f *Factory) AlertmanagerMain(trustedCABundleCM *v1.ConfigMap) (*monv1.Aler
 			f.config.ClusterMonitoringConfiguration.AlertmanagerMainConfig.TopologySpreadConstraints
 	}
 
-	if trustedCABundleCM != nil {
-		volumeName := "alertmanager-trusted-ca-bundle"
-		a.Spec.VolumeMounts = append(a.Spec.VolumeMounts, trustedCABundleVolumeMount(volumeName))
-
-		volume := trustedCABundleVolume(trustedCABundleCM.Name, volumeName)
-		volume.VolumeSource.ConfigMap.Items = append(volume.VolumeSource.ConfigMap.Items, v1.KeyToPath{
-			Key:  TrustedCABundleKey,
-			Path: "tls-ca-bundle.pem",
-		})
-		a.Spec.Volumes = append(a.Spec.Volumes, volume)
-	}
-
 	for i, c := range a.Spec.Containers {
 		switch c.Name {
 		case "kube-rbac-proxy", "kube-rbac-proxy-metric", "kube-rbac-proxy-web":

pkg/manifests/manifests_test.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net/url"
 	"reflect"
+	"slices"
 	"sort"
 	"strings"
 	"testing"
@@ -33,7 +34,6 @@ import (
 	policyv1 "k8s.io/api/policy/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/utils/strings/slices"
 )
 
 type fakeInfrastructureReader struct {
@@ -251,7 +251,7 @@ func TestUnconfiguredManifests(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	_, err = f.AlertmanagerMain(nil)
+	_, err = f.AlertmanagerMain()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -3041,9 +3041,7 @@ func TestAlertmanagerMainStartupProbe(t *testing.T) {
 				t.Fatal(err)
 			}
 			f := NewFactory("openshift-monitoring", "openshift-user-workload-monitoring", c, tc.infrastructure, &fakeProxyReader{}, NewAssets(assetsPath), &APIServerConfig{}, &configv1.Console{})
-			a, err := f.AlertmanagerMain(
-				&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
-			)
+			a, err := f.AlertmanagerMain()
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -3115,9 +3113,7 @@ ingress:
 	})
 
 	f := NewFactory("openshift-monitoring", "openshift-user-workload-monitoring", c, defaultInfrastructureReader(), &fakeProxyReader{}, NewAssets(assetsPath), &APIServerConfig{}, &configv1.Console{Status: configv1.ConsoleStatus{ConsoleURL: "https://console-openshift-console.apps.foo.devcluster.openshift.com"}})
-	a, err := f.AlertmanagerMain(
-		&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
-	)
+	a, err := f.AlertmanagerMain()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -3241,6 +3237,17 @@ ingress:
 		}
 	}
 
+	if !slices.ContainsFunc(a.Spec.Volumes, func(v v1.Volume) bool {
+		return v.Name == "alertmanager-trusted-ca-bundle"
+	}) {
+		t.Fatalf("expected to find volume with name 'alertmanager-trusted-ca-bundle' but got none: %#v", a.Spec.Volumes)
+	}
+
+	if !slices.ContainsFunc(a.Spec.VolumeMounts, func(v v1.VolumeMount) bool {
+		return v.Name == "alertmanager-trusted-ca-bundle"
+	}) {
+		t.Fatalf("expected to find volume mount with name 'alertmanager-trusted-ca-bundle' but got none: %#v", a.Spec.VolumeMounts)
+	}
 }
 
 func TestAlertManagerUserWorkloadConfiguration(t *testing.T) {
@@ -4360,9 +4367,7 @@ func TestNonHighlyAvailableInfrastructure(t *testing.T) {
 		{
 			name: "Alertmanager",
 			getSpec: func(f *Factory) (spec, error) {
-				a, err := f.AlertmanagerMain(
-					&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
-				)
+				a, err := f.AlertmanagerMain()
 				if err != nil {
 					return spec{}, err
 				}

pkg/tasks/alertmanager.go

@@ -185,17 +185,12 @@ func (t *AlertmanagerTask) create(ctx context.Context) error {
 			return fmt.Errorf("initializing Alertmanager CA bundle ConfigMap failed: %w", err)
 		}
 
-		cbs := &caBundleSyncer{
-			client:  t.client,
-			factory: t.factory,
-			prefix:  "alertmanager",
-		}
-		trustedCA, err = cbs.syncTrustedCABundle(ctx, trustedCA)
+		err = t.client.CreateOrUpdateConfigMap(ctx, trustedCA)
 		if err != nil {
-			return fmt.Errorf("syncing Alertmanager trusted CA bundle ConfigMap failed: %w", err)
+			return fmt.Errorf("reconciling Alertmanager trusted CA bundle ConfigMap failed: %w", err)
 		}
 
-		a, err := t.factory.AlertmanagerMain(trustedCA)
+		a, err := t.factory.AlertmanagerMain()
 		if err != nil {
 			return fmt.Errorf("initializing Alertmanager object failed: %w", err)
 		}
@@ -239,6 +234,12 @@ func (t *AlertmanagerTask) create(ctx context.Context) error {
 		return fmt.Errorf("deleting Alertmanager proxy Secret failed: %w", err)
 	}
 
+	// TODO(simonpasquier): remove this step after OCP 4.16 is released.
+	err = t.client.DeleteHashedConfigMap(ctx, "openshift-monitoring", "alertmanager-trusted-ca-bundle", "")
+	if err != nil {
+		return fmt.Errorf("deleting all hashed Alertmanager trusted CA bundle ConfigMap failed: %w", err)
+	}
+
 	return nil
 }
 
@@ -356,7 +357,7 @@ func (t *AlertmanagerTask) destroy(ctx context.Context) error {
 			return fmt.Errorf("deleting Alertmanager trusted CA bundle failed: %w", err)
 		}
 
-		a, err := t.factory.AlertmanagerMain(trustedCA)
+		a, err := t.factory.AlertmanagerMain()
 		if err != nil {
 			return fmt.Errorf("initializing Alertmanager object failed: %w", err)
 		}

test/e2e/alertmanager_test.go

@@ -22,12 +22,12 @@ import (
 	"io"
 	"net/http"
 	"reflect"
+	"slices"
 	"testing"
 	"time"
 
 	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	monitoringv1beta1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1beta1"
-	"k8s.io/utils/strings/slices"
 
 	"github.com/Jeffail/gabs/v2"
 	statusv1 "github.com/openshift/api/config/v1"
@@ -39,46 +39,6 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 )
 
-func TestAlertmanagerTrustedCA(t *testing.T) {
-	var (
-		newCM   *v1.ConfigMap
-		lastErr error
-	)
-
-	cm := f.MustGetConfigMap(t, "alertmanager-trusted-ca-bundle", f.Ns)
-	newCM, err := f.ManifestsFactory.HashTrustedCA(cm, "alertmanager")
-	if err != nil {
-		t.Fatal(fmt.Errorf("no trusted CA bundle data available: %w", err))
-	}
-
-	// Wait for the new hashed trusted CA bundle ConfigMap to be created
-	f.AssertConfigmapExists(newCM.Name, f.Ns)(t)
-
-	// Get Alertmanager StatefulSet and make sure it has a volume mounted.
-	err = wait.Poll(time.Second, 5*time.Minute, func() (bool, error) {
-		ss := f.MustGetStatefulSet(t, "alertmanager-main", f.Ns)
-
-		if len(ss.Spec.Template.Spec.Containers[0].VolumeMounts) == 0 {
-			return false, errors.New("Could not find any VolumeMounts, expected at least 1")
-		}
-
-		for _, mount := range ss.Spec.Template.Spec.Containers[0].VolumeMounts {
-			if mount.Name == "alertmanager-trusted-ca-bundle" {
-				return true, nil
-			}
-		}
-
-		lastErr = fmt.Errorf("no volume %s mounted", newCM.Name)
-		return false, nil
-	})
-	if err != nil {
-		if err == wait.ErrWaitTimeout && lastErr != nil {
-			err = lastErr
-		}
-		t.Fatal(err)
-	}
-}
-
 // TestAlertmanagerTenancyAPI ensures that the Alertmanager API exposed on the
 // tenancy port enforces the namespace value.
 func TestAlertmanagerTenancyAPI(t *testing.T) {