From 9f8c7165addc91a22384a3ea7ea1ee0d49ec4e89 Mon Sep 17 00:00:00 2001 From: Mircea Cosbuc Date: Thu, 24 Nov 2022 20:57:48 +0100 Subject: [PATCH 1/3] Use different mount path for prometheus TLS secret --- controllers/mongodb_tls.go | 16 ++--- controllers/mongodb_tls_test.go | 87 ++++++++++++++++++++++++-- controllers/prometheus.go | 2 +- test/e2e/prometheus/prometheus_test.go | 9 ++- 4 files changed, 96 insertions(+), 18 deletions(-) diff --git a/controllers/mongodb_tls.go b/controllers/mongodb_tls.go index eff16b031..a3d690e1b 100644 --- a/controllers/mongodb_tls.go +++ b/controllers/mongodb_tls.go @@ -22,12 +22,13 @@ import ( ) const ( - tlsCAMountPath = "/var/lib/tls/ca/" - tlsCACertName = "ca.crt" - tlsOperatorSecretMountPath = "/var/lib/tls/server/" //nolint - tlsSecretCertName = "tls.crt" //nolint - tlsSecretKeyName = "tls.key" - tlsSecretPemName = "tls.pem" + tlsCAMountPath = "/var/lib/tls/ca/" + tlsCACertName = "ca.crt" + tlsOperatorSecretMountPath = "/var/lib/tls/server/" //nolint + tlsPrometheusSecretMountPath = "/var/lib/tls/prometheus/" //nolint + tlsSecretCertName = "tls.crt" + tlsSecretKeyName = "tls.key" + tlsSecretPemName = "tls.pem" ) // validateTLSConfig will check that the configured ConfigMap and Secret exist and that they have the correct fields. @@ -316,8 +317,7 @@ func buildTLSPrometheus(mdb mdbv1.MongoDBCommunity) podtemplatespec.Modification // The same key-certificate pair is used for all servers tlsSecretVolume := statefulset.CreateVolumeFromSecret("prom-tls-secret", mdb.PrometheusTLSOperatorSecretNamespacedName().Name) - // TODO: Is it ok to use the same `tlsOperatorSecretMountPath` - tlsSecretVolumeMount := statefulset.CreateVolumeMount(tlsSecretVolume.Name, tlsOperatorSecretMountPath, statefulset.WithReadOnly(true)) + tlsSecretVolumeMount := statefulset.CreateVolumeMount(tlsSecretVolume.Name, tlsPrometheusSecretMountPath, statefulset.WithReadOnly(true)) // MongoDB expects both key and certificate to be provided in a single PEM file // We are using a secret format where they are stored in separate fields, tls.crt and tls.key diff --git a/controllers/mongodb_tls_test.go b/controllers/mongodb_tls_test.go index c1506553e..717712e5d 100644 --- a/controllers/mongodb_tls_test.go +++ b/controllers/mongodb_tls_test.go @@ -38,11 +38,17 @@ func TestStatefulSet_IsCorrectlyConfiguredWithTLS(t *testing.T) { err = mgr.GetClient().Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) assert.NoError(t, err) - assertStatefulsetVolumesAndVolumeMounts(t, sts, mdb.TLSOperatorCASecretNamespacedName().Name, mdb.TLSOperatorSecretNamespacedName().Name) + assertStatefulsetVolumesAndVolumeMounts(t, sts, mdb.TLSOperatorCASecretNamespacedName().Name, mdb.TLSOperatorSecretNamespacedName().Name, "") } -func assertStatefulsetVolumesAndVolumeMounts(t *testing.T, sts appsv1.StatefulSet, expectedTLSCASecretName string, expectedTLSOperatorSecretName string) { - assert.Len(t, sts.Spec.Template.Spec.Volumes, 8) +func assertStatefulsetVolumesAndVolumeMounts(t *testing.T, sts appsv1.StatefulSet, expectedTLSCASecretName string, expectedTLSOperatorSecretName string, expectedPromTLSSecretName string) { + prometheusTLSEnabled := expectedPromTLSSecretName != "" + + if prometheusTLSEnabled { + assert.Len(t, sts.Spec.Template.Spec.Volumes, 9) + } else { + assert.Len(t, sts.Spec.Template.Spec.Volumes, 8) + } permission := int32(416) assert.Contains(t, sts.Spec.Template.Spec.Volumes, corev1.Volume{ Name: "tls-ca", @@ -62,6 +68,17 @@ func assertStatefulsetVolumesAndVolumeMounts(t *testing.T, sts appsv1.StatefulSe }, }, }) + if prometheusTLSEnabled { + assert.Contains(t, sts.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: "prom-tls-secret", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: expectedPromTLSSecretName, + DefaultMode: &permission, + }, + }, + }) + } tlsSecretVolumeMount := corev1.VolumeMount{ Name: "tls-secret", @@ -73,16 +90,70 @@ func assertStatefulsetVolumesAndVolumeMounts(t *testing.T, sts appsv1.StatefulSe ReadOnly: true, MountPath: tlsCAMountPath, } + tlsPrometheusSecretVolumeMount := corev1.VolumeMount{ + Name: "prom-tls-secret", + ReadOnly: true, + MountPath: tlsPrometheusSecretMountPath, + } assert.Len(t, sts.Spec.Template.Spec.InitContainers, 2) agentContainer := sts.Spec.Template.Spec.Containers[0] assert.Contains(t, agentContainer.VolumeMounts, tlsSecretVolumeMount) assert.Contains(t, agentContainer.VolumeMounts, tlsCAVolumeMount) + if prometheusTLSEnabled { + assert.Contains(t, agentContainer.VolumeMounts, tlsPrometheusSecretVolumeMount) + } mongodbContainer := sts.Spec.Template.Spec.Containers[1] assert.Contains(t, mongodbContainer.VolumeMounts, tlsSecretVolumeMount) assert.Contains(t, mongodbContainer.VolumeMounts, tlsCAVolumeMount) + if prometheusTLSEnabled { + assert.Contains(t, mongodbContainer.VolumeMounts, tlsPrometheusSecretVolumeMount) + } +} + +func TestStatefulSet_IsCorrectlyConfiguredWithPrometheusTLS(t *testing.T) { + mdb := newTestReplicaSetWithTLS() + mdb.Spec.Prometheus = &mdbv1.Prometheus{ + Username: "username", + PasswordSecretRef: mdbv1.SecretKeyReference{ + Name: "prom-password-secret", + }, + Port: 4321, + TLSSecretRef: mdbv1.SecretKeyReference{ + Name: "prom-secret-cert", + }, + } + + mgr := kubeClient.NewManager(&mdb) + cli := mdbClient.NewClient(mgr.GetClient()) + + err := secret.CreateOrUpdate(mgr.Client, + secret.Builder(). + SetName("prom-password-secret"). + SetNamespace(mdb.Namespace). + SetField("password", "my-password"). + Build(), + ) + assert.NoError(t, err) + err = createTLSSecret(cli, mdb, "CERT", "KEY", "") + assert.NoError(t, err) + err = createPrometheusTLSSecret(cli, mdb, "CERT", "KEY", "") + assert.NoError(t, err) + + err = createTLSConfigMap(cli, mdb) + assert.NoError(t, err) + + r := NewReconciler(mgr) + res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) + assertReconciliationSuccessful(t, res, err) + + sts := appsv1.StatefulSet{} + err = mgr.GetClient().Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) + assert.NoError(t, err) + + assertStatefulsetVolumesAndVolumeMounts(t, sts, mdb.TLSOperatorCASecretNamespacedName().Name, mdb.TLSOperatorSecretNamespacedName().Name, mdb.PrometheusTLSOperatorSecretNamespacedName().Name) } func TestStatefulSet_IsCorrectlyConfiguredWithTLSAfterChangingExistingVolumes(t *testing.T) { @@ -110,7 +181,7 @@ func TestStatefulSet_IsCorrectlyConfiguredWithTLSAfterChangingExistingVolumes(t err = mgr.GetClient().Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) assert.NoError(t, err) - assertStatefulsetVolumesAndVolumeMounts(t, sts, tlsCAVolumeSecretName, mdb.TLSOperatorSecretNamespacedName().Name) + assertStatefulsetVolumesAndVolumeMounts(t, sts, tlsCAVolumeSecretName, mdb.TLSOperatorSecretNamespacedName().Name, "") // updating sts tls-ca volume directly to simulate changing of underlying volume's secret for i := range sts.Spec.Template.Spec.Volumes { @@ -122,7 +193,7 @@ func TestStatefulSet_IsCorrectlyConfiguredWithTLSAfterChangingExistingVolumes(t err = mgr.GetClient().Update(context.TODO(), &sts) assert.NoError(t, err) - assertStatefulsetVolumesAndVolumeMounts(t, sts, changedTLSCAVolumeSecretName, mdb.TLSOperatorSecretNamespacedName().Name) + assertStatefulsetVolumesAndVolumeMounts(t, sts, changedTLSCAVolumeSecretName, mdb.TLSOperatorSecretNamespacedName().Name, "") res, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}}) assertReconciliationSuccessful(t, res, err) @@ -130,7 +201,7 @@ func TestStatefulSet_IsCorrectlyConfiguredWithTLSAfterChangingExistingVolumes(t sts = appsv1.StatefulSet{} err = mgr.GetClient().Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts) assert.NoError(t, err) - assertStatefulsetVolumesAndVolumeMounts(t, sts, tlsCAVolumeSecretName, mdb.TLSOperatorSecretNamespacedName().Name) + assertStatefulsetVolumesAndVolumeMounts(t, sts, tlsCAVolumeSecretName, mdb.TLSOperatorSecretNamespacedName().Name, "") } func TestAutomationConfig_IsCorrectlyConfiguredWithTLS(t *testing.T) { @@ -422,6 +493,10 @@ func createTLSSecret(c k8sClient.Client, mdb mdbv1.MongoDBCommunity, crt string, return createTLSSecretWithNamespaceAndName(c, mdb.Namespace, mdb.Spec.Security.TLS.CertificateKeySecret.Name, crt, key, pem) } +func createPrometheusTLSSecret(c k8sClient.Client, mdb mdbv1.MongoDBCommunity, crt string, key string, pem string) error { + return createTLSSecretWithNamespaceAndName(c, mdb.Namespace, mdb.Spec.Prometheus.TLSSecretRef.Name, crt, key, pem) +} + func createUserPasswordSecret(c k8sClient.Client, mdb mdbv1.MongoDBCommunity, userPasswordSecretName string, password string) error { sBuilder := secret.Builder(). SetName(userPasswordSecretName). diff --git a/controllers/prometheus.go b/controllers/prometheus.go index c704ffb55..8e6151908 100644 --- a/controllers/prometheus.go +++ b/controllers/prometheus.go @@ -39,7 +39,7 @@ func getPrometheusModification(getUpdateCreator secret.GetUpdateCreator, mdb mdb if err != nil { return automationconfig.NOOP(), err } - tlsPEMPath = tlsOperatorSecretMountPath + tlsOperatorSecretFileName(certKey) + tlsPEMPath = tlsPrometheusSecretMountPath + tlsOperatorSecretFileName(certKey) scheme = "https" } else { scheme = "http" diff --git a/test/e2e/prometheus/prometheus_test.go b/test/e2e/prometheus/prometheus_test.go index 84fb8219f..f7c074665 100644 --- a/test/e2e/prometheus/prometheus_test.go +++ b/test/e2e/prometheus/prometheus_test.go @@ -23,13 +23,16 @@ func TestMain(m *testing.M) { } func TestPrometheus(t *testing.T) { - ctx, testConfig := setup.SetupWithTLS(t, "") + resourceName := "mdb0" + ctx, testConfig := setup.SetupWithTLS(t, resourceName) defer ctx.Teardown() - mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", testConfig.Namespace) + mdb, user := e2eutil.NewTestMongoDB(ctx, resourceName, testConfig.Namespace) + + mdb.Spec.Security.TLS = e2eutil.NewTestTLSConfig(false) mdb.Spec.Prometheus = e2eutil.NewPrometheusConfig(mdb.Namespace) - _, err := setup.GeneratePasswordForUser(ctx, user, "") + _, err := setup.GeneratePasswordForUser(ctx, user, testConfig.Namespace) if err != nil { t.Fatal(err) } From 58e5c70ba70de6f9d5ff98c4427acf42b3be0104 Mon Sep 17 00:00:00 2001 From: Mircea Cosbuc Date: Thu, 24 Nov 2022 21:34:23 +0100 Subject: [PATCH 2/3] Fix prometheus test assertions for TLS --- test/e2e/prometheus/prometheus_test.go | 28 ++++++++++++++------------ 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/test/e2e/prometheus/prometheus_test.go b/test/e2e/prometheus/prometheus_test.go index f7c074665..39e355019 100644 --- a/test/e2e/prometheus/prometheus_test.go +++ b/test/e2e/prometheus/prometheus_test.go @@ -44,21 +44,23 @@ func TestPrometheus(t *testing.T) { t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) - t.Run("Keyfile authentication is configured", tester.HasKeyfileAuth(3)) - t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds()) - t.Run("Test Prometheus endpoint is active", tester.PrometheusEndpointIsReachable("prom-user", "prom-password", false)) - t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) - t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) + mongodbtests.SkipTestIfLocal(t, "Ensure MongoDB with Prometheus configuration", func(t *testing.T) { + t.Run("Resource has TLS Mode", tester.HasTlsMode("requireSSL", 60, WithTls(mdb))) + t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds(WithTls(mdb))) + t.Run("Test Prometheus endpoint is active", tester.PrometheusEndpointIsReachable("prom-user", "prom-password", false)) + t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3, WithTls(mdb))) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1)) - t.Run("Enabling HTTPS on the Prometheus endpoint", func(t *testing.T) { - err = e2eutil.UpdateMongoDBResource(&mdb, func(mdb *v1.MongoDBCommunity) { - mdb.Spec.Prometheus.TLSSecretRef.Name = "tls-certificate" - }) - assert.NoError(t, err) + t.Run("Enabling HTTPS on the Prometheus endpoint", func(t *testing.T) { + err = e2eutil.UpdateMongoDBResource(&mdb, func(mdb *v1.MongoDBCommunity) { + mdb.Spec.Prometheus.TLSSecretRef.Name = "tls-certificate" + }) + assert.NoError(t, err) - t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) - t.Run("Test Prometheus HTTPS endpoint is active", tester.PrometheusEndpointIsReachable("prom-user", "prom-password", true)) + t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb)) + t.Run("Test Prometheus HTTPS endpoint is active", tester.PrometheusEndpointIsReachable("prom-user", "prom-password", true)) + t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 2)) + }) }) - t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 2)) } From 56cbb7a971ffa4a32685b2403278563875c6138a Mon Sep 17 00:00:00 2001 From: Mircea Cosbuc Date: Fri, 25 Nov 2022 01:27:40 +0100 Subject: [PATCH 3/3] Bump agent version and FCV test mongod versions --- release.json | 4 ++-- .../feature_compatibility_version_test.go | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/release.json b/release.json index 99c49129c..5827bbd5e 100644 --- a/release.json +++ b/release.json @@ -4,7 +4,7 @@ "version-upgrade-hook": "1.0.6", "readiness-probe": "1.0.12", "mongodb-agent": { - "version": "12.0.10.7591-1", - "tools_version": "100.5.3" + "version": "12.0.14.7630-1", + "tools_version": "100.6.0" } } diff --git a/test/e2e/feature_compatibility_version/feature_compatibility_version_test.go b/test/e2e/feature_compatibility_version/feature_compatibility_version_test.go index 9938192c6..7367cbeb1 100644 --- a/test/e2e/feature_compatibility_version/feature_compatibility_version_test.go +++ b/test/e2e/feature_compatibility_version/feature_compatibility_version_test.go @@ -29,8 +29,8 @@ func TestFeatureCompatibilityVersion(t *testing.T) { defer ctx.Teardown() mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "") - mdb.Spec.Version = "4.0.6" - mdb.Spec.FeatureCompatibilityVersion = "4.0" + mdb.Spec.Version = "4.2.23" + mdb.Spec.FeatureCompatibilityVersion = "4.2" _, err := setup.GeneratePasswordForUser(ctx, user, "") if err != nil { @@ -45,24 +45,24 @@ func TestFeatureCompatibilityVersion(t *testing.T) { t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx)) t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb)) t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3)) - t.Run("Test FeatureCompatibilityVersion is 4.0", tester.HasFCV("4.0", 3)) + t.Run("Test FeatureCompatibilityVersion is 4.2", tester.HasFCV("4.2", 3)) // Upgrade version to 4.2.6 while keeping the FCV set to 4.0 t.Run("MongoDB is reachable while version is upgraded", func(t *testing.T) { - defer tester.StartBackgroundConnectivityTest(t, time.Second*10)() - t.Run("Test Version can be upgraded", mongodbtests.ChangeVersion(&mdb, "4.2.6")) + defer tester.StartBackgroundConnectivityTest(t, time.Second*20)() + t.Run("Test Version can be upgraded", mongodbtests.ChangeVersion(&mdb, "4.4.11")) t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb, wait.Timeout(20*time.Minute))) }) t.Run("Test Basic Connectivity after upgrade has completed", tester.ConnectivitySucceeds()) - t.Run("Test FeatureCompatibilityVersion, after upgrade, is 4.0", tester.HasFCV("4.0", 3)) + t.Run("Test FeatureCompatibilityVersion, after upgrade, is 4.2", tester.HasFCV("4.2", 3)) // Downgrade version back to 4.0.6, checks that the FeatureCompatibilityVersion stayed at 4.0 t.Run("MongoDB is reachable while version is downgraded", func(t *testing.T) { defer tester.StartBackgroundConnectivityTest(t, time.Second*10)() - t.Run("Test Version can be downgraded", mongodbtests.ChangeVersion(&mdb, "4.0.6")) + t.Run("Test Version can be downgraded", mongodbtests.ChangeVersion(&mdb, "4.2.23")) t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb, wait.Timeout(20*time.Minute))) }) - t.Run("Test FeatureCompatibilityVersion, after downgrade, is 4.0", tester.HasFCV("4.0", 3)) + t.Run("Test FeatureCompatibilityVersion, after downgrade, is 4.2", tester.HasFCV("4.2", 3)) }