Skip to content

Commit 4631a9c

Browse files
mircea-cosbucmircea-cosbuc
authored
CLOUDP-137882: Use different mount path for prometheus TLS secret (#1155)
1 parent f178780 commit 4631a9c

File tree

6 files changed

+121
-41
lines changed

6 files changed

+121
-41
lines changed

‎controllers/mongodb_tls.go

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,13 @@ import (
2222
)
2323

2424
const (
25-
tlsCAMountPath = "/var/lib/tls/ca/"
26-
tlsCACertName = "ca.crt"
27-
tlsOperatorSecretMountPath = "/var/lib/tls/server/" //nolint
28-
tlsSecretCertName = "tls.crt" //nolint
29-
tlsSecretKeyName = "tls.key"
30-
tlsSecretPemName = "tls.pem"
25+
tlsCAMountPath = "/var/lib/tls/ca/"
26+
tlsCACertName = "ca.crt"
27+
tlsOperatorSecretMountPath = "/var/lib/tls/server/" //nolint
28+
tlsPrometheusSecretMountPath = "/var/lib/tls/prometheus/" //nolint
29+
tlsSecretCertName = "tls.crt"
30+
tlsSecretKeyName = "tls.key"
31+
tlsSecretPemName = "tls.pem"
3132
)
3233

3334
// validateTLSConfig will check that the configured ConfigMap and Secret exist and that they have the correct fields.
@@ -316,8 +317,7 @@ func buildTLSPrometheus(mdb mdbv1.MongoDBCommunity) podtemplatespec.Modification
316317
// The same key-certificate pair is used for all servers
317318
tlsSecretVolume := statefulset.CreateVolumeFromSecret("prom-tls-secret", mdb.PrometheusTLSOperatorSecretNamespacedName().Name)
318319

319-
// TODO: Is it ok to use the same `tlsOperatorSecretMountPath`
320-
tlsSecretVolumeMount := statefulset.CreateVolumeMount(tlsSecretVolume.Name, tlsOperatorSecretMountPath, statefulset.WithReadOnly(true))
320+
tlsSecretVolumeMount := statefulset.CreateVolumeMount(tlsSecretVolume.Name, tlsPrometheusSecretMountPath, statefulset.WithReadOnly(true))
321321

322322
// MongoDB expects both key and certificate to be provided in a single PEM file
323323
// We are using a secret format where they are stored in separate fields, tls.crt and tls.key

‎controllers/mongodb_tls_test.go

Lines changed: 81 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,17 @@ func TestStatefulSet_IsCorrectlyConfiguredWithTLS(t *testing.T) {
3838
err = mgr.GetClient().Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts)
3939
assert.NoError(t, err)
4040

41-
assertStatefulsetVolumesAndVolumeMounts(t, sts, mdb.TLSOperatorCASecretNamespacedName().Name, mdb.TLSOperatorSecretNamespacedName().Name)
41+
assertStatefulsetVolumesAndVolumeMounts(t, sts, mdb.TLSOperatorCASecretNamespacedName().Name, mdb.TLSOperatorSecretNamespacedName().Name, "")
4242
}
4343

44-
func assertStatefulsetVolumesAndVolumeMounts(t *testing.T, sts appsv1.StatefulSet, expectedTLSCASecretName string, expectedTLSOperatorSecretName string) {
45-
assert.Len(t, sts.Spec.Template.Spec.Volumes, 8)
44+
func assertStatefulsetVolumesAndVolumeMounts(t *testing.T, sts appsv1.StatefulSet, expectedTLSCASecretName string, expectedTLSOperatorSecretName string, expectedPromTLSSecretName string) {
45+
prometheusTLSEnabled := expectedPromTLSSecretName != ""
46+
47+
if prometheusTLSEnabled {
48+
assert.Len(t, sts.Spec.Template.Spec.Volumes, 9)
49+
} else {
50+
assert.Len(t, sts.Spec.Template.Spec.Volumes, 8)
51+
}
4652
permission := int32(416)
4753
assert.Contains(t, sts.Spec.Template.Spec.Volumes, corev1.Volume{
4854
Name: "tls-ca",
@@ -62,6 +68,17 @@ func assertStatefulsetVolumesAndVolumeMounts(t *testing.T, sts appsv1.StatefulSe
6268
},
6369
},
6470
})
71+
if prometheusTLSEnabled {
72+
assert.Contains(t, sts.Spec.Template.Spec.Volumes, corev1.Volume{
73+
Name: "prom-tls-secret",
74+
VolumeSource: corev1.VolumeSource{
75+
Secret: &corev1.SecretVolumeSource{
76+
SecretName: expectedPromTLSSecretName,
77+
DefaultMode: &permission,
78+
},
79+
},
80+
})
81+
}
6582

6683
tlsSecretVolumeMount := corev1.VolumeMount{
6784
Name: "tls-secret",
@@ -73,16 +90,70 @@ func assertStatefulsetVolumesAndVolumeMounts(t *testing.T, sts appsv1.StatefulSe
7390
ReadOnly: true,
7491
MountPath: tlsCAMountPath,
7592
}
93+
tlsPrometheusSecretVolumeMount := corev1.VolumeMount{
94+
Name: "prom-tls-secret",
95+
ReadOnly: true,
96+
MountPath: tlsPrometheusSecretMountPath,
97+
}
7698

7799
assert.Len(t, sts.Spec.Template.Spec.InitContainers, 2)
78100

79101
agentContainer := sts.Spec.Template.Spec.Containers[0]
80102
assert.Contains(t, agentContainer.VolumeMounts, tlsSecretVolumeMount)
81103
assert.Contains(t, agentContainer.VolumeMounts, tlsCAVolumeMount)
104+
if prometheusTLSEnabled {
105+
assert.Contains(t, agentContainer.VolumeMounts, tlsPrometheusSecretVolumeMount)
106+
}
82107

83108
mongodbContainer := sts.Spec.Template.Spec.Containers[1]
84109
assert.Contains(t, mongodbContainer.VolumeMounts, tlsSecretVolumeMount)
85110
assert.Contains(t, mongodbContainer.VolumeMounts, tlsCAVolumeMount)
111+
if prometheusTLSEnabled {
112+
assert.Contains(t, mongodbContainer.VolumeMounts, tlsPrometheusSecretVolumeMount)
113+
}
114+
}
115+
116+
func TestStatefulSet_IsCorrectlyConfiguredWithPrometheusTLS(t *testing.T) {
117+
mdb := newTestReplicaSetWithTLS()
118+
mdb.Spec.Prometheus = &mdbv1.Prometheus{
119+
Username: "username",
120+
PasswordSecretRef: mdbv1.SecretKeyReference{
121+
Name: "prom-password-secret",
122+
},
123+
Port: 4321,
124+
TLSSecretRef: mdbv1.SecretKeyReference{
125+
Name: "prom-secret-cert",
126+
},
127+
}
128+
129+
mgr := kubeClient.NewManager(&mdb)
130+
cli := mdbClient.NewClient(mgr.GetClient())
131+
132+
err := secret.CreateOrUpdate(mgr.Client,
133+
secret.Builder().
134+
SetName("prom-password-secret").
135+
SetNamespace(mdb.Namespace).
136+
SetField("password", "my-password").
137+
Build(),
138+
)
139+
assert.NoError(t, err)
140+
err = createTLSSecret(cli, mdb, "CERT", "KEY", "")
141+
assert.NoError(t, err)
142+
err = createPrometheusTLSSecret(cli, mdb, "CERT", "KEY", "")
143+
assert.NoError(t, err)
144+
145+
err = createTLSConfigMap(cli, mdb)
146+
assert.NoError(t, err)
147+
148+
r := NewReconciler(mgr)
149+
res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}})
150+
assertReconciliationSuccessful(t, res, err)
151+
152+
sts := appsv1.StatefulSet{}
153+
err = mgr.GetClient().Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts)
154+
assert.NoError(t, err)
155+
156+
assertStatefulsetVolumesAndVolumeMounts(t, sts, mdb.TLSOperatorCASecretNamespacedName().Name, mdb.TLSOperatorSecretNamespacedName().Name, mdb.PrometheusTLSOperatorSecretNamespacedName().Name)
86157
}
87158

88159
func TestStatefulSet_IsCorrectlyConfiguredWithTLSAfterChangingExistingVolumes(t *testing.T) {
@@ -110,7 +181,7 @@ func TestStatefulSet_IsCorrectlyConfiguredWithTLSAfterChangingExistingVolumes(t
110181
err = mgr.GetClient().Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts)
111182
assert.NoError(t, err)
112183

113-
assertStatefulsetVolumesAndVolumeMounts(t, sts, tlsCAVolumeSecretName, mdb.TLSOperatorSecretNamespacedName().Name)
184+
assertStatefulsetVolumesAndVolumeMounts(t, sts, tlsCAVolumeSecretName, mdb.TLSOperatorSecretNamespacedName().Name, "")
114185

115186
// updating sts tls-ca volume directly to simulate changing of underlying volume's secret
116187
for i := range sts.Spec.Template.Spec.Volumes {
@@ -122,15 +193,15 @@ func TestStatefulSet_IsCorrectlyConfiguredWithTLSAfterChangingExistingVolumes(t
122193
err = mgr.GetClient().Update(context.TODO(), &sts)
123194
assert.NoError(t, err)
124195

125-
assertStatefulsetVolumesAndVolumeMounts(t, sts, changedTLSCAVolumeSecretName, mdb.TLSOperatorSecretNamespacedName().Name)
196+
assertStatefulsetVolumesAndVolumeMounts(t, sts, changedTLSCAVolumeSecretName, mdb.TLSOperatorSecretNamespacedName().Name, "")
126197

127198
res, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Namespace: mdb.Namespace, Name: mdb.Name}})
128199
assertReconciliationSuccessful(t, res, err)
129200

130201
sts = appsv1.StatefulSet{}
131202
err = mgr.GetClient().Get(context.TODO(), types.NamespacedName{Name: mdb.Name, Namespace: mdb.Namespace}, &sts)
132203
assert.NoError(t, err)
133-
assertStatefulsetVolumesAndVolumeMounts(t, sts, tlsCAVolumeSecretName, mdb.TLSOperatorSecretNamespacedName().Name)
204+
assertStatefulsetVolumesAndVolumeMounts(t, sts, tlsCAVolumeSecretName, mdb.TLSOperatorSecretNamespacedName().Name, "")
134205
}
135206

136207
func TestAutomationConfig_IsCorrectlyConfiguredWithTLS(t *testing.T) {
@@ -422,6 +493,10 @@ func createTLSSecret(c k8sClient.Client, mdb mdbv1.MongoDBCommunity, crt string,
422493
return createTLSSecretWithNamespaceAndName(c, mdb.Namespace, mdb.Spec.Security.TLS.CertificateKeySecret.Name, crt, key, pem)
423494
}
424495

496+
func createPrometheusTLSSecret(c k8sClient.Client, mdb mdbv1.MongoDBCommunity, crt string, key string, pem string) error {
497+
return createTLSSecretWithNamespaceAndName(c, mdb.Namespace, mdb.Spec.Prometheus.TLSSecretRef.Name, crt, key, pem)
498+
}
499+
425500
func createUserPasswordSecret(c k8sClient.Client, mdb mdbv1.MongoDBCommunity, userPasswordSecretName string, password string) error {
426501
sBuilder := secret.Builder().
427502
SetName(userPasswordSecretName).

‎controllers/prometheus.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ func getPrometheusModification(getUpdateCreator secret.GetUpdateCreator, mdb mdb
3939
if err != nil {
4040
return automationconfig.NOOP(), err
4141
}
42-
tlsPEMPath = tlsOperatorSecretMountPath + tlsOperatorSecretFileName(certKey)
42+
tlsPEMPath = tlsPrometheusSecretMountPath + tlsOperatorSecretFileName(certKey)
4343
scheme = "https"
4444
} else {
4545
scheme = "http"

‎release.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
"version-upgrade-hook": "1.0.6",
55
"readiness-probe": "1.0.12",
66
"mongodb-agent": {
7-
"version": "12.0.10.7591-1",
8-
"tools_version": "100.5.3"
7+
"version": "12.0.14.7630-1",
8+
"tools_version": "100.6.0"
99
}
1010
}

‎test/e2e/feature_compatibility_version/feature_compatibility_version_test.go

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@ func TestFeatureCompatibilityVersion(t *testing.T) {
2929
defer ctx.Teardown()
3030

3131
mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "")
32-
mdb.Spec.Version = "4.0.6"
33-
mdb.Spec.FeatureCompatibilityVersion = "4.0"
32+
mdb.Spec.Version = "4.2.23"
33+
mdb.Spec.FeatureCompatibilityVersion = "4.2"
3434

3535
_, err := setup.GeneratePasswordForUser(ctx, user, "")
3636
if err != nil {
@@ -45,24 +45,24 @@ func TestFeatureCompatibilityVersion(t *testing.T) {
4545
t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx))
4646
t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb))
4747
t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3))
48-
t.Run("Test FeatureCompatibilityVersion is 4.0", tester.HasFCV("4.0", 3))
48+
t.Run("Test FeatureCompatibilityVersion is 4.2", tester.HasFCV("4.2", 3))
4949

5050
// Upgrade version to 4.2.6 while keeping the FCV set to 4.0
5151
t.Run("MongoDB is reachable while version is upgraded", func(t *testing.T) {
52-
defer tester.StartBackgroundConnectivityTest(t, time.Second*10)()
53-
t.Run("Test Version can be upgraded", mongodbtests.ChangeVersion(&mdb, "4.2.6"))
52+
defer tester.StartBackgroundConnectivityTest(t, time.Second*20)()
53+
t.Run("Test Version can be upgraded", mongodbtests.ChangeVersion(&mdb, "4.4.11"))
5454
t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb, wait.Timeout(20*time.Minute)))
5555
})
5656

5757
t.Run("Test Basic Connectivity after upgrade has completed", tester.ConnectivitySucceeds())
58-
t.Run("Test FeatureCompatibilityVersion, after upgrade, is 4.0", tester.HasFCV("4.0", 3))
58+
t.Run("Test FeatureCompatibilityVersion, after upgrade, is 4.2", tester.HasFCV("4.2", 3))
5959

6060
// Downgrade version back to 4.0.6, checks that the FeatureCompatibilityVersion stayed at 4.0
6161
t.Run("MongoDB is reachable while version is downgraded", func(t *testing.T) {
6262
defer tester.StartBackgroundConnectivityTest(t, time.Second*10)()
63-
t.Run("Test Version can be downgraded", mongodbtests.ChangeVersion(&mdb, "4.0.6"))
63+
t.Run("Test Version can be downgraded", mongodbtests.ChangeVersion(&mdb, "4.2.23"))
6464
t.Run("Stateful Set Reaches Ready State, after Upgrading", mongodbtests.StatefulSetBecomesReady(&mdb, wait.Timeout(20*time.Minute)))
6565
})
6666

67-
t.Run("Test FeatureCompatibilityVersion, after downgrade, is 4.0", tester.HasFCV("4.0", 3))
67+
t.Run("Test FeatureCompatibilityVersion, after downgrade, is 4.2", tester.HasFCV("4.2", 3))
6868
}

‎test/e2e/prometheus/prometheus_test.go

Lines changed: 21 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,16 @@ func TestMain(m *testing.M) {
2323
}
2424

2525
func TestPrometheus(t *testing.T) {
26-
ctx, testConfig := setup.SetupWithTLS(t, "")
26+
resourceName := "mdb0"
27+
ctx, testConfig := setup.SetupWithTLS(t, resourceName)
2728
defer ctx.Teardown()
2829

29-
mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", testConfig.Namespace)
30+
mdb, user := e2eutil.NewTestMongoDB(ctx, resourceName, testConfig.Namespace)
31+
32+
mdb.Spec.Security.TLS = e2eutil.NewTestTLSConfig(false)
3033
mdb.Spec.Prometheus = e2eutil.NewPrometheusConfig(mdb.Namespace)
3134

32-
_, err := setup.GeneratePasswordForUser(ctx, user, "")
35+
_, err := setup.GeneratePasswordForUser(ctx, user, testConfig.Namespace)
3336
if err != nil {
3437
t.Fatal(err)
3538
}
@@ -41,21 +44,23 @@ func TestPrometheus(t *testing.T) {
4144

4245
t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx))
4346
t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb))
44-
t.Run("Keyfile authentication is configured", tester.HasKeyfileAuth(3))
45-
t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds())
4647

47-
t.Run("Test Prometheus endpoint is active", tester.PrometheusEndpointIsReachable("prom-user", "prom-password", false))
48-
t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3))
49-
t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1))
48+
mongodbtests.SkipTestIfLocal(t, "Ensure MongoDB with Prometheus configuration", func(t *testing.T) {
49+
t.Run("Resource has TLS Mode", tester.HasTlsMode("requireSSL", 60, WithTls(mdb)))
50+
t.Run("Test Basic Connectivity", tester.ConnectivitySucceeds(WithTls(mdb)))
51+
t.Run("Test Prometheus endpoint is active", tester.PrometheusEndpointIsReachable("prom-user", "prom-password", false))
52+
t.Run("Ensure Authentication", tester.EnsureAuthenticationIsConfigured(3, WithTls(mdb)))
53+
t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 1))
5054

51-
t.Run("Enabling HTTPS on the Prometheus endpoint", func(t *testing.T) {
52-
err = e2eutil.UpdateMongoDBResource(&mdb, func(mdb *v1.MongoDBCommunity) {
53-
mdb.Spec.Prometheus.TLSSecretRef.Name = "tls-certificate"
54-
})
55-
assert.NoError(t, err)
55+
t.Run("Enabling HTTPS on the Prometheus endpoint", func(t *testing.T) {
56+
err = e2eutil.UpdateMongoDBResource(&mdb, func(mdb *v1.MongoDBCommunity) {
57+
mdb.Spec.Prometheus.TLSSecretRef.Name = "tls-certificate"
58+
})
59+
assert.NoError(t, err)
5660

57-
t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb))
58-
t.Run("Test Prometheus HTTPS endpoint is active", tester.PrometheusEndpointIsReachable("prom-user", "prom-password", true))
61+
t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb))
62+
t.Run("Test Prometheus HTTPS endpoint is active", tester.PrometheusEndpointIsReachable("prom-user", "prom-password", true))
63+
t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 2))
64+
})
5965
})
60-
t.Run("AutomationConfig has the correct version", mongodbtests.AutomationConfigVersionHasTheExpectedVersion(&mdb, 2))
6166
}

0 commit comments

Comments
 (0)