diff --git a/docker/mongodb-enterprise-tests/kubetester/certs.py b/docker/mongodb-enterprise-tests/kubetester/certs.py index 6328b56da..d6c34a4d2 100644 --- a/docker/mongodb-enterprise-tests/kubetester/certs.py +++ b/docker/mongodb-enterprise-tests/kubetester/certs.py @@ -13,15 +13,27 @@ from kubeobject import CustomObject from kubernetes import client from kubernetes.client.rest import ApiException -from kubetester import create_secret, delete_secret, random_k8s_name, read_secret +from kubetester import ( + create_secret, + delete_secret, + kubetester, + random_k8s_name, + read_secret, +) from kubetester.kubetester import KubernetesTester +from kubetester.mongodb import Phase from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from opentelemetry import trace +from tests import test_logger from tests.vaultintegration import ( store_secret_in_vault, vault_namespace_name, vault_sts_name, ) +TRACER = trace.get_tracer("evergreen-agent") +logger = test_logger.get_test_logger(__name__) + ISSUER_CA_NAME = "ca-issuer" SUBJECT = { @@ -161,11 +173,13 @@ def generate_cert( return secret_name -def rotate_cert(namespace, certificate_name): +def rotate_cert(namespace, certificate_name, should_block_until_ready=False): cert = Certificate(name=certificate_name, namespace=namespace) cert.load() cert["spec"]["dnsNames"].append("foo") # Append DNS to cert to rotate the certificate cert.update() + if should_block_until_ready: + cert.block_until_ready() def create_tls_certs( @@ -879,3 +893,36 @@ def yield_existing_csrs(csr_names: List[str], timeout: int = 300) -> Generator[s raise AssertionError( f"Expected to find {total_csrs} csrs, but only found {seen_csrs} after {timeout} seconds. Expected csrs {csr_names}" ) + + +@TRACER.start_as_current_span("assert_certificate_rotation") +def rotate_and_assert_certificates(mdb, namespace, certificate_name): + """ + Verifies certificate rotation completes successfully. + + Rotates the specified certificate and validates that: + 1. Automation config version increases, as cert changes causes a new ac version + 2. All MongoDB processes reach the new goal version + 3. MongoDB instance returns/stays to Running state + + """ + + old_ac_version = KubernetesTester.get_automation_config()["version"] + rotate_cert(namespace, certificate_name, should_block_until_ready=True) + + # Create named function to check version and process status + def check_version_increased(): + + current_version = KubernetesTester.get_automation_config()["version"] + version_increased = current_version > old_ac_version + + return version_increased + + timeout = 600 + KubernetesTester.wait_until( + check_version_increased, + timeout=timeout, + ) + kubetester.wait_processes_ready() + + mdb.assert_reaches_phase(Phase.Running, timeout=1200) diff --git a/docker/mongodb-enterprise-tests/kubetester/kubetester.py b/docker/mongodb-enterprise-tests/kubetester/kubetester.py index 27c161cec..8614eae0e 100644 --- a/docker/mongodb-enterprise-tests/kubetester/kubetester.py +++ b/docker/mongodb-enterprise-tests/kubetester/kubetester.py @@ -26,9 +26,13 @@ from kubernetes.client.rest import ApiException from kubernetes.stream import stream from kubetester.crypto import wait_for_certs_to_be_issued +from opentelemetry import trace from requests.auth import HTTPBasicAuth, HTTPDigestAuth from tests import test_logger +TRACER = trace.get_tracer("evergreen-agent") +logger = test_logger.get_test_logger(__name__) + SSL_CA_CERT = "/var/run/secrets/kubernetes.io/serviceaccount/..data/ca.crt" EXTERNALLY_MANAGED_TAG = "EXTERNALLY_MANAGED_BY_KUBERNETES" MAX_TAG_LEN = 32 @@ -188,12 +192,17 @@ def decode_secret(cls, data: Dict[str, str]) -> Dict[str, str]: return {k: b64decode(v).decode("utf-8") for (k, v) in data.items()} @classmethod - def read_configmap(cls, namespace: str, name: str, api_client: Optional[client.ApiClient] = None) -> Dict[str, str]: + def read_configmap( + cls, namespace: str, name: str, api_client: Optional[client.ApiClient] = None, with_metadata=False + ) -> Dict[str, str]: corev1 = cls.clients("corev1") if api_client is not None: corev1 = client.CoreV1Api(api_client=api_client) - return corev1.read_namespaced_config_map(name, namespace).data + cm = corev1.read_namespaced_config_map(name, namespace) + if with_metadata: + return cm + return cm.data @classmethod def read_pod(cls, namespace: str, name: str) -> Dict[str, str]: @@ -962,6 +971,16 @@ def get_automation_status(group_id=None, group_name=None): return response.json() + @staticmethod + def get_automation_status(group_id=None, group_name=None): + if group_id is None: + group_id = KubernetesTester.get_om_group_id(group_name=group_name) + + url = build_automation_status_endpoint(KubernetesTester.get_om_base_url(), group_id) + response = KubernetesTester.om_request("get", url) + + return response.json() + @staticmethod def get_monitoring_config(group_id=None): if group_id is None: diff --git a/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_x509_to_scram_transition.py b/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_x509_to_scram_transition.py index 822c61048..5d3c2a2df 100644 --- a/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_x509_to_scram_transition.py +++ b/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_x509_to_scram_transition.py @@ -14,9 +14,12 @@ from kubetester.mongodb import MongoDB, Phase from kubetester.mongotester import ShardedClusterTester from kubetester.omtester import get_sc_cert_names +from opentelemetry import trace from pytest import fixture from tests import test_logger +TRACER = trace.get_tracer("evergreen-agent") + MDB_RESOURCE = "sharded-cluster-x509-to-scram-256" USER_NAME = "mms-user-1" PASSWORD_SECRET_NAME = "mms-user-1-password" @@ -97,11 +100,6 @@ def test_ops_manager_state_updated_correctly(self): tester.assert_authentication_disabled() -from opentelemetry import trace - -TRACER = trace.get_tracer("evergreen-agent") - - @pytest.mark.e2e_sharded_cluster_x509_to_scram_transition class TestCanEnableScramSha256: @TRACER.start_as_current_span("test_can_enable_scram_sha_256") diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_x509_configure_all_options_rs.py b/docker/mongodb-enterprise-tests/tests/tls/tls_x509_configure_all_options_rs.py index 3151d294d..f2cdc7b0e 100644 --- a/docker/mongodb-enterprise-tests/tests/tls/tls_x509_configure_all_options_rs.py +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_x509_configure_all_options_rs.py @@ -4,7 +4,7 @@ ISSUER_CA_NAME, create_x509_agent_tls_certs, create_x509_mongodb_tls_certs, - rotate_cert, + rotate_and_assert_certificates, ) from kubetester.kubetester import KubernetesTester from kubetester.kubetester import fixture as load_fixture @@ -47,20 +47,10 @@ def test_ops_manager_state_correctly_updated(self): ac_tester.assert_internal_cluster_authentication_enabled() ac_tester.assert_authentication_enabled() - def test_rotate_certificate(self, mdb: MongoDB, namespace: str): - rotate_cert(namespace, "{}-cert".format(MDB_RESOURCE)) - mdb.assert_abandons_phase(Phase.Running, timeout=900) - mdb.assert_reaches_phase(Phase.Running, timeout=900) - def test_rotate_certificate_with_sts_restarting(self, mdb: MongoDB, namespace: str): mdb.trigger_sts_restart() - assert_certificate_rotation(mdb, namespace, "{}-cert".format(MDB_RESOURCE)) + rotate_and_assert_certificates(mdb, namespace, "{}-cert".format(MDB_RESOURCE)) def test_rotate_clusterfile_with_sts_restarting(self, mdb: MongoDB, namespace: str): mdb.trigger_sts_restart() - assert_certificate_rotation(mdb, namespace, "{}-clusterfile".format(MDB_RESOURCE)) - - -def assert_certificate_rotation(mdb, namespace, certificate_name): - rotate_cert(namespace, certificate_name) - mdb.assert_reaches_phase(Phase.Running, timeout=900) + rotate_and_assert_certificates(mdb, namespace, "{}-clusterfile".format(MDB_RESOURCE)) diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_x509_configure_all_options_sc.py b/docker/mongodb-enterprise-tests/tests/tls/tls_x509_configure_all_options_sc.py index 93e107410..3b5c54d4b 100644 --- a/docker/mongodb-enterprise-tests/tests/tls/tls_x509_configure_all_options_sc.py +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_x509_configure_all_options_sc.py @@ -4,16 +4,19 @@ from kubetester.certs import ( create_sharded_cluster_certs, create_x509_agent_tls_certs, - rotate_cert, + rotate_and_assert_certificates, ) -from kubetester.kubetester import KubernetesTester, ensure_ent_version, is_multi_cluster +from kubetester.kubetester import KubernetesTester, is_multi_cluster from kubetester.mongodb import MongoDB, Phase from kubetester.operator import Operator from pytest import fixture +from tests import test_logger from tests.shardedcluster.conftest import enable_multi_cluster_deployment MDB_RESOURCE_NAME = "test-x509-all-options-sc" +logger = test_logger.get_test_logger(__name__) + @fixture(scope="module") def agent_certs(issuer: str, namespace: str) -> str: @@ -86,58 +89,14 @@ def test_ops_manager_state_correctly_updated(self): ac_tester.assert_authentication_enabled() ac_tester.assert_expected_users(0) - def test_rotate_shard_cert(self, sc: MongoDB, namespace: str): - rotate_cert(namespace, f"{MDB_RESOURCE_NAME}-0-cert") - sc.assert_abandons_phase(Phase.Running, timeout=900) - sc.assert_reaches_phase(Phase.Running, timeout=1200) - - def test_rotate_config_cert(self, sc: MongoDB, namespace: str): - rotate_cert(namespace, f"{MDB_RESOURCE_NAME}-config-cert") - sc.assert_abandons_phase(Phase.Running, timeout=900) - sc.assert_reaches_phase(Phase.Running, timeout=1200) - - def test_rotate_mongos_cert(self, sc: MongoDB, namespace: str): - rotate_cert(namespace, f"{MDB_RESOURCE_NAME}-mongos-cert") - sc.assert_abandons_phase(Phase.Running, timeout=900) - sc.assert_reaches_phase(Phase.Running, timeout=1200) - def test_rotate_shard_cert_with_sts_restarting(self, sc: MongoDB, namespace: str): sc.trigger_sts_restart("shard") - assert_certificate_rotation(sc, namespace, f"{MDB_RESOURCE_NAME}-0-cert") + rotate_and_assert_certificates(sc, namespace, f"{MDB_RESOURCE_NAME}-0-cert") def test_rotate_config_cert_with_sts_restarting(self, sc: MongoDB, namespace: str): sc.trigger_sts_restart("config") - assert_certificate_rotation(sc, namespace, f"{MDB_RESOURCE_NAME}-config-cert") + rotate_and_assert_certificates(sc, namespace, f"{MDB_RESOURCE_NAME}-config-cert") def test_rotate_mongos_cert_with_sts_restarting(self, sc: MongoDB, namespace: str): sc.trigger_sts_restart("mongos") - assert_certificate_rotation(sc, namespace, f"{MDB_RESOURCE_NAME}-mongos-cert") - - def test_rotate_shard_certfile_with_sts_restarting(self, sc: MongoDB, namespace: str): - sc.trigger_sts_restart("shard") - assert_certificate_rotation( - sc, - namespace, - f"{MDB_RESOURCE_NAME}-0-clusterfile", - ) - - def test_rotate_config_certfile_with_sts_restarting(self, sc: MongoDB, namespace: str): - sc.trigger_sts_restart("config") - assert_certificate_rotation( - sc, - namespace, - f"{MDB_RESOURCE_NAME}-config-clusterfile", - ) - - def test_rotate_mongos_certfile_with_sts_restarting(self, sc: MongoDB, namespace: str): - sc.trigger_sts_restart("mongos") - assert_certificate_rotation( - sc, - namespace, - f"{MDB_RESOURCE_NAME}-mongos-clusterfile", - ) - - -def assert_certificate_rotation(sharded_cluster, namespace, certificate_name): - rotate_cert(namespace, certificate_name) - sharded_cluster.assert_reaches_phase(Phase.Running, timeout=1200) + rotate_and_assert_certificates(sc, namespace, f"{MDB_RESOURCE_NAME}-mongos-cert")