flakiness: fix flaky scram auth (#20)

# Summary

- pr migrated from:
10gen/ops-manager-kubernetes#4305
- will be cherry-picked over later

## Proof of Work

- test suite is green

Author: nammn

docker/mongodb-enterprise-tests/kubetester/kubetester.py

@@ -27,6 +27,7 @@
 from kubernetes.stream import stream
 from kubetester.crypto import wait_for_certs_to_be_issued
 from requests.auth import HTTPBasicAuth, HTTPDigestAuth
+from tests import test_logger
 
 SSL_CA_CERT = "/var/run/secrets/kubernetes.io/serviceaccount/..data/ca.crt"
 EXTERNALLY_MANAGED_TAG = "EXTERNALLY_MANAGED_BY_KUBERNETES"
@@ -46,6 +47,11 @@
     "MongoDBMultiCluster": "mongodbmulticluster",
 }
 
+from opentelemetry import trace
+
+TRACER = trace.get_tracer("evergreen-agent")
+logger = test_logger.get_test_logger(__name__)
+
 
 def running_locally():
     return os.getenv("POD_NAME", "local") == "local"
@@ -946,6 +952,16 @@ def get_automation_config(group_id=None, group_name=None):
 
         return response.json()
 
+    @staticmethod
+    def get_automation_status(group_id=None, group_name=None):
+        if group_id is None:
+            group_id = KubernetesTester.get_om_group_id(group_name=group_name)
+
+        url = build_automation_status_endpoint(KubernetesTester.get_om_base_url(), group_id)
+        response = KubernetesTester.om_request("get", url)
+
+        return response.json()
+
     @staticmethod
     def get_monitoring_config(group_id=None):
         if group_id is None:
@@ -1544,6 +1560,10 @@ def build_automation_config_endpoint(base_url, group_id):
     return "{}/api/public/v1.0/groups/{}/automationConfig".format(base_url, group_id)
 
 
+def build_automation_status_endpoint(base_url, group_id):
+    return "{}/api/public/v1.0/groups/{}/automationStatus".format(base_url, group_id)
+
+
 def build_monitoring_config_endpoint(base_url, group_id):
     return "{}/api/public/v1.0/groups/{}/automationConfig/monitoringAgentConfig".format(base_url, group_id)
 
@@ -1683,3 +1703,35 @@ def ensure_ent_version(mdb_version: str) -> str:
     if "-ent" not in mdb_version:
         return mdb_version + "-ent"
     return mdb_version
+
+
+@TRACER.start_as_current_span("wait_processes_ready")
+def wait_processes_ready():
+    # Get current automation status
+    def processes_are_ready():
+        auto_status = KubernetesTester.get_automation_status()
+        goal_version = auto_status.get("goalVersion")
+
+        logger.info(f"Checking if all processes have reached goal version: {goal_version}")
+        processes_not_ready = []
+        for process in auto_status.get("processes", []):
+            process_name = process.get("name", "unknown")
+            process_version = process.get("lastGoalVersionAchieved")
+            if process_version != goal_version:
+                logger.info(f"Process {process_name} at version {process_version}, expected {goal_version}")
+                processes_not_ready.append(process_name)
+
+        all_processes_ready = len(processes_not_ready) == 0
+        if all_processes_ready:
+            logger.info("All processes have reached the goal version")
+        else:
+            logger.info(f"{len(processes_not_ready)} processes have not yet reached the goal version")
+
+        return all_processes_ready
+
+    timeout = 600  # 5 minutes timeout
+    KubernetesTester.wait_until(
+        processes_are_ready,
+        timeout=timeout,
+        sleep_time=5,
+    )

docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_x509_to_scram_transition.py

@@ -1,4 +1,7 @@
+import time
+
 import pytest
+from kubetester import kubetester
 from kubetester.automation_config_tester import AutomationConfigTester
 from kubetester.certs import (
     ISSUER_CA_NAME,
@@ -12,11 +15,13 @@
 from kubetester.mongotester import ShardedClusterTester
 from kubetester.omtester import get_sc_cert_names
 from pytest import fixture
+from tests import test_logger
 
 MDB_RESOURCE = "sharded-cluster-x509-to-scram-256"
 USER_NAME = "mms-user-1"
 PASSWORD_SECRET_NAME = "mms-user-1-password"
 USER_PASSWORD = "my-password"
+logger = test_logger.get_test_logger(__name__)
 
 
 @fixture(scope="module")
@@ -76,6 +81,8 @@ def test_x509_is_still_configured():
 @pytest.mark.e2e_sharded_cluster_x509_to_scram_transition
 class TestShardedClusterDisableAuthentication(KubernetesTester):
     def test_disable_auth(self, sharded_cluster: MongoDB):
+        kubetester.wait_processes_ready()
+        sharded_cluster.assert_reaches_phase(Phase.Running, timeout=800)
         sharded_cluster.load()
         sharded_cluster["spec"]["security"]["authentication"]["enabled"] = False
         sharded_cluster.update()
@@ -90,17 +97,26 @@ def test_ops_manager_state_updated_correctly(self):
         tester.assert_authentication_disabled()
 
 
+from opentelemetry import trace
+
+TRACER = trace.get_tracer("evergreen-agent")
+
+
 @pytest.mark.e2e_sharded_cluster_x509_to_scram_transition
 class TestCanEnableScramSha256:
-    def test_can_enable_scram_sha_256(self, sharded_cluster: MongoDB):
+    @TRACER.start_as_current_span("test_can_enable_scram_sha_256")
+    def test_can_enable_scram_sha_256(self, sharded_cluster: MongoDB, ca_path: str):
+        kubetester.wait_processes_ready()
+        sharded_cluster.assert_reaches_phase(Phase.Running, timeout=800)
+
         sharded_cluster.load()
         sharded_cluster["spec"]["security"]["authentication"]["enabled"] = True
         sharded_cluster["spec"]["security"]["authentication"]["modes"] = [
             "SCRAM",
         ]
         sharded_cluster["spec"]["security"]["authentication"]["agents"]["mode"] = "SCRAM"
         sharded_cluster.update()
-        sharded_cluster.assert_reaches_phase(Phase.Running, timeout=1200)
+        sharded_cluster.assert_reaches_phase(Phase.Running, timeout=800)
 
     def test_assert_connectivity(self, ca_path: str):
         ShardedClusterTester(MDB_RESOURCE, 1, ssl=True, ca_path=ca_path).assert_connectivity(attempts=25)