Running search test under OM 8.2

.evergreen.yml

@@ -1131,6 +1131,15 @@ task_groups:
       - e2e_om_ops_manager_prometheus
     <<: *teardown_group
 
+  # Tests features only supported on OM80
+  - name: e2e_ops_manager_kind_8_0_only_task_group
+    max_hosts: -1
+    <<: *setup_group
+    <<: *setup_and_teardown_task
+    tasks:
+      - e2e_search_enterprise_tls
+    <<: *teardown_group
+
   # Tests features only supported on OM70 and OM80, its only upgrade test as we test upgrading from 6 to 7 or 7 to 8
   - name: e2e_ops_manager_upgrade_only_task_group
     max_hosts: -1
@@ -1341,6 +1350,7 @@ buildvariants:
       - name: e2e_ops_manager_kind_5_0_only_task_group_without_queryable_backup
       - name: e2e_ops_manager_kind_6_0_only_task_group
       - name: e2e_ops_manager_upgrade_only_task_group
+      - name: e2e_ops_manager_kind_8_0_only_task_group
 
   - name: e2e_static_om80_kind_ubi
     display_name: e2e_static_om80_kind_ubi

docker/mongodb-kubernetes-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_disaster_recovery.py

@@ -1,4 +1,3 @@
-import os
 import time
 from typing import Optional
 

docker/mongodb-kubernetes-tests/tests/opsmanager/fixtures/om_ops_manager_basic.yaml

@@ -15,3 +15,16 @@ spec:
 
   backup:
     enabled: false
+
+  # adding this just to avoid wizard when opening OM UI
+  configuration:
+    automation.versions.source: mongodb
+    mms.adminEmailAddr: [email protected]
+    mms.fromEmailAddr: [email protected]
+    mms.ignoreInitialUiSetup: "true"
+    mms.mail.hostname: email-smtp.us-east-1.amazonaws.com
+    mms.mail.port: "465"
+    mms.mail.ssl: "true"
+    mms.mail.transport: smtp
+    mms.minimumTLSVersion: TLSv1.2
+    mms.replyToEmailAddr: [email protected]

docker/mongodb-kubernetes-tests/tests/search/fixtures/enterprise-replicaset-sample-mflix.yaml

@@ -19,16 +19,15 @@ spec:
       - SCRAM
   agent:
     logLevel: DEBUG
-  statefulSet:
-    spec:
-      template:
-        spec:
-          containers:
-          - name: mongodb-enterprise-database
-            resources:
-              limits:
-                cpu: "2"
-                memory: 2Gi
-              requests:
-                cpu: "1"
-                memory: 1Gi
+  podSpec:
+    podTemplate:
+      spec:
+        containers:
+        - name: mongodb-enterprise-database
+          resources:
+            limits:
+              cpu: "2"
+              memory: 2Gi
+            requests:
+              cpu: "1"
+              memory: 1Gi

docker/mongodb-kubernetes-tests/tests/search/om_deployment.py

@@ -0,0 +1,31 @@
+from typing import Optional
+
+from kubetester import try_load
+from kubetester.kubetester import fixture as yaml_fixture
+from kubetester.kubetester import is_multi_cluster
+from kubetester.opsmanager import MongoDBOpsManager
+from pytest import fixture
+from tests.common.ops_manager.cloud_manager import is_cloud_qa
+from tests.conftest import get_custom_appdb_version, get_custom_om_version
+from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment
+
+
+def get_ops_manager(namespace: str) -> Optional[MongoDBOpsManager]:
+    if is_cloud_qa():
+        return None
+
+    resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml(
+        yaml_fixture("om_ops_manager_basic.yaml"), namespace=namespace
+    )
+
+    if try_load(resource):
+        return resource
+
+    resource.set_version(get_custom_om_version())
+    resource.set_appdb_version(get_custom_appdb_version())
+    resource.allow_mdb_rc_versions()
+
+    if is_multi_cluster():
+        enable_multi_cluster_deployment(resource)
+
+    return resource

docker/mongodb-kubernetes-tests/tests/search/search_enterprise_tls.py

@@ -1,19 +1,22 @@
 import pymongo
 import yaml
-from kubetester import create_or_update_secret, try_load
+from kubetester import create_or_update_secret, run_periodically, try_load
 from kubetester.certs import create_mongodb_tls_certs, create_tls_certs
 from kubetester.kubetester import KubernetesTester
 from kubetester.kubetester import fixture as yaml_fixture
 from kubetester.mongodb import MongoDB
 from kubetester.mongodb_search import MongoDBSearch
 from kubetester.mongodb_user import MongoDBUser
+from kubetester.omtester import skip_if_cloud_manager
 from kubetester.phase import Phase
+from mypyc.irbuild.function import check_native_override
 from pytest import fixture, mark
 from tests import test_logger
 from tests.common.search import movies_search_helper
-from tests.common.search.movies_search_helper import SampleMoviesSearchHelper
 from tests.common.search.search_tester import SearchTester
-from tests.conftest import get_default_operator
+from tests.conftest import get_default_operator, get_issuer_ca_filepath
+from tests.opsmanager.conftest import custom_om_prev_version
+from tests.search.om_deployment import get_ops_manager
 
 logger = test_logger.get_test_logger(__name__)
 
@@ -26,11 +29,14 @@
 USER_NAME = "mdb-user"
 USER_PASSWORD = f"{USER_NAME}-password"
 
-MDB_RESOURCE_NAME = "mdb-rs"
+MDB_RESOURCE_NAME = "mdb-ent-tls"
 
 # MongoDBSearch TLS configuration
 MDBS_TLS_SECRET_NAME = "mdbs-tls-secret"
 
+MDB_VERSION_WITHOUT_BUILT_IN_ROLE = "8.0.10-ent"
+MDB_VERSION_WITH_BUILT_IN_ROLE = "8.2.0-ent"
+
 
 @fixture(scope="function")
 def mdb(namespace: str, issuer_ca_configmap: str) -> MongoDB:
@@ -39,6 +45,8 @@ def mdb(namespace: str, issuer_ca_configmap: str) -> MongoDB:
         name=MDB_RESOURCE_NAME,
         namespace=namespace,
     )
+    resource.configure(om=get_ops_manager(namespace), project_name=MDB_RESOURCE_NAME)
+    resource.set_version(MDB_VERSION_WITHOUT_BUILT_IN_ROLE)
 
     if try_load(resource):
         return resource
@@ -73,6 +81,7 @@ def admin_user(namespace: str) -> MongoDBUser:
     if try_load(resource):
         return resource
 
+    resource["spec"]["mongodbResourceRef"]["name"] = MDB_RESOURCE_NAME
     resource["spec"]["username"] = resource.name
     resource["spec"]["passwordSecretKeyRef"]["name"] = f"{resource.name}-password"
 
@@ -86,6 +95,7 @@ def user(namespace: str) -> MongoDBUser:
     if try_load(resource):
         return resource
 
+    resource["spec"]["mongodbResourceRef"]["name"] = MDB_RESOURCE_NAME
     resource["spec"]["username"] = resource.name
     resource["spec"]["passwordSecretKeyRef"]["name"] = f"{resource.name}-password"
 
@@ -103,6 +113,7 @@ def mongot_user(namespace: str, mdbs: MongoDBSearch) -> MongoDBUser:
     if try_load(resource):
         return resource
 
+    resource["spec"]["mongodbResourceRef"]["name"] = MDB_RESOURCE_NAME
     resource["spec"]["username"] = MONGOT_USER_NAME
     resource["spec"]["passwordSecretKeyRef"]["name"] = f"{resource.name}-password"
 
@@ -115,6 +126,16 @@ def test_install_operator(namespace: str, operator_installation_config: dict[str
     operator.assert_is_running()
 
 
+@mark.e2e_search_enterprise_tls
+@skip_if_cloud_manager
+def test_create_ops_manager(namespace: str):
+    ops_manager = get_ops_manager(namespace)
+    if ops_manager is not None:
+        ops_manager.update()
+    ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=1200)
+    ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600)
+
+
 @mark.e2e_search_enterprise_tls
 def test_install_tls_secrets_and_configmaps(namespace: str, mdb: MongoDB, mdbs: MongoDBSearch, issuer: str):
     create_mongodb_tls_certs(issuer, namespace, mdb.name, f"certs-{mdb.name}-cert", mdb.get_members())
@@ -144,19 +165,19 @@ def test_create_users(
     create_or_update_secret(
         namespace, name=admin_user["spec"]["passwordSecretKeyRef"]["name"], data={"password": ADMIN_USER_PASSWORD}
     )
-    admin_user.create()
+    admin_user.update()
     admin_user.assert_reaches_phase(Phase.Updated, timeout=300)
 
     create_or_update_secret(
         namespace, name=user["spec"]["passwordSecretKeyRef"]["name"], data={"password": USER_PASSWORD}
     )
-    user.create()
+    user.update()
     user.assert_reaches_phase(Phase.Updated, timeout=300)
 
     create_or_update_secret(
         namespace, name=mongot_user["spec"]["passwordSecretKeyRef"]["name"], data={"password": MONGOT_USER_PASSWORD}
     )
-    mongot_user.create()
+    mongot_user.update()
     # we deliberately don't wait for this user to be ready, because to be reconciled successfully it needs the searchCoordinator role
     # which the ReplicaSet reconciler will only define in the automation config after the MongoDBSearch resource is created.
 
@@ -169,27 +190,117 @@ def test_create_search_resource(mdbs: MongoDBSearch):
 
 @mark.e2e_search_enterprise_tls
 def test_wait_for_database_resource_ready(mdb: MongoDB):
-    mdb.assert_abandons_phase(Phase.Running, timeout=300)
     mdb.assert_reaches_phase(Phase.Running, timeout=300)
 
-    for idx in range(mdb.get_members()):
-        mongod_config = yaml.safe_load(
-            KubernetesTester.run_command_in_pod_container(
-                f"{mdb.name}-{idx}", mdb.namespace, ["cat", "/data/automation-mongod.conf"]
+
+@mark.e2e_search_enterprise_tls
+def test_wait_for_mongod_parameters(mdb: MongoDB):
+    def check_mongod_parameters():
+        parameters_are_set = True
+        pod_parameters = []
+        for idx in range(mdb.get_members()):
+            mongod_config = yaml.safe_load(
+                KubernetesTester.run_command_in_pod_container(
+                    f"{mdb.name}-{idx}", mdb.namespace, ["cat", "/data/automation-mongod.conf"]
+                )
             )
-        )
-        setParameter = mongod_config.get("setParameter", {})
-        assert (
-            "mongotHost" in setParameter and "searchIndexManagementHostAndPort" in setParameter
-        ), "mongot parameters not found in mongod config"
+            set_parameter = mongod_config.get("setParameter", {})
+            parameters_are_set = parameters_are_set and (
+                "mongotHost" in set_parameter and "searchIndexManagementHostAndPort" in set_parameter
+            )
+            pod_parameters.append(f"pod {idx} setParameter: {set_parameter}")
+
+        return parameters_are_set, f'Not all pods have mongot parameters set:\n{"\n".join(pod_parameters)}'
+
+    run_periodically(lambda: check_mongod_parameters(), timeout=200)
+
+
+@mark.e2e_search_enterprise_tls
+def test_wait_for_database_resource_ready2(mdb: MongoDB):
+    mdb.assert_reaches_phase(Phase.Running, timeout=300)
+
+
+@mark.e2e_search_enterprise_tls
+def test_validate_tls_connections(mdb: MongoDB, mdbs: MongoDBSearch, namespace: str):
+    validate_tls_connections(mdb, mdbs, namespace)
+
+
+@mark.e2e_search_enterprise_tls
+def test_search_restore_sample_database(mdb: MongoDB):
+    get_admin_sample_movies_helper(mdb).restore_sample_database()
+
+
+@mark.e2e_search_enterprise_tls
+def test_search_create_search_index(mdb: MongoDB):
+    get_user_sample_movies_helper(mdb).create_search_index()
+
+
+@mark.e2e_search_enterprise_tls
+def test_search_assert_search_query(mdb: MongoDB):
+    get_user_sample_movies_helper(mdb).assert_search_query(retry_timeout=60)
 
 
 @mark.e2e_search_enterprise_tls
-def test_validate_tls_connections(mdb: MongoDB, mdbs: MongoDBSearch, namespace: str, issuer_ca_filepath: str):
+class TestUpgradeMongod:
+    def test_check_polyfilled_role_in_ac(self, mdb: MongoDB):
+        custom_roles = mdb.get_automation_config_tester().automation_config.get("roles", [])
+        assert len(custom_roles) > 0
+        assert "searchCoordinator" in [role["role"] for role in custom_roles]
+
+    def test_mongod_version(self, mdb: MongoDB):
+        mdb.tester(ca_path=get_issuer_ca_filepath(), use_ssl=True).assert_version(MDB_VERSION_WITHOUT_BUILT_IN_ROLE)
+
+    def test_upgrade_to_mongo_8_2(self, mdb: MongoDB):
+        mdb.set_version(MDB_VERSION_WITH_BUILT_IN_ROLE)
+        mdb.update()
+        mdb.assert_reaches_phase(Phase.Running, timeout=600)
+
+    def test_check_polyfilled_role_not_in_ac(self, mdb: MongoDB):
+        custom_roles = mdb.get_automation_config_tester().automation_config.get("roles", [])
+        assert len(custom_roles) >= 0
+        assert "searchCoordinator" not in [role["role"] for role in custom_roles]
+
+    def test_mongod_version_after_upgrade(self, mdb: MongoDB):
+        mdb_tester = mdb.tester(ca_path=get_issuer_ca_filepath(), use_ssl=True)
+        mdb_tester.assert_scram_sha_authentication(
+            ADMIN_USER_NAME, ADMIN_USER_PASSWORD, "SCRAM-SHA-256", 1, ssl=True, tlsCAFile=get_issuer_ca_filepath()
+        )
+        # TODO check why assert version works without auth for 8.0 and not for 8.2
+        mdb_tester.assert_version(MDB_VERSION_WITH_BUILT_IN_ROLE)
+
+
+@mark.e2e_search_enterprise_tlssh
+def test_search_assert_search_query_2(mdb: MongoDB):
+    get_user_sample_movies_helper(mdb).assert_search_query(retry_timeout=60)
+
+
+def get_connection_string(mdb: MongoDB, user_name: str, user_password: str) -> str:
+    return f"mongodb://{user_name}:{user_password}@{mdb.name}-0.{mdb.name}-svc.{mdb.namespace}.svc.cluster.local:27017/?replicaSet={mdb.name}"
+
+
+def get_admin_sample_movies_helper(mdb):
+    return movies_search_helper.SampleMoviesSearchHelper(
+        SearchTester(
+            get_connection_string(mdb, ADMIN_USER_NAME, ADMIN_USER_PASSWORD),
+            use_ssl=True,
+            ca_path=get_issuer_ca_filepath(),
+        )
+    )
+
+
+def get_user_sample_movies_helper(mdb):
+    return movies_search_helper.SampleMoviesSearchHelper(
+        SearchTester(
+            get_connection_string(mdb, USER_NAME, USER_PASSWORD), use_ssl=True, ca_path=get_issuer_ca_filepath()
+        )
+    )
+
+
+def validate_tls_connections(mdb: MongoDB, mdbs: MongoDBSearch, namespace: str):
     with pymongo.MongoClient(
         f"mongodb://{mdb.name}-0.{mdb.name}-svc.{namespace}.svc.cluster.local:27017/?replicaSet={mdb.name}",
         tls=True,
-        tlsCAFile=issuer_ca_filepath,
+        tlsCAFile=get_issuer_ca_filepath(),
         tlsAllowInvalidHostnames=False,
         serverSelectionTimeoutMS=30000,
         connectTimeoutMS=20000,
@@ -200,40 +311,10 @@ def test_validate_tls_connections(mdb: MongoDB, mdbs: MongoDBSearch, namespace:
     with pymongo.MongoClient(
         f"mongodb://{mdbs.name}-search-svc.{namespace}.svc.cluster.local:27027",
         tls=True,
-        tlsCAFile=issuer_ca_filepath,
+        tlsCAFile=get_issuer_ca_filepath(),
         tlsAllowInvalidHostnames=False,
         serverSelectionTimeoutMS=10000,
         connectTimeoutMS=10000,
     ) as search_client:
         search_info = search_client.admin.command("hello")
         assert search_info.get("ok") == 1, "MongoDBSearch connection failed"
-
-
-@mark.e2e_search_enterprise_tls
-def test_search_restore_sample_database(mdb: MongoDB, issuer_ca_filepath: str):
-    sample_movies_helper = movies_search_helper.SampleMoviesSearchHelper(
-        SearchTester(
-            get_connection_string(mdb, ADMIN_USER_NAME, ADMIN_USER_PASSWORD), use_ssl=True, ca_path=issuer_ca_filepath
-        )
-    )
-    sample_movies_helper.restore_sample_database()
-
-
-@mark.e2e_search_enterprise_tls
-def test_search_create_search_index(mdb: MongoDB, issuer_ca_filepath: str):
-    sample_movies_helper = movies_search_helper.SampleMoviesSearchHelper(
-        SearchTester(get_connection_string(mdb, USER_NAME, USER_PASSWORD), use_ssl=True, ca_path=issuer_ca_filepath)
-    )
-    sample_movies_helper.create_search_index()
-
-
-@mark.e2e_search_enterprise_tls
-def test_search_assert_search_query(mdb: MongoDB, issuer_ca_filepath: str):
-    sample_movies_helper = movies_search_helper.SampleMoviesSearchHelper(
-        SearchTester(get_connection_string(mdb, USER_NAME, USER_PASSWORD), use_ssl=True, ca_path=issuer_ca_filepath)
-    )
-    sample_movies_helper.assert_search_query(retry_timeout=60)
-
-
-def get_connection_string(mdb: MongoDB, user_name: str, user_password: str) -> str:
-    return f"mongodb://{user_name}:{user_password}@{mdb.name}-0.{mdb.name}-svc.{mdb.namespace}.svc.cluster.local:27017/?replicaSet={mdb.name}"