diff --git a/api/v1/search/mongodbsearch_types.go b/api/v1/search/mongodbsearch_types.go index 43e9be1be..cdb391781 100644 --- a/api/v1/search/mongodbsearch_types.go +++ b/api/v1/search/mongodbsearch_types.go @@ -1,6 +1,8 @@ package search import ( + "fmt" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -14,8 +16,9 @@ import ( ) const ( - MongotDefaultPort = 27027 - MongotDefaultMetricsPort = 9946 + MongotDefaultPort = 27027 + MongotDefaultMetricsPort = 9946 + MongotDefaultSyncSourceUsername = "mongot-user" ) func init() { @@ -38,6 +41,10 @@ type MongoDBSearchSpec struct { type MongoDBSource struct { // +optional MongoDBResourceRef *userv1.MongoDBResourceRef `json:"mongodbResourceRef,omitempty"` + // +optional + PasswordSecretRef *userv1.SecretKeyRef `json:"passwordSecretRef,omitempty"` + // +optional + Username *string `json:"username,omitempty"` } type MongoDBSearchStatus struct { @@ -105,6 +112,25 @@ func (s *MongoDBSearch) MongotConfigConfigMapNamespacedName() types.NamespacedNa return types.NamespacedName{Name: s.Name + "-search-config", Namespace: s.Namespace} } +func (s *MongoDBSearch) SourceUserPasswordSecretRef() *userv1.SecretKeyRef { + if s.Spec.Source != nil && s.Spec.Source.PasswordSecretRef != nil { + return s.Spec.Source.PasswordSecretRef + } + + return &userv1.SecretKeyRef{ + Name: fmt.Sprintf("%s-%s-password", s.Name, MongotDefaultSyncSourceUsername), + Key: "password", + } +} + +func (s *MongoDBSearch) SourceUsername() string { + if s.Spec.Source != nil && s.Spec.Source.Username != nil { + return *s.Spec.Source.Username + } + + return MongotDefaultSyncSourceUsername +} + func (s *MongoDBSearch) StatefulSetNamespacedName() types.NamespacedName { return types.NamespacedName{Name: s.Name + "-search", Namespace: s.Namespace} } diff --git a/config/crd/bases/mongodb.com_mongodbsearch.yaml b/config/crd/bases/mongodb.com_mongodbsearch.yaml index 272e7bc59..eeba52768 100644 --- a/config/crd/bases/mongodb.com_mongodbsearch.yaml +++ b/config/crd/bases/mongodb.com_mongodbsearch.yaml @@ -160,6 +160,21 @@ spec: required: - name type: object + passwordSecretRef: + description: |- + SecretKeyRef is a reference to a value in a given secret in the same + namespace. Based on: + https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.15/#secretkeyselector-v1-core + properties: + key: + type: string + name: + type: string + required: + - name + type: object + username: + type: string type: object statefulSet: description: |- diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 3e4f72934..ce5ce623d 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -392,9 +392,9 @@ spec: - name: RELATED_IMAGE_MONGODB_IMAGE_8_0_0_ubi9 value: "quay.io/mongodb/mongodb-enterprise-server:8.0.0-ubi9" - name: RELATED_IMAGE_MDB_SEARCH_IMAGE_1_47_0 - value: "quay.io/mongodb/mongodb-search-community:1.47.0" + value: "268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-search-community:1.47.0" - name: MDB_SEARCH_COMMUNITY_REPO_URL - value: "quay.io/mongodb" + value: "268558157000.dkr.ecr.us-east-1.amazonaws.com/dev" - name: MDB_SEARCH_COMMUNITY_NAME value: "mongodb-search-community" - name: MDB_SEARCH_COMMUNITY_VERSION diff --git a/controllers/operator/mongodbsearch_controller_test.go b/controllers/operator/mongodbsearch_controller_test.go index 3c4cfb556..974ec3935 100644 --- a/controllers/operator/mongodbsearch_controller_test.go +++ b/controllers/operator/mongodbsearch_controller_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/workqueue" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllertest" "sigs.k8s.io/controller-runtime/pkg/event" @@ -73,21 +74,42 @@ func newSearchReconciler( } func buildExpectedMongotConfig(search *searchv1.MongoDBSearch, mdbc *mdbcv1.MongoDBCommunity) mongot.Config { - return mongot.Config{CommunityPrivatePreview: mongot.CommunityPrivatePreview{ - MongodHostAndPort: fmt.Sprintf( - "%s.%s.svc.cluster.local:%d", - mdbc.ServiceName(), mdbc.Namespace, - mdbc.GetMongodConfiguration().GetDBPort(), - ), - QueryServerAddress: fmt.Sprintf("localhost:%d", search.GetMongotPort()), - KeyFilePath: "/mongot/keyfile/keyfile", - DataPath: "/mongot/data/config.yml", - Metrics: mongot.Metrics{ + return mongot.Config{ + SyncSource: mongot.ConfigSyncSource{ + ReplicaSet: mongot.ConfigReplicaSet{ + HostAndPort: fmt.Sprintf("%s.%s.svc.cluster.local:%d", mdbc.Name+"-svc", search.Namespace, 27017), + Username: "mongot-user", + PasswordFile: "/tmp/sourceUserPassword", + TLS: ptr.To(false), + ReadPreference: ptr.To("secondaryPreferred"), + ReplicaSetName: "mdb", + }, + }, + Storage: mongot.ConfigStorage{ + DataPath: "/mongot/data/config.yml", + }, + Server: mongot.ConfigServer{ + Wireproto: &mongot.ConfigWireproto{ + Address: "0.0.0.0:27027", + Authentication: &mongot.ConfigAuthentication{ + Mode: "keyfile", + KeyFile: "/tmp/keyfile", + }, + TLS: mongot.ConfigTLS{Mode: "disabled"}, + }, + }, + Metrics: mongot.ConfigMetrics{ Enabled: true, - Address: fmt.Sprintf("localhost:%d", search.GetMongotMetricsPort()), + Address: fmt.Sprintf("localhost:%d", searchv1.MongotDefaultMetricsPort), + }, + HealthCheck: mongot.ConfigHealthCheck{ + Address: "0.0.0.0:8080", }, - Logging: mongot.Logging{Verbosity: "DEBUG"}, - }} + Logging: mongot.ConfigLogging{ + Verbosity: "TRACE", + LogPath: nil, + }, + } } func TestMongoDBSearchReconcile_NotFound(t *testing.T) { diff --git a/controllers/search_controller/mongodbsearch_reconcile_helper.go b/controllers/search_controller/mongodbsearch_reconcile_helper.go index 59523ce8e..65e49f4f9 100644 --- a/controllers/search_controller/mongodbsearch_reconcile_helper.go +++ b/controllers/search_controller/mongodbsearch_reconcile_helper.go @@ -13,6 +13,7 @@ import ( "golang.org/x/xerrors" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -207,26 +208,50 @@ func buildSearchHeadlessService(search *searchv1.MongoDBSearch) corev1.Service { } func createMongotConfig(search *searchv1.MongoDBSearch, db SearchSourceDBResource) mongot.Config { - return mongot.Config{CommunityPrivatePreview: mongot.CommunityPrivatePreview{ - MongodHostAndPort: fmt.Sprintf("%s.%s.svc.cluster.local:%d", db.DatabaseServiceName(), db.GetNamespace(), db.DatabasePort()), - QueryServerAddress: fmt.Sprintf("localhost:%d", search.GetMongotPort()), - KeyFilePath: "/mongot/keyfile/keyfile", - DataPath: "/mongot/data/config.yml", - Metrics: mongot.Metrics{ + return mongot.Config{ + SyncSource: mongot.ConfigSyncSource{ + ReplicaSet: mongot.ConfigReplicaSet{ + HostAndPort: fmt.Sprintf("%s.%s.svc.cluster.local:%d", db.DatabaseServiceName(), db.GetNamespace(), db.DatabasePort()), + Username: search.SourceUsername(), + PasswordFile: "/tmp/sourceUserPassword", + TLS: ptr.To(false), + ReadPreference: ptr.To("secondaryPreferred"), + ReplicaSetName: db.Name(), + }, + }, + Storage: mongot.ConfigStorage{ + DataPath: "/mongot/data/config.yml", + }, + Server: mongot.ConfigServer{ + Wireproto: &mongot.ConfigWireproto{ + Address: "0.0.0.0:27027", + Authentication: &mongot.ConfigAuthentication{ + Mode: "keyfile", + KeyFile: "/tmp/keyfile", + }, + TLS: mongot.ConfigTLS{Mode: "disabled"}, + }, + }, + Metrics: mongot.ConfigMetrics{ Enabled: true, Address: fmt.Sprintf("localhost:%d", search.GetMongotMetricsPort()), }, - Logging: mongot.Logging{ - Verbosity: "DEBUG", + HealthCheck: mongot.ConfigHealthCheck{ + Address: "0.0.0.0:8080", }, - }} + Logging: mongot.ConfigLogging{ + Verbosity: "TRACE", + LogPath: nil, + }, + } } func GetMongodConfigParameters(search *searchv1.MongoDBSearch) map[string]interface{} { return map[string]interface{}{ "setParameter": map[string]interface{}{ - "mongotHost": mongotHostAndPort(search), - "searchIndexManagementHostAndPort": mongotHostAndPort(search), + "mongotHost": mongotHostAndPort(search), + "searchIndexManagementHostAndPort": mongotHostAndPort(search), + "skipAuthenticationToSearchIndexManagementServer": false, }, } } diff --git a/controllers/search_controller/search_construction.go b/controllers/search_controller/search_construction.go index ea169339d..2de1364e1 100644 --- a/controllers/search_controller/search_construction.go +++ b/controllers/search_controller/search_construction.go @@ -95,14 +95,18 @@ func CreateSearchStatefulSetFunc(mdbSearch *searchv1.MongoDBSearch, sourceDBReso dataVolumeName := "data" keyfileVolumeName := "keyfile" + sourceUserPasswordVolumeName := "password" mongotConfigVolumeName := "config" pvcVolumeMount := statefulset.CreateVolumeMount(dataVolumeName, "/mongot/data", statefulset.WithSubPath("data")) - keyfileVolume := statefulset.CreateVolumeFromSecret("keyfile", sourceDBResource.KeyfileSecretName()) + keyfileVolume := statefulset.CreateVolumeFromSecret(keyfileVolumeName, sourceDBResource.KeyfileSecretName()) keyfileVolumeMount := statefulset.CreateVolumeMount(keyfileVolumeName, "/mongot/keyfile", statefulset.WithReadOnly(true)) - mongotConfigVolume := statefulset.CreateVolumeFromConfigMap("config", mdbSearch.MongotConfigConfigMapNamespacedName().Name) + sourceUserPasswordVolume := statefulset.CreateVolumeFromSecret(sourceUserPasswordVolumeName, mdbSearch.SourceUserPasswordSecretRef().Name) + sourceUserPasswordVolumeMount := statefulset.CreateVolumeMount(sourceUserPasswordVolumeName, "/mongot/sourceUserPassword", statefulset.WithReadOnly(true)) + + mongotConfigVolume := statefulset.CreateVolumeFromConfigMap(mongotConfigVolumeName, mdbSearch.MongotConfigConfigMapNamespacedName().Name) mongotConfigVolumeMount := statefulset.CreateVolumeMount(mongotConfigVolumeName, "/mongot/config", statefulset.WithReadOnly(true)) var persistenceConfig *common.PersistenceConfig @@ -120,12 +124,14 @@ func CreateSearchStatefulSetFunc(mdbSearch *searchv1.MongoDBSearch, sourceDBReso keyfileVolumeMount, tmpVolumeMount, mongotConfigVolumeMount, + sourceUserPasswordVolumeMount, } volumes := []corev1.Volume{ tmpVolume, keyfileVolume, mongotConfigVolume, + sourceUserPasswordVolume, } stsModifications := []statefulset.Modification{ @@ -180,7 +186,17 @@ func mongodbSearchContainer(mdbSearch *searchv1.MongoDBSearch, volumeMounts []co container.WithCommand([]string{"sh"}), container.WithArgs([]string{ "-c", - "/mongot-community/mongot --config /mongot/config/config.yml", + ` +cp /mongot/keyfile/keyfile /tmp/keyfile +chown 2000:2000 /tmp/keyfile +chmod 0600 /tmp/keyfile + +cp /mongot/sourceUserPassword/password /tmp/sourceUserPassword +chown 2000:2000 /tmp/sourceUserPassword +chmod 0600 /tmp/sourceUserPassword + +/mongot-community/mongot --config /mongot/config/config.yml +`, }), containerSecurityContext, ) diff --git a/docker/mongodb-kubernetes-tests/kubetester/helm.py b/docker/mongodb-kubernetes-tests/kubetester/helm.py index da6887b81..9e0cca8fe 100644 --- a/docker/mongodb-kubernetes-tests/kubetester/helm.py +++ b/docker/mongodb-kubernetes-tests/kubetester/helm.py @@ -120,6 +120,9 @@ def helm_repo_add(repo_name: str, url: str): def process_run_and_check(args, **kwargs): + if "check" not in kwargs: + kwargs["check"] = True + try: logger.debug(f"subprocess.run: {args}") completed_process = subprocess.run(args, **kwargs) diff --git a/docker/mongodb-kubernetes-tests/tests/common/search/movies_search_helper.py b/docker/mongodb-kubernetes-tests/tests/common/search/movies_search_helper.py index e5629bda0..807d97493 100644 --- a/docker/mongodb-kubernetes-tests/tests/common/search/movies_search_helper.py +++ b/docker/mongodb-kubernetes-tests/tests/common/search/movies_search_helper.py @@ -1,3 +1,8 @@ +import logging + +import pymongo.errors + +from kubetester import kubetester from tests import test_logger from tests.common.search.search_tester import SearchTester @@ -26,9 +31,33 @@ def create_search_index(self): def wait_for_search_indexes(self): self.search_tester.wait_for_search_indexes_ready(self.db_name, self.col_name) - def assert_search_query(self): - # sample taken from: https://www.mongodb.com/docs/atlas/atlas-search/tutorial/run-query/#run-a-complex-query-on-the-movies-collection-7 - result = self.search_tester.client[self.db_name][self.col_name].aggregate( + def assert_search_query(self, retry_timeout: int = 1): + def wait_for_search_results(): + # sample taken from: https://www.mongodb.com/docs/atlas/atlas-search/tutorial/run-query/#run-a-complex-query-on-the-movies-collection-7 + count = 0 + status_msg = "" + try: + result = self.execute_example_search_query() + status_msg = f"{self.db_name}/{self.col_name}: search query results:\n" + for r in result: + status_msg += f"{r}\n" + count += 1 + status_msg += f"Count: {count}" + logger.debug(status_msg) + except pymongo.errors.PyMongoError as e: + logger.debug(f"error: {e}") + + return count == 4, status_msg + + kubetester.run_periodically( + fn=wait_for_search_results, + timeout=retry_timeout, + sleep_time=1, + msg="Search query to return correct data", + ) + + def execute_example_search_query(self): + return self.search_tester.client[self.db_name][self.col_name].aggregate( [ { "$search": { @@ -47,11 +76,3 @@ def assert_search_query(self): {"$project": {"title": 1, "plot": 1, "genres": 1, "_id": 0}}, ] ) - - logger.debug(f"{self.db_name}/{self.col_name}: search query results:") - count = 0 - for r in result: - logger.debug(r) - count += 1 - - assert count == 4 diff --git a/docker/mongodb-kubernetes-tests/tests/common/search/search_tester.py b/docker/mongodb-kubernetes-tests/tests/common/search/search_tester.py index bbca34b3b..7c5b78eac 100644 --- a/docker/mongodb-kubernetes-tests/tests/common/search/search_tester.py +++ b/docker/mongodb-kubernetes-tests/tests/common/search/search_tester.py @@ -49,6 +49,10 @@ def wait_for_search_indexes_ready(self, database_name: str, collection_name: str def search_indexes_ready(self, database_name: str, collection_name: str): search_indexes = self.get_search_indexes(database_name, collection_name) + if len(search_indexes) == 0: + logger.error(f"There are no search indexes available in {database_name}.{collection_name}") + return False + for idx in search_indexes: if idx.get("status") != "READY": logger.debug(f"{database_name}/{collection_name}: search index {idx} is not ready") diff --git a/docker/mongodb-kubernetes-tests/tests/search/fixtures/community-replicaset-sample-mflix.yaml b/docker/mongodb-kubernetes-tests/tests/search/fixtures/community-replicaset-sample-mflix.yaml index 4ba3d40b6..4042b7d65 100644 --- a/docker/mongodb-kubernetes-tests/tests/search/fixtures/community-replicaset-sample-mflix.yaml +++ b/docker/mongodb-kubernetes-tests/tests/search/fixtures/community-replicaset-sample-mflix.yaml @@ -9,22 +9,36 @@ spec: version: "8.0.5" security: authentication: - modes: ["SCRAM"] + modes: ["SCRAM-SHA-1"] agent: logLevel: DEBUG users: - - name: my-user + - name: mdb-admin db: admin passwordSecretRef: # a reference to the secret that will be used to generate the user's password - name: my-user-password + name: mdb-admin-user-password roles: - - name: clusterAdmin - db: admin - - name: userAdminAnyDatabase + - name: root db: admin + scramCredentialsSecretName: mdb-admin-user-scram + - name: mdb-user + db: admin + passwordSecretRef: # a reference to the secret that will be used to generate the user's password + name: mdb-user-password + roles: + - name: restore + db: sample_mflix - name: readWrite db: sample_mflix - scramCredentialsSecretName: my-scram + scramCredentialsSecretName: mdb-user-scram + - name: mongot-user + db: admin + passwordSecretRef: # a reference to the secret that will be used to generate the user's password + name: mdbc-rs-mongot-user-password + roles: + - name: searchCoordinator + db: admin + scramCredentialsSecretName: mongot-user-scram statefulSet: spec: template: diff --git a/docker/mongodb-kubernetes-tests/tests/search/fixtures/search-with-user-password.yaml b/docker/mongodb-kubernetes-tests/tests/search/fixtures/search-with-user-password.yaml new file mode 100644 index 000000000..33648e04a --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/search/fixtures/search-with-user-password.yaml @@ -0,0 +1,5 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBSearch +metadata: + name: mdbc-rs +spec: {} diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_community_basic.py b/docker/mongodb-kubernetes-tests/tests/search/search_community_basic.py index 69cbca895..18e2ec025 100644 --- a/docker/mongodb-kubernetes-tests/tests/search/search_community_basic.py +++ b/docker/mongodb-kubernetes-tests/tests/search/search_community_basic.py @@ -12,8 +12,15 @@ logger = test_logger.get_test_logger(__name__) -USER_PASSWORD = "Passw0rd." -USER_NAME = "my-user" +ADMIN_USER_NAME = "mdb-admin-user" +ADMIN_USER_PASSWORD = "mdb-admin-user-pass" + +MONGOT_USER_NAME = "mongot-user" +MONGOT_USER_PASSWORD = "mongot-user-password" + +USER_NAME = "mdb-user" +USER_PASSWORD = "mdb-user-pass" + MDBC_RESOURCE_NAME = "mdbc-rs" @@ -27,8 +34,8 @@ def mdbc(namespace: str, custom_mdb_version: str) -> MongoDBCommunity: resource["spec"]["version"] = custom_mdb_version - if try_load(resource): - return resource + # if try_load(resource): + # return resource return resource @@ -40,8 +47,8 @@ def mdbs(namespace: str) -> MongoDBSearch: namespace=namespace, ) - if try_load(resource): - return resource + # if try_load(resource): + # return resource return resource @@ -53,8 +60,14 @@ def test_install_operator(namespace: str, operator_installation_config: dict[str @mark.e2e_search_community_basic -def test_install_secret(namespace: str): - create_or_update_secret(namespace=namespace, name="my-user-password", data={"password": USER_PASSWORD}) +def test_install_secrets(namespace: str, mdbs: MongoDBSearch): + create_or_update_secret(namespace=namespace, name=f"{USER_NAME}-password", data={"password": USER_PASSWORD}) + create_or_update_secret( + namespace=namespace, name=f"{ADMIN_USER_NAME}-password", data={"password": ADMIN_USER_PASSWORD} + ) + create_or_update_secret( + namespace=namespace, name=f"{mdbs.name}-{MONGOT_USER_NAME}-password", data={"password": MONGOT_USER_PASSWORD} + ) @mark.e2e_search_community_basic @@ -76,7 +89,9 @@ def test_wait_for_community_resource_ready(mdbc: MongoDBCommunity): @fixture(scope="function") def sample_movies_helper(mdbc: MongoDBCommunity) -> SampleMoviesSearchHelper: - return movies_search_helper.SampleMoviesSearchHelper(SearchTester(get_connection_string(mdbc))) + return movies_search_helper.SampleMoviesSearchHelper( + SearchTester(get_connection_string(mdbc, USER_NAME, USER_PASSWORD)) + ) @mark.e2e_search_community_basic @@ -89,15 +104,15 @@ def test_search_create_search_index(sample_movies_helper: SampleMoviesSearchHelp sample_movies_helper.create_search_index() -@mark.e2e_search_community_basic -def test_search_wait_for_search_indexes(sample_movies_helper: SampleMoviesSearchHelper): - sample_movies_helper.wait_for_search_indexes() +# @mark.e2e_search_community_basic +# def test_search_wait_for_search_indexes(sample_movies_helper: SampleMoviesSearchHelper): +# sample_movies_helper.wait_for_search_indexes() @mark.e2e_search_community_basic def test_search_assert_search_query(sample_movies_helper: SampleMoviesSearchHelper): - sample_movies_helper.assert_search_query() + sample_movies_helper.assert_search_query(retry_timeout=60) -def get_connection_string(mdbc: MongoDBCommunity) -> str: - return f"mongodb://{USER_NAME}:{USER_PASSWORD}@{mdbc.name}-0.{mdbc.name}-svc.{mdbc.namespace}.svc.cluster.local:27017/?replicaSet={mdbc.name}" +def get_connection_string(mdbc: MongoDBCommunity, user_name: str, user_password: str) -> str: + return f"mongodb://{user_name}:{user_password}@{mdbc.name}-0.{mdbc.name}-svc.{mdbc.namespace}.svc.cluster.local:27017/?replicaSet={mdbc.name}" diff --git a/helm_chart/crds/mongodb.com_mongodbsearch.yaml b/helm_chart/crds/mongodb.com_mongodbsearch.yaml index 272e7bc59..eeba52768 100644 --- a/helm_chart/crds/mongodb.com_mongodbsearch.yaml +++ b/helm_chart/crds/mongodb.com_mongodbsearch.yaml @@ -160,6 +160,21 @@ spec: required: - name type: object + passwordSecretRef: + description: |- + SecretKeyRef is a reference to a value in a given secret in the same + namespace. Based on: + https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.15/#secretkeyselector-v1-core + properties: + key: + type: string + name: + type: string + required: + - name + type: object + username: + type: string type: object statefulSet: description: |- diff --git a/helm_chart/values.yaml b/helm_chart/values.yaml index 0a44d5ca0..281ff3c30 100644 --- a/helm_chart/values.yaml +++ b/helm_chart/values.yaml @@ -222,7 +222,11 @@ community: search: community: # Full Search container image url used for the MongoDB Community Search container will be constructed as {search.community.repo}/{search.community.name}:{search.community.version} - repo: quay.io/mongodb - name: mongodb-search-community +# repo: quay.io/mongodb +# name: mongodb-search-community +# # default MongoDB Search version used; can be overridden by setting MongoDBSearch.spec.version field. +# version: 1.47.0 + repo: 268558157000.dkr.ecr.eu-west-1.amazonaws.com + name: mongot/community # default MongoDB Search version used; can be overridden by setting MongoDBSearch.spec.version field. - version: 1.47.0 + version: b9b80915f5571bfa5fc2aa70acb20d784e68d79b diff --git a/mongodb-community-operator/controllers/replica_set_controller.go b/mongodb-community-operator/controllers/replica_set_controller.go index 6c0190fb0..644719294 100644 --- a/mongodb-community-operator/controllers/replica_set_controller.go +++ b/mongodb-community-operator/controllers/replica_set_controller.go @@ -14,6 +14,7 @@ import ( "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -727,6 +728,7 @@ func (r ReplicaSetReconciler) buildAutomationConfig(ctx context.Context, mdb mdb prometheusModification, processPortManager.GetPortsModification(), getMongodConfigSearchModification(search), + searchCoordinatorCustomRoleModification(search), ) if err != nil { return automationconfig.AutomationConfig{}, fmt.Errorf("could not create an automation config: %s", err) @@ -739,6 +741,66 @@ func (r ReplicaSetReconciler) buildAutomationConfig(ctx context.Context, mdb mdb return automationConfig, nil } +// TODO: remove this as soon as searchCoordinator builtin role is backported +func searchCoordinatorCustomRoleModification(search *searchv1.MongoDBSearch) automationconfig.Modification { + if search == nil { + return automationconfig.NOOP() + } + + return func(ac *automationconfig.AutomationConfig) { + searchCoordinatorRole := searchCoordinatorCustomRoleStruct() + ac.Roles = append(ac.Roles, searchCoordinatorRole) + } +} + +func searchCoordinatorCustomRoleStruct() automationconfig.CustomRole { + // direct translation of https://github.com/10gen/mongo/blob/6f8d95a513eea8f91ea9f5d895dd8a288dfcf725/src/mongo/db/auth/builtin_roles.yml#L652 + return automationconfig.CustomRole{ + Role: "searchCoordinator", + DB: "admin", + Roles: []automationconfig.Role{ + { + Role: "clusterMonitor", + Database: "admin", + }, + { + Role: "directShardOperations", + Database: "admin", + }, + { + Role: "readAnyDatabase", + Database: "admin", + }, + }, + Privileges: []automationconfig.Privilege{ + { + Resource: automationconfig.Resource{ + DB: ptr.To("__mdb_internal_search"), + Collection: ptr.To(""), + }, + Actions: []string{ + "changeStream", "collStats", "dbHash", "dbStats", "find", + "killCursors", "listCollections", "listIndexes", "listSearchIndexes", + // performRawDataOperations is available only on mongod master + // "performRawDataOperations", + "planCacheRead", "cleanupStructuredEncryptionData", + "compactStructuredEncryptionData", "convertToCapped", "createCollection", + "createIndex", "createSearchIndexes", "dropCollection", "dropIndex", + "dropSearchIndex", "insert", "remove", "renameCollectionSameDB", + "update", "updateSearchIndex", + }, + }, + { + Resource: automationconfig.Resource{ + Cluster: true, + }, + Actions: []string{"bypassDefaultMaxTimeMS"}, + }, + }, + AuthenticationRestrictions: nil, + } +} + // OverrideToAutomationConfig turns an automation config override from the resource spec into an automation config // which can be used to merge. func OverrideToAutomationConfig(override mdbv1.AutomationConfigOverride) automationconfig.AutomationConfig { @@ -773,9 +835,7 @@ func getMongodConfigModification(mdb mdbv1.MongoDBCommunity) automationconfig.Mo // into the configuration set up by the operator. func getMongodConfigSearchModification(search *searchv1.MongoDBSearch) automationconfig.Modification { if search == nil { - return func(config *automationconfig.AutomationConfig) { - // do nothing - } + return automationconfig.NOOP() } searchConfigParameters := search_controller.GetMongodConfigParameters(search) diff --git a/mongodb-community-operator/pkg/mongot/mongot_config.go b/mongodb-community-operator/pkg/mongot/mongot_config.go index 80d4c1421..fed5938c6 100644 --- a/mongodb-community-operator/pkg/mongot/mongot_config.go +++ b/mongodb-community-operator/pkg/mongot/mongot_config.go @@ -1,41 +1,68 @@ package mongot type Config struct { - CommunityPrivatePreview CommunityPrivatePreview `json:"communityPrivatePreview"` + SyncSource ConfigSyncSource `json:"syncSource"` + Storage ConfigStorage `json:"storage"` + Server ConfigServer `json:"server"` + Metrics ConfigMetrics `json:"metrics"` + HealthCheck ConfigHealthCheck `json:"healthCheck"` + Logging ConfigLogging `json:"logging"` } -// CommunityPrivatePreview structure reflects private preview configuration from mongot: -// https://github.com/10gen/mongot/blob/060ec179af062ac2639678f4a613b8ab02c21597/src/main/java/com/xgen/mongot/config/provider/community/CommunityConfig.java#L100 -// Comments are from the default config file: https://github.com/10gen/mongot/blob/375379e56a580916695a2f53e12fd4a99aa24f0b/deploy/community-resources/config.default.yml#L1-L0 -type CommunityPrivatePreview struct { - // Socket (IPv4/6) address of the sync source mongod - MongodHostAndPort string `json:"mongodHostAndPort"` - - // Socket (IPv4/6) address on which to listen for wire protocol connections - QueryServerAddress string `json:"queryServerAddress"` +type ConfigSyncSource struct { + ReplicaSet ConfigReplicaSet `json:"replicaSet"` +} - // Keyfile used for mongod -> mongot authentication - KeyFilePath string `json:"keyFilePath"` +type ConfigReplicaSet struct { + HostAndPort string `json:"hostAndPort"` + Username string `json:"username"` + PasswordFile string `json:"passwordFile"` + ReplicaSetName string `json:"replicaSetName"` + TLS *bool `json:"tls,omitempty"` + ReadPreference *string `json:"readPreference,omitempty"` +} - // Filesystem path that all mongot data will be stored at +type ConfigStorage struct { DataPath string `json:"dataPath"` +} - // Options for metrics - Metrics Metrics `json:"metrics,omitempty"` +type ConfigServer struct { + Wireproto *ConfigWireproto `json:"wireproto,omitempty"` +} + +type ConfigWireproto struct { + Address string `json:"address"` + Authentication *ConfigAuthentication `json:"authentication,omitempty"` + TLS ConfigTLS `json:"tls"` +} - // Options for logging - Logging Logging `json:"logging,omitempty"` +type ConfigAuthentication struct { + Mode string `json:"mode"` + KeyFile string `json:"keyFile"` } -type Metrics struct { - // Whether to enable the Prometheus metrics endpoint - Enabled bool `json:"enabled"` +type ConfigTLSMode string + +const ( + ConfigTLSModeTLS ConfigTLSMode = "TLS" + ConfigTLSModeDisabled ConfigTLSMode = "Disabled" +) + +type ConfigTLS struct { + Mode ConfigTLSMode `json:"mode"` + CertificateKeyFile *string `json:"certificateKeyFile,omitempty"` +} + +type ConfigMetrics struct { + Enabled bool `json:"enabled"` + Address string `json:"address"` +} - // Socket address (IPv4/6) on which the Prometheus /metrics endpoint will be exposed +type ConfigHealthCheck struct { Address string `json:"address"` } -type Logging struct { - // Log level - Verbosity string `json:"verbosity"` +type ConfigLogging struct { + Verbosity string `json:"verbosity"` + LogPath *string `json:"logPath,omitempty"` } diff --git a/public/crds.yaml b/public/crds.yaml index 31015b6d5..d5d28ca3a 100644 --- a/public/crds.yaml +++ b/public/crds.yaml @@ -4828,6 +4828,21 @@ spec: required: - name type: object + passwordSecretRef: + description: |- + SecretKeyRef is a reference to a value in a given secret in the same + namespace. Based on: + https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.15/#secretkeyselector-v1-core + properties: + key: + type: string + name: + type: string + required: + - name + type: object + username: + type: string type: object statefulSet: description: |- diff --git a/public/mongodb-kubernetes-multi-cluster.yaml b/public/mongodb-kubernetes-multi-cluster.yaml index f53630a0e..a83643211 100644 --- a/public/mongodb-kubernetes-multi-cluster.yaml +++ b/public/mongodb-kubernetes-multi-cluster.yaml @@ -432,7 +432,7 @@ spec: value: "ubi8" # Community Env Vars End - name: MDB_SEARCH_COMMUNITY_REPO_URL - value: "quay.io/mongodb" + value: "268558157000.dkr.ecr.us-east-1.amazonaws.com/dev" - name: MDB_SEARCH_COMMUNITY_NAME value: "mongodb-search-community" - name: MDB_SEARCH_COMMUNITY_VERSION diff --git a/public/mongodb-kubernetes-openshift.yaml b/public/mongodb-kubernetes-openshift.yaml index ce0cd73b9..cfe6359ca 100644 --- a/public/mongodb-kubernetes-openshift.yaml +++ b/public/mongodb-kubernetes-openshift.yaml @@ -696,9 +696,9 @@ spec: - name: RELATED_IMAGE_MONGODB_IMAGE_8_0_0_ubi9 value: "quay.io/mongodb/mongodb-enterprise-server:8.0.0-ubi9" - name: RELATED_IMAGE_MDB_SEARCH_IMAGE_1_47_0 - value: "quay.io/mongodb/mongodb-search-community:1.47.0" + value: "268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-search-community:1.47.0" - name: MDB_SEARCH_COMMUNITY_REPO_URL - value: "quay.io/mongodb" + value: "268558157000.dkr.ecr.us-east-1.amazonaws.com/dev" - name: MDB_SEARCH_COMMUNITY_NAME value: "mongodb-search-community" - name: MDB_SEARCH_COMMUNITY_VERSION diff --git a/public/mongodb-kubernetes.yaml b/public/mongodb-kubernetes.yaml index 69ff3f0e8..16406a9da 100644 --- a/public/mongodb-kubernetes.yaml +++ b/public/mongodb-kubernetes.yaml @@ -428,7 +428,7 @@ spec: value: "ubi8" # Community Env Vars End - name: MDB_SEARCH_COMMUNITY_REPO_URL - value: "quay.io/mongodb" + value: "268558157000.dkr.ecr.us-east-1.amazonaws.com/dev" - name: MDB_SEARCH_COMMUNITY_NAME value: "mongodb-search-community" - name: MDB_SEARCH_COMMUNITY_VERSION diff --git a/scripts/dev/contexts/e2e_mdb_community b/scripts/dev/contexts/e2e_mdb_community index 4f096c2d7..ebce584eb 100644 --- a/scripts/dev/contexts/e2e_mdb_community +++ b/scripts/dev/contexts/e2e_mdb_community @@ -10,3 +10,11 @@ source "${script_dir}/variables/mongodb_latest" # This variable is needed otherwise the `fetch_om_information.sh` script is called and fails the test export OM_EXTERNALLY_CONFIGURED="true" + +# Temporary development images built from mongot master +#export MDB_SEARCH_COMMUNITY_VERSION="776d43523d185b6b234289e17c191712a3e6569b" # master +#export MDB_SEARCH_COMMUNITY_VERSION="d6884ae132aab30497af55dbaff05e8274e9775f" # Local->Admin +#export MDB_SEARCH_COMMUNITY_VERSION="ad8acf5c3a045d6e0306ad67d61fcb5be40f57ae" # hardcoded mdbc-rs replicaset name +export MDB_SEARCH_COMMUNITY_VERSION="b9b80915f5571bfa5fc2aa70acb20d784e68d79b" # hardcoded Local->Admin, handled replicaSetName in config +export MDB_SEARCH_COMMUNITY_NAME="mongot/community" +export MDB_SEARCH_COMMUNITY_REPO_URL="268558157000.dkr.ecr.eu-west-1.amazonaws.com" diff --git a/scripts/dev/contexts/root-context b/scripts/dev/contexts/root-context index 7b147aa34..6ec7c7a61 100644 --- a/scripts/dev/contexts/root-context +++ b/scripts/dev/contexts/root-context @@ -10,6 +10,7 @@ export PROJECT_DIR="${PWD}" export IMAGE_TYPE=ubi export UBI_IMAGE_WITHOUT_SUFFIX=true export WATCH_NAMESPACE=${WATCH_NAMESPACE:-${NAMESPACE}} +export OPERATOR_NAME="mongodb-kubernetes-operator" # # changing variables below should not be necessary