CE-477 Couchbase Version 7.1.3 Support

plugin_config.yml

@@ -1,7 +1,7 @@
 id: 18f4ff11-b758-4bf2-9a37-719a22f5a4b8
 name: sc:couchbase
-externalVersion: "1.2.1.3"
-buildNumber: 1.2.1.3
+externalVersion: "1.2.2"
+buildNumber: 1.2.2
 language: PYTHON38
 hostTypes:
 - UNIX

schema.json

@@ -275,8 +275,7 @@
       "bucketSize",
       "couchbaseAdmin",
       "couchbaseAdminPassword",
-      "xdcrAdmin",
-      "xdcrAdminPassword"
+      "xdcrAdmin"
     ],
     "ordering" : [
       "dSourceType",

src/controller/couchbase_lib/_bucket.py

@@ -214,7 +214,7 @@ def monitor_bucket(self, bucket_name, staging_UUID):
         while pending_docs != 0:
             logger.debug("Documents pending for replication: {}".format(pending_docs))
             helper_lib.sleepForSecond(30)
-            stdout, stderr, exit_code = utilities.execute_bash(self.connection, command, **kwargs)
+            stdout, stderr, exit_code = utilities.execute_expect(self.connection, command, **kwargs)
             content = json.loads(stdout)
             pending_docs = self._get_last_value_of_node_stats(list(content["nodeStats"].values())[0])
         else:

src/controller/couchbase_lib/_cb_backup.py

@@ -6,6 +6,7 @@
 """This class contains methods for all cb backup manager.
 This is child class of Resource and parent class of CouchbaseOperation
 """
+import json
 #######################################################################################################################
 import logging
 from utils import utilities
@@ -61,14 +62,36 @@ def cb_backup_full(self, csv_bucket):
         #                                     **env)
         # logger.debug("Backup restore: {}".format(cmd))
         # utilities.execute_bash(self.connection, cmd, **kwargs)
-
+        map_data_list = []
+        if int(self.repository.version.split(".")[0]) >= 7:
+            for bucket_name in csv_bucket.split(","):
+                logger.debug(f"bucket_name: {bucket_name}")
+                stdout, _, _ = self.run_couchbase_command(
+                    couchbase_command='get_scope_list_expect',
+                    base_path=helper_lib.get_base_directory_of_given_path(
+                        self.repository.cb_shell_path),
+                    bucket_name=bucket_name
+                )
+                json_scope_data = json.loads(stdout)
+                for s in json_scope_data["scopes"]:
+                    scope_name = s["name"]
+                    if scope_name == "_default":
+                        continue
+                    collection_list = s["collections"]
+                    for c in collection_list:
+                        collection_name = c["name"]
+                        if collection_name == "_default":
+                            continue
+                        map_data_list.append(f"{bucket_name}.{scope_name}.{collection_name}={bucket_name}.{scope_name}.{collection_name}")
 
         stdout, stderr, exit_code = self.run_couchbase_command(couchbase_command='cb_backup_full',
                                                             backup_location=self.parameters.couchbase_bak_loc,
                                                             csv_bucket_list=csv_bucket,
                                                             backup_repo=self.parameters.couchbase_bak_repo, 
                                                             skip=skip, 
-                                                            base_path=helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path)
+                                                            base_path=helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path),
+                                                            map_data=",".join(map_data_list),
+                                                            repo_version = self.repository.version
                                                         )
 
         if exit_code != 0:

src/controller/couchbase_lib/_cluster.py

@@ -69,10 +69,23 @@ def cluster_init(self):
         lambda_expr = lambda output: bool(re.search(ALREADY_CLUSTER_INIT, output))
         env = _ClusterMixin.generate_environment_map(self)
         env['additional_services'] = additional_service
-        cmd = CommandFactory.cluster_init(cluster_name=cluster_name, **env)
-        logger.debug("Cluster init: {}".format(cmd))
-        stdout, stderr, exit_code = utilities.execute_bash(self.connection, command_name=cmd, callback_func=lambda_expr,
-                                                           **kwargs)
+        if int(self.repository.version.split(".")[0]) >= 7:
+            env.update(kwargs[ENV_VAR_KEY])
+            if "(CE)" in self.repository.version:
+                env["cluster_eventing_ramsize"] = None
+                env["cluster_analytics_ramsize"] = None
+                env["indexerStorageMode"] = "forestdb"
+            else:
+                env["indexerStorageMode"] = "plasma"
+            cmd, env_vars = CommandFactory.cluster_init_rest_expect(cluster_name=cluster_name, **env)
+            kwargs[ENV_VAR_KEY].update(env_vars)
+            stdout, stderr, exit_code = utilities.execute_expect(self.connection,
+                                                                 cmd, **kwargs)
+        else:
+            cmd = CommandFactory.cluster_init(cluster_name=cluster_name, **env)
+            logger.debug("Cluster init: {}".format(cmd))
+            stdout, stderr, exit_code = utilities.execute_bash(self.connection, command_name=cmd, callback_func=lambda_expr,
+                                                               **kwargs)
         if re.search(r"ERROR", str(stdout)):
             if re.search(r"ERROR: Cluster is already initialized", stdout):
                 logger.debug("Performing cluster setting as cluster is already initialized")

src/controller/couchbase_lib/_xdcr.py

@@ -7,6 +7,7 @@
 This class contains methods for XDCR related operations
 This is child class of Resource and parent class of CouchbaseOperation
 """
+import json
 #######################################################################################################################
 import logging
 from utils import utilities
@@ -260,7 +261,7 @@ def setup_replication(self):
         config_setting = self.staged_source.parameters.config_settings_prov
 
         if len(config_setting) > 0:
-            bucket_list = [ config_bucket["bucketName"] for config_bucket in config_setting ]
+            bucket_list = [config_bucket["bucketName"] for config_bucket in config_setting]
         else:
             bucket_details_source = self.source_bucket_list()
             bucket_list = helper_lib.filter_bucket_name_from_json(bucket_details_source)
@@ -272,6 +273,45 @@ def setup_replication(self):
 
         for bkt_name in bucket_list:
             if bkt_name not in alredy_replicated_buckets:
+                if int(self.repository.version.split(".")[0]) >= 7:
+                    logger.debug(f"bucket_name: {bkt_name}")
+                    stdout, _, _ = self.run_couchbase_command(
+                        couchbase_command='get_scope_list_expect',
+                        base_path=helper_lib.get_base_directory_of_given_path(
+                            self.repository.cb_shell_path),
+                        hostname=self.source_config.couchbase_src_host,
+                        port=self.source_config.couchbase_src_port,
+                        username=self.staged_source.parameters.xdcr_admin,
+                        password=self.staged_source.parameters.xdcr_admin_password,
+                        bucket_name=bkt_name
+                    )
+                    json_scope_data = json.loads(stdout)
+                    logger.debug(f"json_scope_data={json_scope_data}")
+                    for s in json_scope_data["scopes"]:
+                        scope_name = s["name"]
+                        if scope_name == "_default":
+                            continue
+                        # create scope
+                        self.run_couchbase_command(
+                            couchbase_command="create_scope_expect",
+                            base_path=helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path),
+                            scope_name=scope_name,
+                            bucket_name=bkt_name
+                        )
+                        collection_list = s["collections"]
+                        for c in collection_list:
+                            collection_name = c["name"]
+                            if collection_name == "_default":
+                                continue
+                            # create collection
+                            self.run_couchbase_command(
+                                couchbase_command="create_collection_expect",
+                                base_path=helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path),
+                                scope_name=scope_name,
+                                bucket_name=bkt_name,
+                                collection_name=collection_name
+                            )
+
                 logger.debug("Creating replication for {}".format(bkt_name))
                 self.xdcr_replicate(bkt_name, bkt_name)
             else: