Skip to content

Commit 73303b9

Browse files
ayesharustgi-delphixayesharustgi-delphix
committed
* Issue #32 - multinode start
* Set theme jekyll-theme-minimal * add end of multi cluster * cbbackup exit code check * cbbackupmgr working with sudo, bucket list in snapshot in JSON * check mount points / doc updates * checks added, upgrade added * doc fixes * docs with multi node * fixing uid/gid regex * initial support for using non couchbase user through sudo * issue #22 - initial deployment, index support changes for backup ingestion, various fixes * new logo * partial work * proper style * resync of xdcr and backupmgr OK * v1.1.0 build 1.1.0.2 * v1.1.1 build 1.1.0.2 * ver 1.0.3.1 * ver 1.0.3.2 - sudo for all * version 1.0.2 * version 1.0.3 - error messages fix * version 1.1.0 build 0 * version 1.1.1 - added field for hostname * version 1.1.2 * workinf with copy config * working multinode * working with indexes - tested with backup only - ee, analytics removed from schema * xdcr fixed for multi IP, and different scenarios CE-59 Couchbase Plugin : Conversion of python2 to python3 (#38) Co-authored-by: Marcin Przepiorowski <[email protected]> Co-authored-by: ayesharustgi-delphix <[email protected]> DXE-683 Rename Select Connector - CouchBase DXE-988 Couchbase - VDB failed to enable (#36) Version 1.1.1 (#31) security fix draft
1 parent 2229226 commit 73303b9

File tree

6 files changed

+904
-47
lines changed

6 files changed

+904
-47
lines changed

‎src/controller/couchbase_lib/_bucket.py‎

Lines changed: 53 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,14 @@ def bucket_edit(self, bucket_name, flush_value=1):
3939
logger.debug("Editing bucket: {} ".format(bucket_name))
4040
self.__validate_bucket_name(bucket_name)
4141
env = _BucketMixin.generate_environment_map(self)
42-
command = CommandFactory.bucket_edit(bucket_name=bucket_name, flush_value=flush_value, **env)
43-
kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}}
42+
kwargs = {ENV_VAR_KEY: {
43+
'password': self.parameters.couchbase_admin_password}}
44+
env.update(kwargs[ENV_VAR_KEY])
45+
# command = CommandFactory.bucket_edit(bucket_name=bucket_name, flush_value=flush_value, **env)
46+
command, env_vars = CommandFactory.bucket_edit_expect(bucket_name=bucket_name, flush_value=flush_value, **env)
47+
kwargs[ENV_VAR_KEY].update(env_vars)
4448
logger.debug("edit bucket {}".format(command))
45-
return utilities.execute_bash(self.connection, command, **kwargs)
49+
return utilities.execute_expect(self.connection, command, **kwargs)
4650

4751
def bucket_edit_ramquota(self, bucket_name, _ramsize):
4852
"""
@@ -53,31 +57,45 @@ def bucket_edit_ramquota(self, bucket_name, _ramsize):
5357
# It requires the before bucket delete
5458
logger.debug("Editing bucket: {} ".format(bucket_name))
5559
self.__validate_bucket_name(bucket_name)
60+
kwargs = {ENV_VAR_KEY: {
61+
'password': self.parameters.couchbase_admin_password}}
5662
env = _BucketMixin.generate_environment_map(self)
57-
command = CommandFactory.bucket_edit_ramquota(bucket_name=bucket_name, ramsize=_ramsize, **env)
58-
kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}}
63+
env.update(kwargs[ENV_VAR_KEY])
64+
# command = CommandFactory.bucket_edit_ramquota(bucket_name=bucket_name, ramsize=_ramsize, **env)
65+
command, env_vars = CommandFactory.bucket_edit_ramquota_expect(bucket_name=bucket_name,
66+
ramsize=_ramsize, **env)
67+
kwargs[ENV_VAR_KEY].update(env_vars)
5968
logger.debug("edit ram bucket {}".format(command))
60-
return utilities.execute_bash(self.connection, command, **kwargs)
69+
return utilities.execute_expect(self.connection, command, **kwargs)
6170

6271
def bucket_delete(self, bucket_name):
6372
# To delete the bucket
6473
logger.debug("Deleting bucket: {} ".format(bucket_name))
6574
self.__validate_bucket_name(bucket_name)
6675
env = _BucketMixin.generate_environment_map(self)
67-
command = CommandFactory.bucket_delete(bucket_name=bucket_name, **env)
68-
kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}}
76+
kwargs = {ENV_VAR_KEY: {
77+
'password': self.parameters.couchbase_admin_password}}
78+
env.update(kwargs[ENV_VAR_KEY])
79+
# command = CommandFactory.bucket_delete(bucket_name=bucket_name, **env)
80+
command, env_vars = CommandFactory.bucket_delete_expect(bucket_name=bucket_name, **env)
81+
kwargs[ENV_VAR_KEY].update(env_vars)
6982
logger.debug("delete bucket {}".format(command))
70-
return utilities.execute_bash(self.connection, command, **kwargs)
83+
return utilities.execute_expect(self.connection, command, **kwargs)
7184

7285
def bucket_flush(self, bucket_name):
7386
# It requires the before bucket delete
7487
logger.debug("Flushing bucket: {} ".format(bucket_name))
7588
self.__validate_bucket_name(bucket_name)
7689
env = _BucketMixin.generate_environment_map(self)
77-
command = CommandFactory.bucket_flush(bucket_name=bucket_name, **env)
78-
kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}}
90+
kwargs = {ENV_VAR_KEY: {
91+
'password': self.parameters.couchbase_admin_password}}
92+
env.update(kwargs[ENV_VAR_KEY])
93+
# command, env_vars = CommandFactory.bucket_flush(bucket_name=bucket_name, **env)
94+
command, env_vars = CommandFactory.bucket_flush_expect(
95+
bucket_name=bucket_name, **env)
96+
kwargs[ENV_VAR_KEY].update(env_vars)
7997
logger.debug("flush bucket {}".format(command))
80-
return utilities.execute_bash(self.connection, command, **kwargs)
98+
return utilities.execute_expect(self.connection, command, **kwargs)
8199

82100
def bucket_remove(self, bucket_name):
83101
logger.debug("Removing bucket: {} ".format(bucket_name))
@@ -107,10 +125,14 @@ def bucket_create(self, bucket_name, ram_size, bucket_type, bucket_compression):
107125

108126
policy = self.parameters.bucket_eviction_policy
109127
env = _BucketMixin.generate_environment_map(self)
110-
command = CommandFactory.bucket_create(bucket_name=bucket_name, ramsize=ram_size, evictionpolicy=policy, bucket_type=bucket_type, bucket_compression=bucket_compression, **env)
111-
kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}}
128+
kwargs = {ENV_VAR_KEY: {
129+
'password': self.parameters.couchbase_admin_password}}
130+
env.update(kwargs[ENV_VAR_KEY])
131+
# command = CommandFactory.bucket_create(bucket_name=bucket_name, ramsize=ram_size, evictionpolicy=policy, bucket_type=bucket_type, bucket_compression=bucket_compression, **env)
132+
command, env_vars = CommandFactory.bucket_create_expect(bucket_name=bucket_name, ramsize=ram_size, evictionpolicy=policy, bucket_type=bucket_type, bucket_compression=bucket_compression, **env)
112133
logger.debug("create bucket {}".format(command))
113-
output, error, exit_code = utilities.execute_bash(self.connection, command, **kwargs)
134+
kwargs[ENV_VAR_KEY].update(env_vars)
135+
output, error, exit_code = utilities.execute_expect(self.connection, command, **kwargs)
114136
logger.debug("create bucket output: {} {} {}".format(output, error, exit_code))
115137
helper_lib.sleepForSecond(2)
116138

@@ -120,10 +142,17 @@ def bucket_list(self, return_type=list):
120142
# It will return also other information like ramused, ramsize etc
121143
logger.debug("Finding staged bucket list")
122144
env = _BucketMixin.generate_environment_map(self)
123-
command = CommandFactory.bucket_list(**env)
124-
kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}}
145+
kwargs = {ENV_VAR_KEY: {
146+
'password': self.parameters.couchbase_admin_password}}
147+
env.update(kwargs[ENV_VAR_KEY])
148+
# command = CommandFactory.bucket_list(**env)
149+
command, env_vars = CommandFactory.bucket_list_expect(**env)
150+
kwargs[ENV_VAR_KEY].update(env_vars)
125151
logger.debug("list bucket {}".format(command))
126-
bucket_list, error, exit_code = utilities.execute_bash(self.connection, command, **kwargs)
152+
# bucket_list, error, exit_code = utilities.execute_bash(self.connection, command, **kwargs)
153+
bucket_list, error, exit_code = utilities.execute_expect(self.connection,
154+
command,
155+
**kwargs)
127156
logger.debug("list bucket output{}".format(bucket_list))
128157
if return_type == list:
129158
#bucket_list = bucket_list.split("\n")
@@ -143,7 +172,6 @@ def bucket_list(self, return_type=list):
143172
logger.debug("Bucket details in staged environment: {}".format(bucket_list))
144173
return bucket_list_dict
145174

146-
147175
def move_bucket(self, bucket_name, direction):
148176
logger.debug("Rename folder")
149177

@@ -172,11 +200,14 @@ def monitor_bucket(self, bucket_name, staging_UUID):
172200
# To monitor the replication
173201
logger.debug("Monitoring the replication for bucket {} ".format(bucket_name))
174202
kwargs = {ENV_VAR_KEY: {'password': self.staged_source.parameters.xdcr_admin_password}}
175-
command = CommandFactory.monitor_replication(source_username=self.staged_source.parameters.xdcr_admin,
203+
env = kwargs[ENV_VAR_KEY]
204+
command, env_vars = CommandFactory.monitor_replication_expect(source_username=self.staged_source.parameters.xdcr_admin,
176205
source_hostname=self.source_config.couchbase_src_host,
177206
source_port=self.source_config.couchbase_src_port,
178-
bucket_name=bucket_name, uuid=staging_UUID)
179-
stdout, stderr, exit_code = utilities.execute_bash(self.connection, command, **kwargs)
207+
bucket_name=bucket_name, uuid=staging_UUID,
208+
**env)
209+
kwargs[ENV_VAR_KEY].update(env_vars)
210+
stdout, stderr, exit_code = utilities.execute_expect(self.connection, command, **kwargs)
180211
logger.debug("stdout: {}".format(stdout))
181212
content = json.loads(stdout)
182213
pending_docs = self._get_last_value_of_node_stats(list(content["nodeStats"].values())[0])

‎src/controller/couchbase_lib/_cluster.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,13 @@ def cluster_setting(self):
9292
kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}}
9393
cluster_name = self._get_cluster_name()
9494
env = _ClusterMixin.generate_environment_map(self)
95-
cmd = CommandFactory.cluster_setting(cluster_name=cluster_name, **env)
96-
stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd, **kwargs)
95+
env.update(kwargs[ENV_VAR_KEY])
96+
# cmd = CommandFactory.cluster_setting(cluster_name=cluster_name, **env)
97+
cmd, env_vars = CommandFactory.cluster_setting_expect(cluster_name=cluster_name, **env)
98+
# stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd, **kwargs)
99+
kwargs[ENV_VAR_KEY].update(env_vars)
100+
stdout, stderr, exit_code = utilities.execute_expect(self.connection,
101+
cmd, **kwargs)
97102
if re.search(r"ERROR", str(stdout)):
98103
logger.error("Cluster modification failed, killing the execution")
99104
raise Exception(stdout)

‎src/controller/couchbase_lib/_xdcr.py‎

Lines changed: 29 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -40,12 +40,20 @@ def generate_environment_map(self):
4040
return env
4141

4242
def xdcr_delete(self, cluster_name):
43-
logger.debug("XDCR deletion for cluster_name {} has started ".format(cluster_name))
44-
kwargs = {ENV_VAR_KEY: {'source_password': self.parameters.xdcr_admin_password,
45-
'password': self.parameters.couchbase_admin_password}}
43+
logger.debug("XDCR deletion for cluster_name {} has started ".format(
44+
cluster_name))
45+
kwargs = {ENV_VAR_KEY: {
46+
'source_password': self.parameters.xdcr_admin_password,
47+
'password': self.parameters.couchbase_admin_password}}
4648
env = _XDCrMixin.generate_environment_map(self)
47-
cmd = CommandFactory.xdcr_delete(cluster_name=cluster_name, **env)
48-
stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd, **kwargs)
49+
env.update(kwargs[ENV_VAR_KEY])
50+
# cmd = CommandFactory.xdcr_delete(cluster_name=cluster_name, **env)
51+
cmd, env_vars = CommandFactory.xdcr_delete_expect(
52+
cluster_name=cluster_name, **env)
53+
kwargs[ENV_VAR_KEY].update(env_vars)
54+
# stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd, **kwargs)
55+
stdout, stderr, exit_code = utilities.execute_expect(self.connection,
56+
cmd, **kwargs)
4957
if exit_code != 0:
5058
logger.error("XDCR Setup deletion failed")
5159
if stdout:
@@ -61,18 +69,29 @@ def xdcr_setup(self):
6169
kwargs = {ENV_VAR_KEY: {'source_password': self.parameters.xdcr_admin_password,
6270
'password': self.parameters.couchbase_admin_password}}
6371
env = _XDCrMixin.generate_environment_map(self)
64-
cmd = CommandFactory.xdcr_setup(cluster_name=self.parameters.stg_cluster_name, **env)
65-
stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd, **kwargs)
72+
env.update(kwargs[ENV_VAR_KEY])
73+
# cmd = CommandFactory.xdcr_setup(cluster_name=self.parameters.stg_cluster_name, **env)
74+
cmd, env_vars = CommandFactory.xdcr_setup_expect(cluster_name=self.parameters.stg_cluster_name, **env)
75+
kwargs[ENV_VAR_KEY].update(env_vars)
76+
stdout, stderr, exit_code = utilities.execute_expect(self.connection, cmd, **kwargs)
6677
helper_lib.sleepForSecond(3)
6778

6879
def xdcr_replicate(self, src, tgt):
6980
try:
7081
logger.debug("Started XDCR replication for bucket {}".format(src))
7182
kwargs = {ENV_VAR_KEY: {'source_password': self.parameters.xdcr_admin_password}}
7283
env = _XDCrMixin.generate_environment_map(self)
73-
cmd = CommandFactory.xdcr_replicate(source_bucket_name=src, target_bucket_name=tgt,
74-
cluster_name=self.parameters.stg_cluster_name, **env)
75-
stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd, **kwargs)
84+
env.update(kwargs[ENV_VAR_KEY])
85+
# cmd = CommandFactory.xdcr_replicate(source_bucket_name=src, target_bucket_name=tgt,
86+
# cluster_name=self.parameters.stg_cluster_name, **env)
87+
cmd, env_vars = CommandFactory.xdcr_replicate_expect(
88+
source_bucket_name=src,
89+
target_bucket_name=tgt,
90+
cluster_name=self.parameters.stg_cluster_name,
91+
**env
92+
)
93+
kwargs[ENV_VAR_KEY].update(env_vars)
94+
stdout, stderr, exit_code = utilities.execute_expect(self.connection, cmd, **kwargs)
7695
if exit_code != 0:
7796
logger.debug("XDCR replication create failed")
7897
raise Exception(stdout)

‎src/controller/couchbase_operation.py‎

Lines changed: 40 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -73,14 +73,14 @@ def uid(self):
7373
def gid(self):
7474
return self.__gid
7575

76-
7776
def run_couchbase_command(self, couchbase_command, **kwargs):
7877
logger.debug('run_couchbase_command')
7978
logger.debug('couchbase_command: {}'.format(couchbase_command))
8079
if "password" in kwargs:
81-
password = kwargs.pop('password')
80+
password = kwargs.get('password')
8281
else:
8382
password = self.parameters.couchbase_admin_password
83+
kwargs["password"] = password
8484

8585
if "username" in kwargs:
8686
username = kwargs.pop('username')
@@ -99,14 +99,27 @@ def run_couchbase_command(self, couchbase_command, **kwargs):
9999
env["newpass"] = kwargs.pop('newpass')
100100

101101
if "source_password" in kwargs:
102-
env["source_password"] = kwargs.pop('source_password')
102+
env["source_password"] = kwargs.get('source_password')
103103

104104
autoparams = [ "shell_path", "install_path", "username", "port", "sudo", "uid", "hostname"]
105105

106106
new_kwargs = {k: v for k, v in kwargs.items() if k not in autoparams}
107-
108-
method_to_call = getattr(CommandFactory, couchbase_command)
109-
command = method_to_call(shell_path=self.repository.cb_shell_path,
107+
if couchbase_command not in ["get_server_list",
108+
"couchbase_server_info",
109+
"cb_backup_full",
110+
"build_index",
111+
"check_index_build",
112+
"get_source_bucket_list",
113+
"get_replication_uuid",
114+
"get_stream_id",
115+
"delete_replication",
116+
"node_init",
117+
"get_indexes_name",
118+
"rename_cluster",
119+
"server_add",
120+
"rebalance"]:
121+
method_to_call = getattr(CommandFactory, couchbase_command)
122+
command = method_to_call(shell_path=self.repository.cb_shell_path,
110123
install_path=self.repository.cb_install_path,
111124
username=username,
112125
port=self.parameters.couchbase_port,
@@ -115,8 +128,26 @@ def run_couchbase_command(self, couchbase_command, **kwargs):
115128
hostname=hostname,
116129
**new_kwargs)
117130

118-
logger.debug("couchbase command to run: {}".format(command))
119-
stdout, stderr, exit_code = utilities.execute_bash(self.connection, command, environment_vars=env)
131+
logger.debug("couchbase command to run: {}".format(command))
132+
stdout, stderr, exit_code = utilities.execute_bash(self.connection, command, environment_vars=env)
133+
else:
134+
couchbase_command = couchbase_command+"_expect"
135+
logger.debug('new_couchbase_command: {}'.format(couchbase_command))
136+
method_to_call = getattr(CommandFactory, couchbase_command)
137+
command, env_vars = method_to_call(shell_path=self.repository.cb_shell_path,
138+
install_path=self.repository.cb_install_path,
139+
username=username,
140+
port=self.parameters.couchbase_port,
141+
sudo=self.need_sudo,
142+
uid=self.uid,
143+
hostname=hostname,
144+
**new_kwargs
145+
)
146+
env.update(env_vars)
147+
logger.debug("couchbase command to run: {}".format(command))
148+
stdout, stderr, exit_code = utilities.execute_expect(self.connection,
149+
command,
150+
environment_vars=env)
120151
return [stdout, stderr, exit_code]
121152

122153

@@ -337,7 +368,7 @@ def status(self, provision=False):
337368

338369
except Exception as error:
339370
# TODO
340-
# rewrite it
371+
# rewrite it
341372
logger.debug("Exception: {}".format(str(error)))
342373
if re.search("Unable to connect to host at", str(error)):
343374
logger.debug("Couchbase service is not running")

0 commit comments

Comments
 (0)