1import copy
2import json
3import os
4import random
5import string, re
6import time
7from threading import Thread
8
9from TestInput import TestInputSingleton
10from clitest.cli_base import CliBaseTest
11from couchbase_cli import CouchbaseCLI
12from upgrade.newupgradebasetest import NewUpgradeBaseTest
13from security.rbacmain import rbacmain
14from security.rbac_base import RbacBase
15from membase.api.rest_client import RestConnection
16from memcached.helper.data_helper import MemcachedClientHelper
17from remote.remote_util import RemoteMachineShellConnection
18from testconstants import CLI_COMMANDS, COUCHBASE_FROM_SPOCK, \
19                          COUCHBASE_FROM_WATSON, COUCHBASE_FROM_SHERLOCK,\
20                          COUCHBASE_FROM_4DOT6
21from couchbase_helper.documentgenerator import BlobGenerator
22
23
24help = {'CLUSTER': '--cluster=HOST[:PORT] or -c HOST[:PORT]',
25 'COMMAND': {'bucket-compact': 'compact database and index data',
26             'bucket-create': 'add a new bucket to the cluster',
27             'bucket-delete': 'delete an existing bucket',
28             'bucket-edit': 'modify an existing bucket',
29             'bucket-flush': 'flush all data from disk for a given bucket',
30             'bucket-list': 'list all buckets in a cluster',
31             'cluster-edit': 'modify cluster settings',
32             'cluster-init': 'set the username,password and port of the cluster',
33             'failover': 'failover one or more servers',
34             'group-manage': 'manage server groups',
35             'help': 'show longer usage/help and examples',
36             'node-init': 'set node specific parameters',
37             'rebalance': 'start a cluster rebalancing',
38             'rebalance-status': 'show status of current cluster rebalancing',
39             'rebalance-stop': 'stop current cluster rebalancing',
40             'recovery': 'recover one or more servers',
41             'server-add': 'add one or more servers to the cluster',
42             'server-info': 'show details on one server',
43             'server-list': 'list all servers in a cluster',
44             'server-readd': 'readd a server that was failed over',
45             'setting-alert': 'set email alert settings',
46             'setting-autofailover': 'set auto failover settings',
47             'setting-compaction': 'set auto compaction settings',
48             'setting-notification': 'set notification settings',
49             'setting-xdcr': 'set xdcr related settings',
50             'ssl-manage': 'manage cluster certificate',
51             'user-manage': 'manage read only user',
52             'xdcr-replicate': 'xdcr operations',
53             'xdcr-setup': 'set up XDCR connection'},
54 'EXAMPLES': {'Add a node to a cluster and rebalance': 'couchbase-cli rebalance -c 192.168.0.1:8091 --server-add=192.168.0.2:8091 --server-add-username=Administrator1 --server-add-password=password1 --group-name=group1 -u Administrator -p password',
55              'Add a node to a cluster, but do not rebalance': 'couchbase-cli server-add -c 192.168.0.1:8091 --server-add=192.168.0.2:8091 --server-add-username=Administrator1 --server-add-password=password1 --group-name=group1 -u Administrator -p password',
56              'Change the data path': 'couchbase-cli node-init -c 192.168.0.1:8091 --node-init-data-path=/tmp -u Administrator -p password',
57              'Create a couchbase bucket and wait for bucket ready': 'couchbase-cli bucket-create -c 192.168.0.1:8091 --bucket=test_bucket --bucket-type=couchbase --bucket-port=11222 --bucket-ramsize=200 --bucket-replica=1 --bucket-priority=low --wait -u Administrator -p password',
58              'Create a new dedicated port couchbase bucket': 'couchbase-cli bucket-create -c 192.168.0.1:8091 --bucket=test_bucket --bucket-type=couchbase --bucket-port=11222 --bucket-ramsize=200 --bucket-replica=1 --bucket-priority=high -u Administrator -p password',
59              'Create a new sasl memcached bucket': 'couchbase-cli bucket-create -c 192.168.0.1:8091 --bucket=test_bucket --bucket-type=memcached --bucket-password=password --bucket-ramsize=200 --bucket-eviction-policy=valueOnly --enable-flush=1 -u Administrator -p password',
60              'Delete a bucket': 'couchbase-cli bucket-delete -c 192.168.0.1:8091 --bucket=test_bucket',
61              'Flush a bucket': 'couchbase-cli xdcr-replicate -c 192.168.0.1:8091 --list -u Administrator -p password',
62              'List buckets in a cluster': 'couchbase-cli bucket-list -c 192.168.0.1:8091',
63              'List read only user in a cluster': 'couchbase-cli ssl-manage -c 192.168.0.1:8091 --regenerate-cert=/tmp/test.pem -u Administrator -p password',
64              'List servers in a cluster': 'couchbase-cli server-list -c 192.168.0.1:8091',
65              'Modify a dedicated port bucket': 'couchbase-cli bucket-edit -c 192.168.0.1:8091 --bucket=test_bucket --bucket-port=11222 --bucket-ramsize=400 --bucket-eviction-policy=fullEviction --enable-flush=1 --bucket-priority=high -u Administrator -p password',
66              'Remove a node from a cluster and rebalance': 'couchbase-cli rebalance -c 192.168.0.1:8091 --server-remove=192.168.0.2:8091 -u Administrator -p password',
67              'Remove and add nodes from/to a cluster and rebalance': 'couchbase-cli rebalance -c 192.168.0.1:8091 --server-remove=192.168.0.2 --server-add=192.168.0.4 --server-add-username=Administrator1 --server-add-password=password1 --group-name=group1 -u Administrator -p password',
68              'Server information': 'couchbase-cli server-info -c 192.168.0.1:8091',
69              'Set data path and hostname for an unprovisioned cluster': 'couchbse-cli node-init -c 192.168.0.1:8091 --node-init-data-path=/tmp/data --node-init-index-path=/tmp/index --node-init-hostname=myhostname -u Administrator -p password',
70              'Set recovery type to a server': 'couchbase-cli rebalance -c 192.168.0.1:8091 --recovery-buckets="default,bucket1" -u Administrator -p password',
71              'Set the username, password, port and ram quota': 'couchbase-cli cluster-init -c 192.168.0.1:8091 --cluster-username=Administrator --cluster-password=password --cluster-port=8080 --cluster-ramsize=300',
72              'Stop the current rebalancing': 'couchbase-cli rebalance-stop -c 192.168.0.1:8091 -u Administrator -p password',
73              'change the cluster username, password, port and ram quota': 'couchbase-cli cluster-edit -c 192.168.0.1:8091 --cluster-username=Administrator1 --cluster-password=password1 --cluster-port=8080 --cluster-ramsize=300 -u Administrator -p password'},
74 'OPTIONS': {'-o KIND, --output=KIND': 'KIND is json or standard\n-d, --debug',
75             '-p PASSWORD, --password=PASSWORD': 'admin password of the cluster',
76             '-u USERNAME, --user=USERNAME': 'admin username of the cluster'},
77 'bucket-* OPTIONS': {'--bucket-eviction-policy=[valueOnly|fullEviction]': ' policy how to retain meta in memory',
78                      '--bucket-password=PASSWORD': 'standard port, exclusive with bucket-port',
79                      '--bucket-port=PORT': 'supports ASCII protocol and is auth-less',
80                      '--bucket-priority=[low|high]': 'priority when compared to other buckets',
81                      '--bucket-ramsize=RAMSIZEMB': 'ram quota in MB',
82                      '--bucket-replica=COUNT': 'replication count',
83                      '--bucket-type=TYPE': 'memcached or couchbase',
84                      '--bucket=BUCKETNAME': ' bucket to act on',
85                      '--data-only': ' compact datbase data only',
86                      '--enable-flush=[0|1]': 'enable/disable flush',
87                      '--enable-index-replica=[0|1]': 'enable/disable index replicas',
88                      '--force': ' force to execute command without asking for confirmation',
89                      '--view-only': ' compact view data only',
90                      '--wait': 'wait for bucket create to be complete before returning'},
91 'cluster-* OPTIONS': {'--cluster-password=PASSWORD': ' new admin password',
92                       '--cluster-port=PORT': ' new cluster REST/http port',
93                       '--cluster-ramsize=RAMSIZEMB': ' per node ram quota in MB',
94                       '--cluster-username=USER': ' new admin username'},
95 'description': 'couchbase-cli - command-line cluster administration tool',
96 'failover OPTIONS': {'--force': ' failover node from cluster right away',
97                      '--server-failover=HOST[:PORT]': ' server to failover'},
98 'group-manage OPTIONS': {'--add-servers=HOST[:PORT];HOST[:PORT]': 'add a list of servers to group\n--move-servers=HOST[:PORT];HOST[:PORT] move a list of servers from group',
99                          '--create': 'create a new group',
100                          '--delete': 'delete an empty group',
101                          '--from-group=GROUPNAME': 'group name that to move servers from',
102                          '--group-name=GROUPNAME': 'group name',
103                          '--list': 'show group/server relationship map',
104                          '--rename=NEWGROUPNAME': ' rename group to new name',
105                          '--to-group=GROUPNAME': 'group name tat to move servers to'},
106 'node-init OPTIONS': {'--node-init-data-path=PATH': 'per node path to store data',
107                       '--node-init-hostname=NAME': ' hostname for the node. Default is 127.0.0.1',
108                       '--node-init-index-path=PATH': ' per node path to store index'},
109 'rebalance OPTIONS': {'--recovery-buckets=BUCKETS': 'comma separated list of bucket name. Default is for all buckets.',
110                       '--server-add*': ' see server-add OPTIONS',
111                       '--server-remove=HOST[:PORT]': ' the server to be removed'},
112 'recovery OPTIONS': {'--recovery-type=TYPE[delta|full]': 'type of recovery to be performed for a node',
113                      '--server-recovery=HOST[:PORT]': ' server to recover'},
114 'server-add OPTIONS': {'--group-name=GROUPNAME': 'group that server belongs',
115                        '--server-add-password=PASSWORD': 'admin password for the\nserver to be added',
116                        '--server-add-username=USERNAME': 'admin username for the\nserver to be added',
117                        '--server-add=HOST[:PORT]': 'server to be added'},
118 'server-readd OPTIONS': {'--group-name=GROUPNAME': 'group that server belongs',
119                          '--server-add-password=PASSWORD': 'admin password for the\nserver to be added',
120                          '--server-add-username=USERNAME': 'admin username for the\nserver to be added',
121                          '--server-add=HOST[:PORT]': 'server to be added'},
122 'setting-alert OPTIONS': {'--alert-auto-failover-cluster-small': " node wasn't auto fail over as cluster was too small",
123                           '--alert-auto-failover-max-reached': ' maximum number of auto failover nodes was reached',
124                           '--alert-auto-failover-node': 'node was auto failover',
125                           '--alert-auto-failover-node-down': " node wasn't auto failover as other nodes are down at the same time",
126                           '--alert-disk-space': 'disk space used for persistent storgage has reached at least 90% capacity',
127                           '--alert-ip-changed': 'node ip address has changed unexpectedly',
128                           '--alert-meta-oom': 'bucket memory on a node is entirely used for metadata',
129                           '--alert-meta-overhead': ' metadata overhead is more than 50%',
130                           '--alert-write-failed': 'writing data to disk for a specific bucket has failed',
131                           '--email-host=HOST': ' email server host',
132                           '--email-password=PWD': 'email server password',
133                           '--email-port=PORT': ' email server port',
134                           '--email-recipients=RECIPIENT': 'email recipents, separate addresses with , or ;',
135                           '--email-sender=SENDER': ' sender email address',
136                           '--email-user=USER': ' email server username',
137                           '--enable-email-alert=[0|1]': 'allow email alert',
138                           '--enable-email-encrypt=[0|1]': 'email encrypt'},
139 'setting-autofailover OPTIONS': {'--auto-failover-timeout=TIMEOUT (>=30)': 'specify timeout that expires to trigger auto failover',
140                                  '--enable-auto-failover=[0|1]': 'allow auto failover'},
141 'setting-compaction OPTIONS': {'--compaction-db-percentage=PERCENTAGE': ' at which point database compaction is triggered',
142                                '--compaction-db-size=SIZE[MB]': ' at which point database compaction is triggered',
143                                '--compaction-period-from=HH:MM': 'allow compaction time period from',
144                                '--compaction-period-to=HH:MM': 'allow compaction time period to',
145                                '--compaction-view-percentage=PERCENTAGE': ' at which point view compaction is triggered',
146                                '--compaction-view-size=SIZE[MB]': ' at which point view compaction is triggered',
147                                '--enable-compaction-abort=[0|1]': ' allow compaction abort when time expires',
148                                '--enable-compaction-parallel=[0|1]': 'allow parallel compaction for database and view',
149                                '--metadata-purge-interval=DAYS': 'how frequently a node will purge metadata on deleted items'},
150 'setting-notification OPTIONS': {'--enable-notification=[0|1]': ' allow notification'},
151 'setting-xdcr OPTIONS': {'--checkpoint-interval=[1800]': ' intervals between checkpoints, 60 to 14400 seconds.',
152                          '--doc-batch-size=[2048]KB': 'document batching size, 10 to 100000 KB',
153                          '--failure-restart-interval=[30]': 'interval for restarting failed xdcr, 1 to 300 seconds\n--optimistic-replication-threshold=[256] document body size threshold (bytes) to trigger optimistic replication',
154                          '--max-concurrent-reps=[32]': ' maximum concurrent replications per bucket, 8 to 256.',
155                          '--worker-batch-size=[500]': 'doc batch size, 500 to 10000.'},
156 'ssl-manage OPTIONS': {'--regenerate-cert=CERTIFICATE': 'regenerate cluster certificate AND save to a pem file',
157                        '--retrieve-cert=CERTIFICATE': 'retrieve cluster certificate AND save to a pem file'},
158 'usage': ' couchbase-cli COMMAND CLUSTER [OPTIONS]',
159 'user-manage OPTIONS': {'--delete': ' delete read only user',
160                         '--list': ' list any read only user',
161                         '--ro-password=PASSWORD': ' readonly user password',
162                         '--ro-username=USERNAME': ' readonly user name',
163                         '--set': 'create/modify a read only user'},
164 'xdcr-replicate OPTIONS': {'--checkpoint-interval=[1800]': ' intervals between checkpoints, 60 to 14400 seconds.',
165                            '--create': ' create and start a new replication',
166                            '--delete': ' stop and cancel a replication',
167                            '--doc-batch-size=[2048]KB': 'document batching size, 10 to 100000 KB',
168                            '--failure-restart-interval=[30]': 'interval for restarting failed xdcr, 1 to 300 seconds\n--optimistic-replication-threshold=[256] document body size threshold (bytes) to trigger optimistic replication',
169                            '--list': ' list all xdcr replications',
170                            '--max-concurrent-reps=[32]': ' maximum concurrent replications per bucket, 8 to 256.',
171                            '--pause': 'pause the replication',
172                            '--resume': ' resume the replication',
173                            '--settings': ' update settings for the replication',
174                            '--worker-batch-size=[500]': 'doc batch size, 500 to 10000.',
175                            '--xdcr-clucter-name=CLUSTERNAME': 'remote cluster to replicate to',
176                            '--xdcr-from-bucket=BUCKET': 'local bucket name to replicate from',
177                            '--xdcr-replication-mode=[xmem|capi]': 'replication protocol, either capi or xmem.',
178                            '--xdcr-replicator=REPLICATOR': ' replication id',
179                            '--xdcr-to-bucket=BUCKETNAME': 'remote bucket to replicate to',
180                            '--source-nozzle-per-node=[1-10]': 'the number of source nozzles per source node',
181                            '--target-nozzle-per-node=[1-100]': 'the number of outgoing nozzles per target node'},
182 'xdcr-setup OPTIONS': {'--create': ' create a new xdcr configuration',
183                        '--delete': ' delete existed xdcr configuration',
184                        '--edit': ' modify existed xdcr configuration',
185                        '--list': ' list all xdcr configurations',
186                        '--xdcr-certificate=CERTIFICATE': ' pem-encoded certificate. Need be present if xdcr-demand-encryption is true',
187                        '--xdcr-cluster-name=CLUSTERNAME': 'cluster name',
188                        '--xdcr-demand-encryption=[0|1]': ' allow data encrypted using ssl',
189                        '--xdcr-hostname=HOSTNAME': ' remote host name to connect to',
190                        '--xdcr-password=PASSWORD': ' remote cluster admin password',
191                        '--xdcr-username=USERNAME': ' remote cluster admin username'}}
192
193""" in 3.0, we add 'recovery  recover one or more servers' into couchbase-cli """
194help_short = {'COMMANDs include': {'bucket-compact': 'compact database and index data',
195                      'bucket-create': 'add a new bucket to the cluster',
196                      'bucket-delete': 'delete an existing bucket',
197                      'bucket-edit': 'modify an existing bucket',
198                      'bucket-flush': 'flush all data from disk for a given bucket',
199                      'bucket-list': 'list all buckets in a cluster',
200                      'cluster-edit': 'modify cluster settings',
201                      'cluster-init': 'set the username,password and port of the cluster',
202                      'recovery': 'recover one or more servers',
203                      'failover': 'failover one or more servers',
204                      'group-manage': 'manage server groups',
205                      'help': 'show longer usage/help and examples',
206                      'node-init': 'set node specific parameters',
207                      'rebalance': 'start a cluster rebalancing',
208                      'rebalance-status': 'show status of current cluster rebalancing',
209                      'rebalance-stop': 'stop current cluster rebalancing',
210                      'server-add': 'add one or more servers to the cluster',
211                      'server-info': 'show details on one server',
212                      'server-list': 'list all servers in a cluster',
213                      'server-readd': 'readd a server that was failed over',
214                      'setting-alert': 'set email alert settings',
215                      'setting-autofailover': 'set auto failover settings',
216                      'setting-compaction': 'set auto compaction settings',
217                      'setting-notification': 'set notification settings',
218                      'setting-xdcr': 'set xdcr related settings',
219                      'ssl-manage': 'manage cluster certificate',
220                      'user-manage': 'manage read only user',
221                      'xdcr-replicate': 'xdcr operations',
222                      'xdcr-setup': 'set up XDCR connection'},
223 'description': 'CLUSTER is --cluster=HOST[:PORT] or -c HOST[:PORT]',
224 'usage': ' couchbase-cli COMMAND CLUSTER [OPTIONS]'}
225
226
227class CouchbaseCliTest(CliBaseTest, NewUpgradeBaseTest):
228    REBALANCE_RUNNING = "Rebalance is running"
229    REBALANCE_NOT_RUN = "Rebalance is not running"
230
231    def setUp(self):
232        TestInputSingleton.input.test_params["default_bucket"] = False
233        super(CouchbaseCliTest, self).setUp()
234
235
236    def tearDown(self):
237        super(CouchbaseCliTest, self).tearDown()
238
239    def _check_output(self, word_check, output):
240        found = False
241        if len(output) >=1 :
242            for x in output:
243                if word_check in x:
244                    self.log.info("Found \"%s\" in CLI output" % word_check)
245                    found = True
246        return found
247
248    def _get_dict_from_output(self, output):
249        result = {}
250        if output[0].startswith("couchbase-cli"):
251            result["description"] = output[0]
252            result["usage"] = output[2].split("usage:")[1]
253        else:
254            result["usage"] = output[0].split("usage:")[1]
255            result["description"] = output[2]
256
257        upper_key = ""
258        for line in output[3:]:
259            line = line.strip()
260            if line == "":
261                if not upper_key == "EXAMPLES":
262                    upper_key = ""
263                continue
264            # line = ""
265            if line.endswith(":") and upper_key != "EXAMPLES":
266                upper_key = line[:-1]
267                result[upper_key] = {}
268                continue
269            elif line == "COMMANDs include":
270                upper_key = line
271                result[upper_key] = {}
272                continue
273            elif upper_key in ["COMMAND", "COMMANDs include"] :
274                result[upper_key][line.split()[0]] = line.split(line.split()[0])[1].strip()
275                if len(line.split(line.split()[0])) > 2:
276                    if line.split()[0] == 'help':
277                        result[upper_key][line.split()[0]] = (line.split(line.split()[0])[1] + 'help' + line.split(line.split()[0])[-1]).strip()
278                    else:
279                        result[upper_key][line.split()[0]] = line.split(line.split(line.split()[0])[1])[-1].strip()
280            elif upper_key in ["CLUSTER"]:
281                result[upper_key] = line
282            elif upper_key.endswith("OPTIONS"):
283                # for u=instance:"   -u USERNAME, --user=USERNAME      admin username of the cluster"
284                temp = line.split("  ")
285                temp = [item for item in temp if item != ""]
286                if len(temp) > 1:
287                    result[upper_key][temp[0]] = temp[1]
288                    previous_key = temp[0]
289                else:
290                    result[upper_key][previous_key] = result[upper_key][previous_key] + "\n" + temp[0]
291            elif upper_key in ["EXAMPLES"] :
292                if line.endswith(":"):
293                    previous_key = line[:-1]
294                    result[upper_key][previous_key] = ""
295                else:
296                    line = line.strip()
297                    if line.startswith("couchbase-cli"):
298                        result[upper_key][previous_key] = line.replace("\\", "")
299                    else:
300                        result[upper_key][previous_key] = (result[upper_key][previous_key] + line).replace("\\", "")
301                    previous_line = result[upper_key][previous_key]
302            elif line == "The default PORT number is 8091.":
303                continue
304            else:
305                self.fail(line)
306        return result
307
308    def _get_cluster_info(self, remote_client, cluster_host="localhost", cluster_port=None, user="Administrator", password="password"):
309        command = "server-info"
310        output, error = remote_client.execute_couchbase_cli(command, cluster_host=cluster_host, cluster_port=cluster_port, user=user, password=password)
311        if not error:
312            content = ""
313            for line in output:
314                content += line
315            return json.loads(content)
316        else:
317            self.fail("server-info return error output")
318
319    def _create_bucket(self, remote_client, bucket="default", bucket_type="couchbase",
320                        bucket_ramsize=200, bucket_replica=1, wait=False, enable_flush=None, enable_index_replica=None):
321        options = "--bucket={0} --bucket-type={1} --bucket-ramsize={2} --bucket-replica={3}".\
322            format(bucket, bucket_type, bucket_ramsize, bucket_replica)
323        options += (" --enable-flush={0}".format(enable_flush), "")[enable_flush is None]
324        options += (" --enable-index-replica={0}".format(enable_index_replica), "")[enable_index_replica is None]
325        options += (" --enable-flush={0}".format(enable_flush), "")[enable_flush is None]
326        options += (" --wait", "")[wait]
327        cli_command = "bucket-create"
328
329        output, error = remote_client.execute_couchbase_cli(cli_command=cli_command,
330                                          options=options, cluster_host="localhost",
331                                          user="Administrator", password="password")
332        if "TIMED OUT" in output[0]:
333            raise Exception("Timed out.  Could not create bucket")
334        else:
335            """ need to remove dot in front of output"""
336            self.assertTrue(self.cli_bucket_create_msg in output[0].lstrip("."), "Fail to create bucket")
337
338    def testHelp(self):
339        command_with_error = {}
340        shell = RemoteMachineShellConnection(self.master)
341        if self.cb_version[:5] in COUCHBASE_FROM_SPOCK:
342            self.log.info("skip moxi because it is removed in spock ")
343            for x in CLI_COMMANDS:
344                if x == "moxi":
345                    CLI_COMMANDS.remove("moxi")
346        for cli in CLI_COMMANDS:
347            """ excluded_commands should separate by ';' """
348            if self.excluded_commands is not None:
349                if ";" in self.excluded_commands:
350                    excluded_commands = self.excluded_commands.split(";")
351                else:
352                    excluded_commands = self.excluded_commands
353                if cli in excluded_commands:
354                    self.log.info("command {0} test will be skipped.".format(cli))
355                    continue
356            if self.os == "windows":
357                cli = '.'.join([cli, "exe"])
358            option = " -h"
359            if cli == "erl":
360                option = " -version"
361            command = ''.join([self.cli_command_path, cli, option])
362            self.log.info("test -h of command {0}".format(cli))
363            output, error = shell.execute_command(command, use_channel=True)
364
365            """ check if the first line is not empty """
366            if not output[0]:
367                self.log.error("this help command {0} may not work!".format(cli))
368            if error:
369                command_with_error[cli] = error[0]
370        if command_with_error:
371            raise Exception("some commands throw out error %s " % command_with_error)
372        shell.disconnect()
373
374    def testInfoCommands(self):
375        remote_client = RemoteMachineShellConnection(self.master)
376
377        cli_command = "server-list"
378        output, error = remote_client.execute_couchbase_cli(cli_command=cli_command,
379                  cluster_host="localhost", user="Administrator", password="password")
380        server_info = self._get_cluster_info(remote_client)
381        """ In new single node not join any cluster yet,
382            IP of node will be 127.0.0.1 """
383        if "127.0.0.1" in server_info["otpNode"]:
384            server_info["otpNode"] = "ns_1@{0}".format(remote_client.ip)
385            server_info["hostname"] = "{0}:8091".format(remote_client.ip)
386        otpNode = remote_client.ip
387        """ need to remove [ ] brackets in ipv6 raw ip address """
388        if "[" in otpNode:
389            otpNode = otpNode.replace("[", "").replace("]", "")
390        result = server_info["otpNode"] + " " + server_info["hostname"] + " " \
391               + server_info["status"] + " " + server_info["clusterMembership"]
392        self.assertEqual(result.lower(), "ns_1@{0} {1}:8091 healthy active" \
393                                           .format(otpNode.lower(),
394                                                   remote_client.ip.lower()))
395
396        cli_command = "bucket-list"
397        output, error = remote_client.execute_couchbase_cli(cli_command=cli_command,
398                  cluster_host="localhost", user="Administrator", password="password")
399        self.assertEqual([], output)
400        remote_client.disconnect()
401
402    def test_priority_start_couchbase_saslauth(self):
403        """
404            In centos 6.x, couchbase server needs to start after saslauth start
405            so the authentication works correctly as mention in ticket MB-25922
406            This test requires saslauth preinstalled on vm.
407        """
408        if "centos 7" in self.os_version:
409            self.log.info("This test only for centos 6.x")
410            return
411        if "windows" in self.os_version:
412            self.log.info("This test is for centos 6.x only, not for windows")
413            return
414        if "centos" in self.os_version:
415            """ If saslauth did not install on vm, mark test as failed. """
416            output, error = self.shell.execute_command("/etc/init.d/saslauthd status")
417            if output and "is running" not in output[0]:
418                self.fail("Need a centos 6.x with saslauth preinstalled.")
419
420            self.shell.execute_command("reboot")
421            self.sleep(240, "sleep while rebooting")
422            shell = RemoteMachineShellConnection(self.master)
423            shell.disable_firewall()
424            output, error = shell.execute_command('find /etc/rc3.d/ '\
425                                                  '| egrep "saslauthd|couchbase"')
426            if output:
427                self.log.info("Order starting: %s " % output)
428                if "saslauthd" in output[0] and "couchbase-serve" in output[1]:
429                    self.log.info("Couchbase Server started after saslauthd")
430                elif "couchbase-server" in output[0]:
431                    self.fail("Couchbase Server started before saslauthd")
432            output, error = shell.execute_command('cat /var/log/boot.log '\
433                                                  '| egrep "saslauthd|couchbase"')
434            if output:
435                self.log.info("Order starting: %s " % output)
436                if "Starting saslauth" in output[0] and "couchbase-server" in output[-1]:
437                    self.log.info("Couchbase Server started after saslauthd")
438                elif "couchbase-server" in output[0]:
439                    self.fail("Couchbase Server started before saslauthd")
440        else:
441            self.log.info("This test only for centos 6.x")
442            return
443
444    def testAddRemoveNodes(self):
445        nodes_add = self.input.param("nodes_add", 1)
446        nodes_rem = self.input.param("nodes_rem", 1)
447        nodes_failover = self.input.param("nodes_failover", 0)
448        nodes_readd = self.input.param("nodes_readd", 0)
449        remote_client = RemoteMachineShellConnection(self.master)
450        cli_command = "server-add"
451        if int(nodes_add) < len(self.servers):
452            for num in xrange(nodes_add):
453                self.log.info("add node {0} to cluster".format(
454                    self.servers[num + 1].ip))
455                options = "--server-add={0}:8091 \
456                           --server-add-username=Administrator \
457                           --server-add-password=password" \
458                            .format(self.servers[num + 1].ip)
459                output, error = \
460                      remote_client.execute_couchbase_cli(cli_command=cli_command,
461                                        options=options, cluster_host="localhost",
462                                          cluster_port=8091, user="Administrator",
463                                                              password="password")
464                server_added = False
465                output_msg = "SUCCESS: Server added"
466                if self.cb_version[:3] == "4.6":
467                    output_msg = "Server %s:%s added" % (self.servers[num + 1].ip,\
468                                                         self.servers[num + 1].port)
469                if len(output) >= 1:
470                    for x in output:
471                        if output_msg in x:
472                            server_added = True
473                            break
474                    if not server_added:
475                        raise Exception("failed to add server {0}"
476                                          .format(self.servers[num + 1].ip))
477        else:
478             raise Exception("Node add should be smaller total number"
479                                                         " vms in ini file")
480
481        cli_command = "rebalance"
482        for num in xrange(nodes_rem):
483            options = "--server-remove={0}:8091".format(self.servers[nodes_add - num].ip)
484            output, error = remote_client.execute_couchbase_cli(cli_command=cli_command, \
485                            options=options, cluster_host="localhost", cluster_port=8091,\
486                            user="Administrator", password="password")
487            self.assertTrue(self.cli_rebalance_msg in output)
488        if nodes_rem == 0 and nodes_add > 0:
489            cli_command = "rebalance"
490            output, error = remote_client.execute_couchbase_cli(cli_command=cli_command,
491                                            cluster_host="localhost", cluster_port=8091,
492                                              user="Administrator", password="password")
493            if len(output) == 4:
494                self.assertEqual(output, ["INFO: rebalancing ", "", "SUCCESS: rebalanced cluster", ""])
495            else:
496                self.assertTrue(self.cli_rebalance_msg in output)
497
498        """ when no bucket, have to add option --force to failover
499            since no data => no graceful failover.  Need to add test
500            to detect no graceful failover if no bucket """
501        self._create_bucket(remote_client, bucket_replica=self.num_replicas)
502        cli_command = "failover"
503        for num in xrange(nodes_failover):
504            self.log.info("failover node {0}" \
505                          .format(self.servers[nodes_add - nodes_rem - num].ip))
506            options = "--server-failover={0}:8091" \
507                      .format(self.servers[nodes_add - nodes_rem - num].ip)
508            if self.force_failover:
509                options += " --force"
510                output, error = remote_client.execute_couchbase_cli(\
511                        cli_command=cli_command, options=options, \
512                        cluster_host="localhost", cluster_port=8091, \
513                        user="Administrator", password="password")
514                if len(output) == 2:
515                    self.assertEqual(output, ["SUCCESS: Server failed over",
516                                              ""])
517                else:
518                    self.assertTrue("SUCCESS: Server failed over" in output)
519            else:
520                output, error = remote_client.execute_couchbase_cli(\
521                        cli_command=cli_command, options=options, \
522                        cluster_host="localhost", cluster_port=8091, \
523                        user="Administrator", password="password")
524                self.assertTrue("SUCCESS: Server failed over" in output)
525
526        cli_command = "server-readd"
527        for num in xrange(nodes_readd):
528            self.log.info("add back node {0} to cluster" \
529                    .format(self.servers[nodes_add - nodes_rem - num ].ip))
530            options = "--server-add={0}:8091".format(self.servers[nodes_add - nodes_rem - num].ip)
531            output, error = remote_client.execute_couchbase_cli(cli_command=cli_command,
532                                                        options=options, cluster_host="localhost",
533                                                          cluster_port=8091, user="Administrator",
534                                                                              password="password")
535            self.assertTrue("DEPRECATED: Please use the recovery command "
536                            "instead" in output and "SUCCESS: Servers "
537                                                    "recovered" in output,
538                            "Server readd failed")
539        cli_command = "rebalance"
540        output, error = remote_client.execute_couchbase_cli(cli_command=cli_command,
541                                                            cluster_host="localhost", cluster_port=8091,
542                                                            user="Administrator", password="password")
543        self.assertTrue(self.cli_rebalance_msg in output)
544        remote_client.disconnect()
545
546    def testAddRemoveNodesWithRecovery(self):
547        nodes_add = self.input.param("nodes_add", 1)
548        nodes_rem = self.input.param("nodes_rem", 1)
549        nodes_failover = self.input.param("nodes_failover", 0)
550        nodes_recovery = self.input.param("nodes_recovery", 0)
551        nodes_readd = self.input.param("nodes_readd", 0)
552        remote_client = RemoteMachineShellConnection(self.master)
553        cli_command = "server-add"
554        if int(nodes_add) < len(self.servers):
555            for num in xrange(nodes_add):
556                self.log.info("add node {0} to cluster".format(self.servers[num + 1].ip))
557                options = "--server-add={0}:8091 --server-add-username=Administrator --server-add-password=password".format(self.servers[num + 1].ip)
558                output, error = remote_client.execute_couchbase_cli(cli_command=cli_command, options=options,
559                                                                    cluster_host="localhost", cluster_port=8091,
560                                                                    user="Administrator", password="password")
561                self.assertTrue("SUCCESS: Server added" in output)
562        else:
563             raise Exception("Node add should be smaller total number vms in ini file")
564
565        cli_command = "rebalance"
566        for num in xrange(nodes_rem):
567            options = "--server-remove={0}:8091".format(self.servers[nodes_add - num].ip)
568            output, error = remote_client.execute_couchbase_cli(cli_command=cli_command, options=options, cluster_host="localhost", user="Administrator", password="password")
569            self.assertTrue(self.cli_rebalance_msg in output)
570
571        if nodes_rem == 0 and nodes_add > 0:
572            cli_command = "rebalance"
573            output, error = remote_client.execute_couchbase_cli(cli_command=cli_command, cluster_host="localhost", user="Administrator", password="password")
574            self.assertTrue(self.cli_rebalance_msg in output)
575
576        self._create_bucket(remote_client)
577
578        cli_command = "failover"
579        for num in xrange(nodes_failover):
580            self.log.info("failover node {0}".format(self.servers[nodes_add - nodes_rem - num].ip))
581            options = "--server-failover={0}:8091".format(self.servers[nodes_add - nodes_rem - num].ip)
582            if self.force_failover or num == nodes_failover - 1:
583                options += " --force"
584            output, error = remote_client.execute_couchbase_cli(cli_command=cli_command, options=options, cluster_host="localhost", user="Administrator", password="password")
585            self.assertTrue("SUCCESS: Server failed over" in output)
586
587        cli_command = "recovery"
588        for num in xrange(nodes_failover):
589            # try to set recovery when nodes failovered (MB-11230)
590            options = "--server-recovery={0}:8091 --recovery-type=delta".format(self.servers[nodes_add - nodes_rem - num].ip)
591            output, error = remote_client.execute_couchbase_cli(cli_command=cli_command, options=options, cluster_host="localhost", user="Administrator", password="password")
592            self.assertTrue("SUCCESS: Servers recovered" in output)
593
594        for num in xrange(nodes_recovery):
595            cli_command = "server-readd"
596            self.log.info("add node {0} back to cluster"\
597                          .format(self.servers[nodes_add - nodes_rem - num].ip))
598            options = "--server-add={0}:8091".format(self.servers[nodes_add - nodes_rem - num].ip)
599            output, error = remote_client.execute_couchbase_cli(cli_command=cli_command,
600                                                                options=options,
601                                                                cluster_host="localhost",
602                                                                user="Administrator",
603                                                                password="password")
604            self.assertTrue("DEPRECATED: Please use the recovery command "
605                            "instead" in output and "SUCCESS: Servers "
606                                                    "recovered" in output,
607                            "Server readd failed")
608
609            cli_command = "recovery"
610            options = "--server-recovery={0}:8091 --recovery-type=delta"\
611                                    .format(self.servers[nodes_add - nodes_rem - num].ip)
612            output, error = remote_client.execute_couchbase_cli(cli_command=cli_command,
613                                                                options=options,
614                                                                cluster_host="localhost",
615                                                                user="Administrator",
616                                                                password="password")
617            self.assertTrue("SUCCESS: Servers recovered" in output)
618
619        cli_command = "server-readd"
620        for num in xrange(nodes_failover):
621            self.log.info("add back node {0} to cluster".format(self.servers[nodes_add - nodes_rem - num ].ip))
622            options = "--server-add={0}:8091".format(self.servers[nodes_add - nodes_rem - num ].ip)
623            output, error = remote_client.execute_couchbase_cli(cli_command=cli_command,
624                                                                options=options,
625                                                                cluster_host="localhost",
626                                                                user="Administrator",
627                                                                password="password")
628            self.assertTrue("DEPRECATED: Please use the recovery command "
629                            "instead" in output and "SUCCESS: Servers "
630                                                    "recovered" in output,
631                            "Server readd failed")
632
633        """ in spock, server-readd does not work to add a node not
634            in cluster back to cluster as server-add
635        """
636        if int(nodes_readd) > int(nodes_failover):
637            cli_command = "server-add"
638            for num in xrange(nodes_readd - nodes_failover):
639                self.log.info("add node {0} to cluster".format(self.servers[nodes_add - num ].ip))
640                options = "--server-add={0}:8091 --server-add-username=Administrator"\
641                                                    " --server-add-password=password"\
642                                               .format(self.servers[nodes_add -num].ip)
643                output, error = remote_client.execute_couchbase_cli(cli_command=cli_command,
644                                                                    options=options,
645                                                                    cluster_host="localhost",
646                                                                    cluster_port=8091,
647                                                                    user="Administrator",
648                                                                    password="password")
649                self.assertTrue("SUCCESS: Server added" in output)
650        cli_command = "rebalance"
651        output, error = remote_client.execute_couchbase_cli(cli_command=cli_command,
652                                                            cluster_host="localhost",
653                                                            user="Administrator",
654                                                            password="password")
655        self.assertTrue(self.cli_rebalance_msg in output)
656        remote_client.disconnect()
657
658    def testStartStopRebalance(self):
659        nodes_add = self.input.param("nodes_add", 1)
660        nodes_rem = self.input.param("nodes_rem", 1)
661        remote_client = RemoteMachineShellConnection(self.master)
662
663        cli_command = "rebalance-status"
664        output, error = remote_client.execute_couchbase_cli(cli_command=cli_command, \
665                  cluster_host="localhost", user="Administrator", password="password")
666        self.assertTrue(self._check_output(self.REBALANCE_NOT_RUN, output))
667
668        cli_command = "server-add"
669        for num in xrange(nodes_add):
670            options = "--server-add={0}:8091 --server-add-username=Administrator \
671                       --server-add-password=password".format(self.servers[num + 1].ip)
672            output, error = remote_client.execute_couchbase_cli(cli_command=cli_command, \
673                                              options=options, cluster_host="localhost", \
674                                                user="Administrator", password="password")
675            output_msg = "SUCCESS: Server added"
676            self.assertEqual(output[0], output_msg)
677
678        cli_command = "rebalance-status"
679        output, error = remote_client.execute_couchbase_cli(cli_command=cli_command, \
680                  cluster_host="localhost", user="Administrator", password="password")
681        self.assertTrue(self._check_output(self.REBALANCE_NOT_RUN, output))
682
683        self._create_bucket(remote_client)
684
685        cli_command = "rebalance"
686        t = Thread(target=remote_client.execute_couchbase_cli, name="rebalance_after_add",
687                       args=(cli_command, "localhost", '', None, "Administrator", "password"))
688        t.start()
689        self.sleep(5)
690
691        cli_command = "rebalance-status"
692        output, error = remote_client.execute_couchbase_cli(cli_command=cli_command, \
693                  cluster_host="localhost", user="Administrator", password="password")
694        self.assertTrue(self._check_output(self.REBALANCE_RUNNING, output))
695
696        t.join()
697
698        cli_command = "rebalance"
699        for num in xrange(nodes_rem):
700            options = "--server-remove={0}:8091".format(self.servers[nodes_add - num].ip)
701            t = Thread(target=remote_client.execute_couchbase_cli, name="rebalance_after_add",
702                       args=(cli_command, "localhost", options, None, "Administrator", "password"))
703            t.start()
704            self.sleep(5)
705            cli_command = "rebalance-status"
706            output, error = remote_client.execute_couchbase_cli(cli_command=cli_command, \
707                      cluster_host="localhost", user="Administrator", password="password")
708            self.assertTrue(self._check_output(self.REBALANCE_RUNNING, output))
709
710            cli_command = "rebalance-stop"
711            output, error = remote_client.execute_couchbase_cli(cli_command=cli_command, \
712                      cluster_host="localhost", user="Administrator", password="password")
713
714            t.join()
715
716            cli_command = "rebalance"
717            output, error = remote_client.execute_couchbase_cli(cli_command=cli_command, \
718                      cluster_host="localhost", user="Administrator", password="password")
719            self.assertTrue(self._check_output("SUCCESS: Rebalance complete", output))
720
721            cli_command = "rebalance-status"
722            output, error = remote_client.execute_couchbase_cli(cli_command=cli_command, \
723                      cluster_host="localhost", user="Administrator", password="password")
724            self.assertTrue(self._check_output(self.REBALANCE_NOT_RUN, output))
725        remote_client.disconnect()
726
727    def testClusterInit(self):
728        username = self.input.param("username", None)
729        password = self.input.param("password", None)
730        data_ramsize = self.input.param("data-ramsize", None)
731        index_ramsize = self.input.param("index-ramsize", None)
732        fts_ramsize = self.input.param("fts-ramsize", None)
733        name = self.input.param("name", None)
734        index_storage_mode = self.input.param("index-storage-mode", None)
735        port = self.input.param("port", None)
736        services = self.input.param("services", None)
737        initialized = self.input.param("initialized", False)
738        expect_error = self.input.param("expect-error")
739        error_msg = self.input.param("error-msg", "")
740
741        initial_server = self.servers[0]
742        server = copy.deepcopy(initial_server)
743
744        if not initialized:
745            rest = RestConnection(server)
746            rest.force_eject_node()
747
748        cli = CouchbaseCLI(server, username, password)
749        stdout, _, _ = cli.cluster_init(data_ramsize, index_ramsize, fts_ramsize, services, index_storage_mode, name,
750                                        username, password, port)
751
752        if username:
753            server.rest_username = username
754        if password:
755            server.rest_password = password
756        # strip quotes if the cluster name contains spaces
757        if name and (name[0] == name[-1]) and name.startswith(("'", '"')):
758            name = name[1:-1]
759        if not index_storage_mode and services and "index" in services:
760            index_storage_mode = "plasma"
761        if not services:
762            services = "data"
763
764        if not expect_error:
765            # Update the cluster manager port if it was specified to be changed
766            if port:
767                server.port = port
768
769            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, "Cluster initialized"),
770                            "Expected command to succeed")
771            self.assertTrue(self.isClusterInitialized(server), "Cluster was not initialized")
772            self.assertTrue(self.verifyServices(server, services), "Services do not match")
773            self.assertTrue(self.verifyNotificationsEnabled(server), "Notification not enabled")
774            self.assertTrue(self.verifyClusterName(server, name), "Cluster name does not match")
775
776            if "index" in services:
777                self.assertTrue(self.verifyIndexSettings(server, None, None, None, index_storage_mode, None, None),
778                                "Index storage mode not properly set")
779
780            self.assertTrue(self.verifyRamQuotas(server, data_ramsize, index_ramsize, fts_ramsize),
781                            "Ram quotas not set properly")
782        else:
783            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, error_msg), "Expected error message not found")
784            if not initialized:
785                self.assertTrue(not self.isClusterInitialized(server), "Cluster was initialized, but error was received")
786
787        # Reset the cluster (This is important for when we change the port number)
788        rest = RestConnection(server)
789        rest.init_cluster(initial_server.rest_username, initial_server.rest_password, initial_server.port)
790
791    def test_set_cluster_name(self):
792        """
793        Test set and rename cluster name
794        1. Reset node back to initial setup
795        2. Set cluster-name by IP (if param is None)
796        3. Rename cluster-name
797        :return: nothing
798        """
799        cluster_name = self.input.param("cluster-name", None)
800        if cluster_name is None:
801            cluster_name = self.master.ip
802        self.log.info("Reset node back to initial page")
803        rest = RestConnection(self.master)
804        rest.force_eject_node()
805        options = "--cluster-username Administrator --cluster-password password " \
806                  "--cluster-port 8091 --cluster-name '%s' " % cluster_name
807        output, error = self.shell.couchbase_cli("cluster-init", self.master.ip,
808                                                                        options)
809        if "SUCCESS: Cluster initialized" not in output[0]:
810            self.fail("Failed to initialize node '%s' " % self.master.ip)
811
812        self.log.info("Verify hostname is set in cluster")
813        settings = rest.get_pools_default()
814        if cluster_name not in settings["clusterName"]:
815            self.fail("Fail to set hostname in cluster. "
816                      "Host name in cluster is '%s' " % settings["clusterName"])
817        else:
818            self.log.info("Cluster name is set to '%s' " % settings["clusterName"])
819
820        change_hostname = self.input.param("change-hostname", None)
821        if change_hostname is not None:
822            if change_hostname == "ip":
823                change_hostname = self.master.ip
824
825            self.log.info("Rename hostname in cluster.")
826            options = "-u Administrator -p password --cluster-name '%s' " \
827                                                        % change_hostname
828            output, error = self.shell.couchbase_cli("setting-cluster",
829                                                     self.master.ip, options)
830            settings = rest.get_pools_default()
831            if change_hostname not in settings["clusterName"]:
832                self.fail("Fail to set new hostname in cluster. "
833                          "Host name in cluster is '%s' " % settings["clusterName"])
834            else:
835                self.log.info("Cluster name is set to '%s' " % settings["clusterName"])
836
837    def test_rebalance_display_bar(self):
838        """
839            Test display bar when rebalance and stop/start rebalance
840        """
841        server = copy.deepcopy(self.servers[0])
842        add_server = self.servers[1]
843        stop_rebalance = self.input.param("stop-rebalance", False)
844        rest = RestConnection(server)
845        self.default_bucket = True
846        self._bucket_creation()
847        cli = CouchbaseCLI(server, "cbadminbucket", "password")
848        output_add, error, msg = cli.server_add(add_server.ip, "Administrator",
849                                                "password", None, "data", None)
850        if "SUCCESS: Server added" not in output_add[0]:
851            self.fail("Could not add node %s to cluster" % add_server.ip)
852
853        reb_result = ""
854        reb_bar = ""
855        ansi_escape = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]')
856
857        if not stop_rebalance:
858            output, error, msg = cli.rebalance(None)
859        else:
860            sts = rest.rebalance(otpNodes=[node.id for node in rest.node_statuses()],
861                                ejectedNodes=[])
862            for bucket in self.buckets:
863                if len(bucket.vbuckets) > 128:
864                    self.sleep(3, "wait for rebalance to run")
865                    break
866            if sts:
867                _, _, stop_status = cli.rebalance_stop()
868                if stop_status:
869                    output, error, msg = cli.rebalance(None)
870                else:
871                    self.fail("Fail sto stop rebalance")
872            else:
873                print "Fail to rebalance using rest call"
874        if output:
875            reb_result = output[-1]
876            """ remove ANSI code """
877            reb_bar = ansi_escape.sub('', output[-2])
878            if "SUCCESS: Rebalance complete" not in reb_result:
879                self.fail("Rebalance failed")
880
881            if "100.0%" not in reb_bar:
882                self.fail("Rebalance failed.  It not reach 100%")
883            reb_bar = reb_bar.replace(" 100.0%", "").strip(" ")
884            print "rebalance bar: %s" % reb_bar
885            if " " in reb_bar:
886                self.fail("rebalance bar did not display correctly")
887        else:
888            self.fail("output is empty %s " % output)
889
890    def testRebalanceStop(self):
891        username = self.input.param("username", None)
892        password = self.input.param("password", None)
893        initialized = self.input.param("initialized", True)
894        expect_error = self.input.param("expect-error")
895        error_msg = self.input.param("error-msg", "")
896        init_start_rebalance = self.input.param("init-rebalance", False)
897
898        server = copy.deepcopy(self.servers[0])
899        add_server = self.servers[1]
900
901        rest = RestConnection(server)
902        rest.force_eject_node()
903
904        cli = CouchbaseCLI(server, username, password)
905        if initialized:
906            _, _, success = cli.cluster_init(256, None, None, None, None, None, server.rest_username,
907                                             server.rest_password, None)
908            self.assertTrue(success, "Cluster initialization failed during test setup")
909        if init_start_rebalance:
910           self.assertTrue(rest.rebalance(otpNodes=["%s:%s" % (add_server.ip, add_server.port)]),
911                           "Rebalance failed to start")
912
913        stdout, _, _ = cli.rebalance_stop()
914
915        if not expect_error:
916            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, "Rebalance stopped"),
917                            "Expected command to succeed")
918            if init_start_rebalance:
919                self.assertTrue(rest.isRebalanced(), "Rebalance does not appear to be stopped")
920
921        else:
922            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, error_msg),
923                            "Expected error message not found")
924            if not initialized:
925                self.assertTrue(not self.isClusterInitialized(server),
926                                "Cluster was initialized, but error was received")
927
928    def testSettingAudit(self):
929        username = self.input.param("username", None)
930        password = self.input.param("password", None)
931        enabled = self.input.param("enabled", None)
932        log_path = self.input.param("log-path", None)
933        rotate_interval = self.input.param("rotate-interval", None)
934        initialized = self.input.param("initialized", True)
935        expect_error = self.input.param("expect-error")
936        error_msg = self.input.param("error-msg", "")
937
938        server = copy.deepcopy(self.servers[0])
939
940        rest = RestConnection(server)
941        rest.force_eject_node()
942
943        if log_path is not None and log_path == "valid":
944                log_path = self.log_path
945
946        cli = CouchbaseCLI(server, username, password)
947        if initialized:
948            _, _, success = cli.cluster_init(256, None, None, None, None, None, server.rest_username,
949                                             server.rest_password, None)
950            self.assertTrue(success, "Cluster initialization failed during test setup")
951
952        stdout, _, _ = cli.setting_audit(enabled, log_path, rotate_interval)
953
954        if not expect_error:
955            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, "Audit settings modified"),
956                            "Expected command to succeed")
957            self.assertTrue(self.verifyAuditSettings(server, enabled, log_path, rotate_interval),
958                            "Audit settings were not set properly")
959
960        else:
961            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, error_msg),
962                            "Expected error message not found")
963            if not initialized:
964                self.assertTrue(not self.isClusterInitialized(server),
965                                "Cluster was initialized, but error was received")
966
967    def testSettingCompaction(self):
968        username = self.input.param("username", None)
969        password = self.input.param("password", None)
970
971        db_frag_perc = self.input.param("db-frag-perc", None)
972        db_frag_size = self.input.param("db-frag-size", None)
973        view_frag_perc = self.input.param("view-frag-perc", None)
974        view_frag_size = self.input.param("view-frag-size", None)
975        from_period = self.input.param("from-period", None)
976        to_period = self.input.param("to-period", None)
977        abort_outside = self.input.param("abort-outside", None)
978        parallel_compact = self.input.param("parallel-compact", None)
979        purgeInt = self.input.param("purge-interval", None)
980
981        initialized = self.input.param("initialized", True)
982        expect_error = self.input.param("expect-error")
983        error_msg = self.input.param("error-msg", "")
984
985        server = copy.deepcopy(self.servers[0])
986
987        rest = RestConnection(server)
988        rest.force_eject_node()
989
990        cli = CouchbaseCLI(server, username, password)
991        if initialized:
992            _, _, success = cli.cluster_init(256, None, None, None, None, None, server.rest_username,
993                                             server.rest_password, None)
994            self.assertTrue(success, "Cluster initialization failed during test setup")
995
996        stdout, _, errored = cli.setting_compaction(db_frag_perc, db_frag_size, view_frag_perc, view_frag_size,
997                                                    from_period, to_period, abort_outside, parallel_compact, purgeInt)
998
999        if not expect_error:
1000            self.assertTrue(errored, "Expected command to succeed")
1001            self.assertTrue(self.verifyCompactionSettings(server, db_frag_perc, db_frag_size, view_frag_perc,
1002                                                          view_frag_size, from_period, to_period, abort_outside,
1003                                                          parallel_compact, purgeInt),
1004                            "Settings don't match")
1005
1006        else:
1007            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, error_msg),
1008                            "Expected error message not found")
1009            if not initialized:
1010                self.assertTrue(not self.isClusterInitialized(server),
1011                                "Cluster was initialized, but error was received")
1012
1013    def test_gsi_compaction(self):
1014        username = self.input.param("username", None)
1015        password = self.input.param("password", None)
1016
1017        compact_mode = self.input.param("compact_mode", None)
1018        compact_percent = self.input.param("compact_percent", None)
1019        compact_interval = self.input.param("compact_interval", None)
1020        from_period = self.input.param("from_period", None)
1021        to_period = self.input.param("to_period", None)
1022        if compact_interval is None:
1023            from_period = "0:0"
1024            to_period = "0:0"
1025        enable_abort = self.input.param("enable_abort", 0)
1026        expect_error = self.input.param("expect-error", False)
1027        error_msg = self.input.param("error-msg", "")
1028
1029        """ reset node to setup services """
1030        self.rest.force_eject_node()
1031        cli = CouchbaseCLI(self.master, username, password, self.cb_version)
1032        _, _, success = cli.cluster_init(256, 512, None, "data,index,query", None, None,
1033                                                 self.master.rest_username,
1034                                                 self.master.rest_password, None)
1035        self.assertTrue(success, "Cluster initialization failed during test setup")
1036
1037        if compact_interval is not None and "-" in compact_interval:
1038            compact_interval = compact_interval.replace("-", ",")
1039        stdout, _, errored = cli.setting_gsi_compaction(compact_mode, compact_percent,
1040                              compact_interval, from_period, to_period, enable_abort)
1041        self.assertTrue(errored, "Expected command to succeed")
1042        if not expect_error:
1043            self.verify_gsi_compact_settings(compact_mode, compact_percent,
1044                                             compact_interval,
1045                                             from_period, to_period,
1046                                             enable_abort)
1047
1048    def testSettingAutoFailover(self):
1049        username = self.input.param("username", None)
1050        password = self.input.param("password", None)
1051        enabled = self.input.param("enabled", None)
1052        timeout = self.input.param("timeout", None)
1053        initialized = self.input.param("initialized", True)
1054        expect_error = self.input.param("expect-error")
1055        error_msg = self.input.param("error-msg", "")
1056
1057        server = copy.deepcopy(self.servers[0])
1058
1059        rest = RestConnection(server)
1060        rest.force_eject_node()
1061
1062        cli = CouchbaseCLI(server, username, password)
1063        if initialized:
1064            _, _, success = cli.cluster_init(256, None, None, None, None, None, server.rest_username,
1065                                             server.rest_password, None)
1066            self.assertTrue(success, "Cluster initialization failed during test setup")
1067
1068        stdout, _, _ = cli.setting_autofailover(enabled, timeout)
1069
1070        if not expect_error:
1071            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, "Auto-failover settings modified"),
1072                            "Expected command to succeed")
1073            self.assertTrue(self.verifyAutofailoverSettings(server, enabled, timeout),
1074                            "Auto-failover settings were not set properly")
1075
1076        else:
1077            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, error_msg),
1078                            "Expected error message not found")
1079            if not initialized:
1080                self.assertTrue(not self.isClusterInitialized(server),
1081                                "Cluster was initialized, but error was received")
1082
1083    def testSettingAutoReprovision(self):
1084        username = self.input.param("username", None)
1085        password = self.input.param("password", None)
1086        enabled = self.input.param("enabled", None)
1087        max_nodes = self.input.param("max-nodes", 1)
1088        initialized = self.input.param("initialized", True)
1089        expect_error = self.input.param("expect-error")
1090        error_msg = self.input.param("error-msg", "")
1091
1092        server = copy.deepcopy(self.servers[0])
1093
1094        rest = RestConnection(server)
1095        rest.force_eject_node()
1096
1097        cli = CouchbaseCLI(server, username, password)
1098        if initialized:
1099            _, _, success = cli.cluster_init(256, None, None, None, None, None, server.rest_username,
1100                                             server.rest_password, None)
1101            self.assertTrue(success, "Cluster initialization failed during test setup")
1102
1103        stdout, _, _ = cli.setting_autoreprovision(enabled, max_nodes)
1104
1105        if not expect_error:
1106            if enabled == 0:
1107                # WARNING: --max-servers will not take affect because auto-reprovision is being disabled
1108                max_nodes = 1
1109                self.assertEquals(stdout[0], 'WARNING: --max-servers will not take affect because auto-reprovision is being disabled')
1110            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, "Auto-reprovision settings modified"),
1111                            "Expected command to succeed")
1112            self.assertTrue(self.verifyAutoreprovisionSettings(server, enabled, max_nodes),
1113                            "Auto-failover settings were not set properly")
1114
1115        else:
1116            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, error_msg),
1117                            "Expected error message not found")
1118            if not initialized:
1119                self.assertTrue(not self.isClusterInitialized(server),
1120                                "Cluster was initialized, but error was received")
1121
1122
1123    def testSettingNotification(self):
1124        enable = self.input.param("enable", None)
1125        username = self.input.param("username", None)
1126        password = self.input.param("password", None)
1127        initialized = self.input.param("initialized", False)
1128        expect_error = self.input.param("expect-error")
1129        error_msg = self.input.param("error-msg", "")
1130        server = copy.deepcopy(self.servers[0])
1131
1132        rest = RestConnection(server)
1133        rest.force_eject_node()
1134
1135        cli = CouchbaseCLI(server, username, password)
1136        if initialized:
1137            _, _, success = cli.cluster_init(256, None, None, None, None, None, server.rest_username,
1138                                             server.rest_password, None)
1139            self.assertTrue(success, "Cluster initialization failed during test setup")
1140
1141        initialy_enabled = self.verifyNotificationsEnabled(server)
1142
1143        cli = CouchbaseCLI(server, username, password)
1144        stdout, _, _ = cli.setting_notification(enable)
1145
1146        if not expect_error:
1147            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, "Notification settings updated"),
1148                            "Expected command to succeed")
1149            if enable == 1:
1150                self.assertTrue(self.verifyNotificationsEnabled(server), "Notification not enabled")
1151            else:
1152                self.assertTrue(not self.verifyNotificationsEnabled(server), "Notification are enabled")
1153        else:
1154            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, error_msg), "Expected error message not found")
1155            self.assertTrue(self.verifyNotificationsEnabled(server) == initialy_enabled, "Notifications changed after error")
1156
1157    def testSettingCluster(self):
1158        self.clusterSettings("setting-cluster")
1159
1160    def testClusterEdit(self):
1161        self.clusterSettings("cluster-edit")
1162
1163    def clusterSettings(self, cmd):
1164        username = self.input.param("username", None)
1165        password = self.input.param("password", None)
1166        new_username = self.input.param("new-username", None)
1167        new_password = self.input.param("new-password", None)
1168        data_ramsize = self.input.param("data-ramsize", None)
1169        index_ramsize = self.input.param("index-ramsize", None)
1170        fts_ramsize = self.input.param("fts-ramsize", None)
1171        name = self.input.param("name", None)
1172        port = self.input.param("port", None)
1173        initialized = self.input.param("initialized", True)
1174        expect_error = self.input.param("expect-error")
1175        error_msg = self.input.param("error-msg", "")
1176
1177        init_data_memory = 256
1178        init_index_memory = 256
1179        init_fts_memory = 256
1180        init_name = "testrunner"
1181
1182        initial_server = self.servers[0]
1183        server = copy.deepcopy(initial_server)
1184
1185        rest = RestConnection(server)
1186        rest.force_eject_node()
1187
1188        cli = CouchbaseCLI(server, username, password)
1189        if initialized:
1190            _, _, success = cli.cluster_init(init_data_memory, init_index_memory, init_fts_memory, None, None,
1191                                             init_name, server.rest_username, server.rest_password, None)
1192            self.assertTrue(success, "Cluster initialization failed during test setup")
1193
1194        if cmd == "cluster-edit":
1195            stdout, _, _ = cli.cluster_edit(data_ramsize, index_ramsize, fts_ramsize, name, new_username,
1196                                            new_password, port)
1197        else:
1198            stdout, _, _ = cli.setting_cluster(data_ramsize, index_ramsize, fts_ramsize, name, new_username,
1199                                               new_password, port)
1200
1201        if new_username and not expect_error:
1202                server.rest_username = new_username
1203        if new_password and not expect_error:
1204            server.rest_password = new_password
1205        if name and (name[0] == name[-1]) and name.startswith(("'", '"')):
1206            name = name[1:-1]
1207
1208        if not expect_error:
1209            # Update the cluster manager port if it was specified to be changed
1210            if port:
1211                server.port = port
1212            if data_ramsize is None:
1213                data_ramsize = init_data_memory
1214            if index_ramsize is None:
1215                index_ramsize = init_index_memory
1216            if fts_ramsize is None:
1217                fts_ramsize = init_fts_memory
1218            if name is None:
1219                name = init_name
1220
1221            if cmd == "cluster-edit":
1222                self.verifyWarningOutput(stdout, "The cluster-edit command is depercated, use setting-cluster instead")
1223            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, "Cluster settings modified"),
1224                            "Expected command to succeed")
1225            self.assertTrue(self.verifyRamQuotas(server, data_ramsize, index_ramsize, fts_ramsize),
1226                            "Ram quotas not set properly")
1227            self.assertTrue(self.verifyClusterName(server, name), "Cluster name does not match")
1228        else:
1229            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, error_msg), "Expected error message not found")
1230            if not initialized:
1231                self.assertTrue(not self.isClusterInitialized(server), "Cluster was initialized, but error was received")
1232
1233        # Reset the cluster (This is important for when we change the port number)
1234        rest = RestConnection(server)
1235        self.assertTrue(rest.init_cluster(initial_server.rest_username, initial_server.rest_password, initial_server.port),
1236                   "Cluster was not re-initialized at the end of the test")
1237
1238    def testSettingIndex(self):
1239        username = self.input.param("username", None)
1240        password = self.input.param("password", None)
1241        max_rollbacks = self.input.param("max-rollback-points", None)
1242        stable_snap_interval = self.input.param("stable-snapshot-interval", None)
1243        mem_snap_interval = self.input.param("memory-snapshot-interval", None)
1244        storage_mode = self.input.param("storage-mode", None)
1245        threads = self.input.param("threads", None)
1246        log_level = self.input.param("log-level", None)
1247        initialized = self.input.param("initialized", True)
1248        expect_error = self.input.param("expect-error")
1249        error_msg = self.input.param("error-msg", "")
1250
1251        server = copy.deepcopy(self.servers[0])
1252
1253        rest = RestConnection(server)
1254        rest.force_eject_node()
1255
1256        cli = CouchbaseCLI(server, username, password)
1257        if initialized:
1258            _, _, success = cli.cluster_init(256, None, None, None, None, None, server.rest_username,
1259                                             server.rest_password, None)
1260            self.assertTrue(success, "Cluster initialization failed during test setup")
1261
1262        stdout, _, _ = cli.setting_index(max_rollbacks, stable_snap_interval, mem_snap_interval, storage_mode, threads,
1263                                         log_level)
1264
1265        if not expect_error:
1266            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, "Indexer settings modified"),
1267                            "Expected command to succeed")
1268            self.assertTrue(self.verifyIndexSettings(server, max_rollbacks, stable_snap_interval, mem_snap_interval,
1269                                                     storage_mode, threads, log_level),
1270                            "Index settings were not set properly")
1271
1272        else:
1273            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, error_msg),
1274                            "Expected error message not found")
1275            if not initialized:
1276                self.assertTrue(not self.isClusterInitialized(server),
1277                                "Cluster was initialized, but error was received")
1278
1279    def testSettingLdap(self):
1280        username = self.input.param("username", None)
1281        password = self.input.param("password", None)
1282        admins = self.input.param("admins", None)
1283        ro_admins = self.input.param("ro-admins", None)
1284        enabled = self.input.param("enabled", False)
1285        default = self.input.param("default", None)
1286        initialized = self.input.param("initialized", True)
1287        expect_error = self.input.param("expect-error")
1288        error_msg = self.input.param("error-msg", "")
1289
1290        server = copy.deepcopy(self.servers[0])
1291
1292        rest = RestConnection(server)
1293        rest.force_eject_node()
1294
1295        cli = CouchbaseCLI(server, username, password)
1296        if initialized:
1297            _, _, success = cli.cluster_init(256, None, None, None, None, None, server.rest_username,
1298                                             server.rest_password, None)
1299            self.assertTrue(success, "Cluster initialization failed during test setup")
1300
1301        stdout, _, _ = cli.setting_ldap(admins, ro_admins, default, enabled)
1302
1303        if not expect_error:
1304            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, "LDAP settings modified"),
1305                            "Expected command to succeed")
1306            self.assertTrue(self.verifyLdapSettings(server, admins, ro_admins, default, enabled), "LDAP settings not set")
1307
1308        else:
1309            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, error_msg),
1310                            "Expected error message not found")
1311            if not initialized:
1312                self.assertTrue(self.verifyLdapSettings(server, None, None, None, 0), "LDAP setting changed")
1313
1314    def testSettingAlert(self):
1315        username = self.input.param("username", None)
1316        password = self.input.param("password", None)
1317        enabled = self.input.param("enabled", None)
1318        email_recipients = self.input.param("email-recipients", None)
1319        email_sender = self.input.param("email-sender", None)
1320        email_username = self.input.param("email-user", None)
1321        email_password = self.input.param("email-password", None)
1322        email_host = self.input.param("email-host", None)
1323        email_port = self.input.param("email-port", None)
1324        encrypted = self.input.param("encrypted", None)
1325        alert_af_node = self.input.param("alert-auto-failover-node", False)
1326        alert_af_max_reached = self.input.param("alert-auto-failover-max-reached", False)
1327        alert_af_node_down = self.input.param("alert-auto-failover-node-down", False)
1328        alert_af_small = self.input.param("alert-auto-failover-cluster-small", False)
1329        alert_af_disable = self.input.param("alert-auto-failover-disable", False)
1330        alert_ip_changed = self.input.param("alert-ip-changed", False)
1331        alert_disk_space = self.input.param("alert-disk-space", False)
1332        alert_meta_overhead = self.input.param("alert-meta-overhead", False)
1333        alert_meta_oom = self.input.param("alert-meta-oom", False)
1334        alert_write_failed = self.input.param("alert-write-failed", False)
1335        alert_audit_dropped = self.input.param("alert-audit-msg-dropped", False)
1336        initialized = self.input.param("initialized", True)
1337        expect_error = self.input.param("expect-error")
1338        error_msg = self.input.param("error-msg", "")
1339
1340        server = copy.deepcopy(self.servers[0])
1341
1342        rest = RestConnection(server)
1343        rest.force_eject_node()
1344
1345        cli = CouchbaseCLI(server, username, password)
1346        if initialized:
1347            _, _, success = cli.cluster_init(256, None, None, None, None, None, server.rest_username,
1348                                             server.rest_password, None)
1349            self.assertTrue(success, "Cluster initialization failed during test setup")
1350
1351        stdout, _, _ = cli.setting_alert(enabled, email_recipients, email_sender, email_username, email_password,
1352                                         email_host, email_port, encrypted, alert_af_node, alert_af_max_reached,
1353                                         alert_af_node_down, alert_af_small, alert_af_disable, alert_ip_changed,
1354                                         alert_disk_space, alert_meta_overhead, alert_meta_oom, alert_write_failed,
1355                                         alert_audit_dropped)
1356
1357        if not expect_error:
1358            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, "Email alert settings modified"),
1359                            "Expected command to succeed")
1360            self.assertTrue(self.verifyAlertSettings(server, enabled, email_recipients, email_sender, email_username,
1361                                                     email_password, email_host, email_port, encrypted, alert_af_node,
1362                                                     alert_af_max_reached, alert_af_node_down, alert_af_small,
1363                                                     alert_af_disable, alert_ip_changed, alert_disk_space,
1364                                                     alert_meta_overhead, alert_meta_oom, alert_write_failed,
1365                                                     alert_audit_dropped),
1366                            "Alerts settings not set")
1367
1368        else:
1369            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, error_msg),
1370                            "Expected error message not found")
1371
1372    def testBucketCompact(self):
1373        username = self.input.param("username", None)
1374        password = self.input.param("password", None)
1375        bucket_name = self.input.param("bucket-name", None)
1376        data_only = self.input.param("data-only", False)
1377        views_only = self.input.param("views-only", False)
1378        initialized = self.input.param("initialized", True)
1379        expect_error = self.input.param("expect-error")
1380        error_msg = self.input.param("error-msg", "")
1381        init_bucket_type = self.input.param("init-bucket-type", None)
1382
1383        server = copy.deepcopy(self.servers[0])
1384
1385        rest = RestConnection(server)
1386        rest.force_eject_node()
1387
1388        if initialized:
1389            cli = CouchbaseCLI(server, server.rest_username, server.rest_password)
1390            _, _, success = cli.cluster_init(256, None, None, None, None, None, server.rest_username,
1391                                             server.rest_password, None)
1392            self.assertTrue(success, "Cluster initialization failed during test setup")
1393            if init_bucket_type is not None:
1394                _, _, success = cli.bucket_create(bucket_name, init_bucket_type, 256, None, None, None, None,
1395                                                  None, None)
1396                self.assertTrue(success, "Bucket not created during test setup")
1397
1398        cli = CouchbaseCLI(server, username, password)
1399        stdout, _, _ = cli.bucket_compact(bucket_name, data_only, views_only)
1400
1401        if not expect_error:
1402            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, "Bucket compaction started"),
1403                            "Expected command to succeed")
1404        else:
1405            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, error_msg),
1406                            "Expected error message not found")
1407
1408    def testBucketCreate(self):
1409        username = self.input.param("username", None)
1410        password = self.input.param("password", None)
1411        bucket_name = self.input.param("bucket-name", None)
1412        bucket_type = self.input.param("bucket-type", None)
1413        memory_quota = self.input.param("memory-quota", None)
1414        eviction_policy = self.input.param("eviction-policy", None)
1415        replica_count = self.input.param("replica-count", None)
1416        enable_index_replica = self.input.param("enable-replica-index", None)
1417        priority = self.input.param("priority", None)
1418        enable_flush = self.input.param("enable-flush", None)
1419        wait = self.input.param("wait", False)
1420        initialized = self.input.param("initialized", True)
1421        expect_error = self.input.param("expect-error")
1422        error_msg = self.input.param("error-msg", "")
1423        reset_node = self.input.param("reset-node", True)
1424
1425        server = copy.deepcopy(self.servers[0])
1426
1427        if reset_node:
1428            rest = RestConnection(server)
1429            rest.force_eject_node()
1430
1431        cli = CouchbaseCLI(server, username, password)
1432        if initialized and reset_node:
1433            _, _, success = cli.cluster_init(512, None, None, None, None, None,
1434                                             server.rest_username,
1435                                             server.rest_password, None)
1436            self.assertTrue(success, "Cluster initialization failed during test setup")
1437
1438        stdout, _, _ = cli.bucket_create(bucket_name, bucket_type, memory_quota,
1439                                         eviction_policy, replica_count,
1440                                         enable_index_replica, priority,
1441                                         enable_flush, wait)
1442
1443        if not expect_error:
1444            self.assertTrue(self.verifyCommandOutput(stdout, expect_error,
1445                                                     "Bucket created"),
1446                                                     "Expected command to succeed")
1447            self.assertTrue(self.verifyBucketSettings(server, bucket_name,
1448                                                bucket_type, memory_quota,
1449                                                eviction_policy, replica_count,
1450                                                enable_index_replica, priority,
1451                                                enable_flush),
1452                                                "Bucket settings not set properly")
1453        else:
1454            self.assertTrue(not self.verifyContainsBucket(server, bucket_name),
1455                            "Bucket was created even though an error occurred")
1456            self.assertTrue(self.verifyCommandOutput(stdout,
1457                                 expect_error, error_msg),
1458                                 "Expected error message not found")
1459
1460    def testRecreateBucket(self):
1461        """
1462            Create a bucket name A
1463            Delete bucket name A
1464            Recreate bucket name A
1465            :return: nothing
1466        """
1467        username = self.input.param("username", None)
1468        password = self.input.param("password", None)
1469        bucket_name = self.input.param("bucket-name", None)
1470
1471        server=copy.deepcopy(self.servers[0])
1472
1473        self.testBucketCreate()
1474        cli = CouchbaseCLI(server, username, password)
1475        cli.bucket_delete(bucket_name)
1476        self.testBucketCreate()
1477
1478
1479    def testBucketEdit(self):
1480        username = self.input.param("username", None)
1481        password = self.input.param("password", None)
1482        bucket_name = self.input.param("bucket-name", None)
1483        memory_quota = self.input.param("memory-quota", None)
1484        eviction_policy = self.input.param("eviction-policy", None)
1485        replica_count = self.input.param("replica-count", None)
1486        priority = self.input.param("priority", None)
1487        enable_flush = self.input.param("enable-flush", None)
1488        initialized = self.input.param("initialized", True)
1489        expect_error = self.input.param("expect-error")
1490        error_msg = self.input.param("error-msg", "")
1491        init_bucket_type = self.input.param("init-bucket-type", None)
1492
1493        server = copy.deepcopy(self.servers[0])
1494
1495        rest = RestConnection(server)
1496        rest.force_eject_node()
1497
1498        if initialized:
1499            cli = CouchbaseCLI(server, server.rest_username, server.rest_password)
1500            _, _, success = cli.cluster_init(512, None, None, None, None, None, server.rest_username,
1501                                             server.rest_password, None)
1502            self.assertTrue(success, "Cluster initialization failed during test setup")
1503            if init_bucket_type is not None:
1504                _, _, success = cli.bucket_create(bucket_name, init_bucket_type, 256, None, None, None,
1505                                                  None, 0, None)
1506                self.assertTrue(success, "Bucket not created during test setup")
1507
1508        cli = CouchbaseCLI(server, username, password)
1509        stdout, _, _ = cli.bucket_edit(bucket_name, memory_quota, eviction_policy, replica_count,
1510                                       priority, enable_flush)
1511
1512        if not expect_error:
1513            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, "Bucket edited"),
1514                            "Expected command to succeed")
1515            self.assertTrue(self.verifyBucketSettings(server, bucket_name, None, memory_quota,
1516                                eviction_policy, replica_count, None, priority, enable_flush),
1517                            "Bucket settings not set properly")
1518        else:
1519            # List buckets
1520            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, error_msg),
1521                            "Expected error message not found")
1522
1523    def testBucketDelete(self):
1524        username = self.input.param("username", None)
1525        password = self.input.param("password", None)
1526        bucket_name = self.input.param("bucket-name", None)
1527        initialized = self.input.param("initialized", True)
1528        expect_error = self.input.param("expect-error")
1529        error_msg = self.input.param("error-msg", "")
1530        init_bucket_type = self.input.param("init-bucket-type", None)
1531
1532        server = copy.deepcopy(self.servers[-1])
1533        hostname = "%s:%s" % (server.ip, server.port)
1534
1535        rest = RestConnection(server)
1536        rest.force_eject_node()
1537
1538        if initialized:
1539            cli = CouchbaseCLI(server, server.rest_username, server.rest_password)
1540            _, _, success = cli.cluster_init(512, None, None, None, None, None, server.rest_username,
1541                                             server.rest_password, None)
1542            self.assertTrue(success, "Cluster initialization failed during test setup")
1543            if init_bucket_type is not None:
1544                _, _, success = cli.bucket_create(bucket_name, init_bucket_type, 256, None, None, None,
1545                                                  None, 0, None)
1546                self.assertTrue(success, "Bucket not created during test setup")
1547
1548        cli = CouchbaseCLI(server, username, password)
1549        stdout, _, _ = cli.bucket_delete(bucket_name)
1550
1551        if not expect_error:
1552            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, "Bucket deleted"),
1553                            "Expected command to succeed")
1554            self.assertTrue(not self.verifyContainsBucket(server, bucket_name), "Bucket was not deleted")
1555        else:
1556            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, error_msg),
1557                            "Expected error message not found")
1558            if initialized and init_bucket_type is not None:
1559                self.assertTrue(self.verifyContainsBucket(server, bucket_name), "Bucket should not have been deleted")
1560
1561    def testBucketFlush(self):
1562        username = self.input.param("username", None)
1563        password = self.input.param("password", None)
1564        bucket_name = self.input.param("bucket-name", None)
1565        force = self.input.param("force", False)
1566        initialized = self.input.param("initialized", True)
1567        expect_error = self.input.param("expect-error")
1568        error_msg = self.input.param("error-msg", "")
1569
1570        init_bucket_type = self.input.param("init-bucket-type", None)
1571        init_enable_flush = int(self.input.param("init-enable-flush", 0))
1572        insert_keys = 12
1573
1574        server = copy.deepcopy(self.servers[0])
1575
1576        rest = RestConnection(server)
1577        rest.force_eject_node()
1578
1579        if initialized:
1580            cli = CouchbaseCLI(server, server.rest_username, server.rest_password)
1581            _, _, success = cli.cluster_init(512, None, None, None, None, None, server.rest_username,
1582                                             server.rest_password, None)
1583            self.assertTrue(success, "Cluster initialization failed during test setup")
1584            if init_bucket_type is not None:
1585                _, _, success = cli.bucket_create(bucket_name, init_bucket_type, 256, None, None, None,
1586                                                  None, init_enable_flush, None)
1587                self.assertTrue(success, "Bucket not created during test setup")
1588                """ Add built-in user for memcached authentication """
1589                self.add_built_in_server_user(node=server)
1590
1591                MemcachedClientHelper.load_bucket_and_return_the_keys([server], name=bucket_name, number_of_threads=1,
1592                                                                      write_only=True, number_of_items=insert_keys,
1593                                                                      moxi=False)
1594                inserted = int(rest.get_bucket_json(bucket_name)["basicStats"]["itemCount"])
1595                self.assertTrue(self.waitForItemCount(server, bucket_name, insert_keys))
1596
1597        cli = CouchbaseCLI(server, username, password)
1598        stdout, _, _ = cli.bucket_flush(bucket_name, force)
1599
1600        if not expect_error:
1601            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, "Bucket flushed"),
1602                            "Expected command to succeed")
1603            self.assertTrue(self.waitForItemCount(server, bucket_name, 0),
1604                            "Expected 0 keys to be in bucket after the flush")
1605        else:
1606            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, error_msg),
1607                            "Expected error message not found")
1608            if initialized and init_bucket_type is not None:
1609                self.assertTrue(self.waitForItemCount(server, bucket_name, insert_keys),
1610                                "Expected keys to exist after the flush failed")
1611
1612    def testServerAdd(self):
1613        username = self.input.param("username", None)
1614        password = self.input.param("password", None)
1615        num_servers = self.input.param("num-add-servers", None)
1616        server_username = self.input.param("server-add-username", None)
1617        server_password = self.input.param("server-add-password", None)
1618        group = self.input.param("group-name", None)
1619        services = self.input.param("services", None)
1620        index_storage_mode = self.input.param("index-storage-mode", None)
1621        initialized = self.input.param("initialized", True)
1622        expect_error = self.input.param("expect-error")
1623        error_msg = self.input.param("error-msg", "")
1624        init_index_storage_mode = self.input.param("init-index-storage-mode", None)
1625        init_services = self.input.param("init-services", None)
1626
1627        server = copy.deepcopy(self.servers[0])
1628
1629        servers_list = list()
1630        for i in range(0, num_servers):
1631            servers_list.append("%s:%s" % (self.servers[i+1].ip, self.servers[i+1].port))
1632        server_to_add = ",".join(servers_list)
1633
1634        rest = RestConnection(server)
1635        rest.force_eject_node()
1636
1637        if initialized:
1638            cli = CouchbaseCLI(server, server.rest_username, server.rest_password)
1639            _, _, success = cli.cluster_init(256, 256, None, init_services,
1640                                             init_index_storage_mode, None,
1641                                             server.rest_username, server.rest_password,
1642                                             None)
1643            self.assertTrue(success, "Cluster initialization failed during test setup")
1644
1645        time.sleep(5)
1646        cli = CouchbaseCLI(server, username, password)
1647        stdout, _, _ = cli.server_add(server_to_add, server_username, server_password,
1648                                                     group, services,
1649                                                     index_storage_mode)
1650
1651        if not services:
1652            services = "kv"
1653        if group:
1654            if (group[0] == group[-1]) and group.startswith(("'", '"')):
1655                group = group[1:-1]
1656        else:
1657            group = "Group 1"
1658
1659        if not index_storage_mode and "index" in services.split(","):
1660            index_storage_mode = "default"
1661
1662        if not expect_error:
1663            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, "Server added"),
1664                            "Expected command to succeed")
1665            self.assertTrue(self.verifyPendingServer(server, server_to_add, group, services),
1666                            "Pending server has incorrect settings")
1667            self.assertTrue(self.verifyIndexSettings(server, None, None, None,
1668                                                     index_storage_mode, None, None),
1669                                                     "Invalid index storage setting")
1670        else:
1671            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, error_msg),
1672                            "Expected error message not found")
1673            if initialized:
1674                self.assertTrue(self.verifyPendingServerDoesNotExist(server, server_to_add),
1675                                "Pending server exists")
1676
1677    def testRebalance(self):
1678        username = self.input.param("username", None)
1679        password = self.input.param("password", None)
1680        num_add_servers = self.input.param("num-add-servers", 0)
1681        num_remove_servers = self.input.param("num-remove-servers", 0)
1682        num_initial_servers = self.input.param("num-initial-servers", 1)
1683        initialized = self.input.param("initialized", True)
1684        expect_error = self.input.param("expect-error")
1685        error_msg = self.input.param("error-msg", "")
1686
1687        self.assertTrue(num_initial_servers > num_remove_servers, "Specified more remove servers than initial servers")
1688
1689        srv_idx = 0
1690        server = copy.deepcopy(self.servers[srv_idx])
1691        srv_idx += 1
1692
1693        initial_servers_list = list()
1694        for _ in range(0, num_initial_servers-1):
1695            initial_servers_list.append("%s:%s" % (self.servers[srv_idx].ip, self.servers[srv_idx].port))
1696            srv_idx += 1
1697        initial_servers = ",".join(initial_servers_list)
1698
1699        add_servers_list = list()
1700        for _ in range(0, num_add_servers):
1701            add_servers_list.append("%s:%s" % (self.servers[srv_idx].ip, self.servers[srv_idx].port))
1702            srv_idx += 1
1703        servers_to_add = ",".join(add_servers_list)
1704
1705        remove_servers_list = list()
1706        for i in range(0, num_remove_servers):
1707            remove_servers_list.append("%s:%s" % (self.servers[i+1].ip, self.servers[i+1].port))
1708        servers_to_remove = ",".join(remove_servers_list)
1709
1710        rest = RestConnection(server)
1711        rest.force_eject_node()
1712
1713        if initialized:
1714            cli = CouchbaseCLI(server, server.rest_username, server.rest_password)
1715            _, _, success = cli.cluster_init(256, None, None, None, None, None, server.rest_username,
1716                                             server.rest_password, None)
1717            self.assertTrue(success, "Cluster initialization failed during test setup")
1718            time.sleep(5)
1719            if initial_servers != "":
1720                _, _, errored = cli.server_add(initial_servers, server.rest_username, server.rest_password, None, None, None)
1721                self.assertTrue(errored, "Unable to add initial servers")
1722                _, _, errored = cli.rebalance(None)
1723                self.assertTrue(errored, "Unable to complete initial rebalance")
1724            if servers_to_add != "":
1725                _, _, errored = cli.server_add(servers_to_add, server.rest_username, server.rest_password, None, None, None)
1726                self.assertTrue(errored, "Unable to add initial servers")
1727
1728        cli = CouchbaseCLI(server, username, password)
1729        stdout, _, _ = cli.rebalance(servers_to_remove)
1730
1731        if not expect_error:
1732            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, "Rebalance complete"),
1733                            "Expected command to succeed")
1734            self.assertTrue(self.verifyActiveServers(server, num_initial_servers + num_add_servers - num_remove_servers),
1735                            "No new servers were added to the cluster")
1736        else:
1737            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, error_msg),
1738                            "Expected error message not found")
1739            if initialized:
1740                self.assertTrue(self.verifyActiveServers(server, num_initial_servers),
1741                                "Expected no new servers to be in the cluster")
1742
1743    def testRebalanceInvalidRemoveServer(self):
1744        error_msg = self.input.param("error-msg", "")
1745
1746        server = copy.deepcopy(self.servers[0])
1747
1748        rest = RestConnection(server)
1749        rest.force_eject_node()
1750
1751        cli = CouchbaseCLI(server, server.rest_username, server.rest_password)
1752        _, _, success = cli.cluster_init(256, None, None, None, None, None, server.rest_username,
1753                                         server.rest_password, None)
1754        self.assertTrue(success, "Cluster initialization failed during test setup")
1755        time.sleep(5)
1756
1757        stdout, _, _ = cli.rebalance("invalid.server:8091")
1758
1759        self.assertTrue(self.verifyCommandOutput(stdout, True, error_msg),
1760                        "Expected error message not found")
1761        self.assertTrue(self.verifyActiveServers(server, 1),
1762                        "Expected no new servers to be in the cluster")
1763
1764    def testFailover(self):
1765        username = self.input.param("username", None)
1766        password = self.input.param("password", None)
1767        num_initial_servers = self.input.param("num-initial-servers", 2)
1768        invalid_node = self.input.param("invalid-node", False)
1769        no_failover_servers = self.input.param("no-failover-servers", False)
1770        force = self.input.param("force", False)
1771        initialized = self.input.param("initialized", True)
1772        expect_error = self.input.param("expect-error")
1773        error_msg = self.input.param("error-msg", "")
1774
1775        server = copy.deepcopy(self.servers[0])
1776
1777        initial_servers_list = list()
1778        for i in range(0, num_initial_servers - 1):
1779            initial_servers_list.append("%s:%s" % (self.servers[i + 1].ip, self.servers[i + 1].port))
1780        initial_servers = ",".join(initial_servers_list)
1781
1782        server_to_failover = "%s:%s" % (self.servers[1].ip, self.servers[1].port)
1783        if invalid_node:
1784            server_to_failover = "invalid.server:8091"
1785        if no_failover_servers:
1786            server_to_failover = None
1787
1788        rest = RestConnection(server)
1789        rest.force_eject_node()
1790
1791        if initialized:
1792            cli = CouchbaseCLI(server, server.rest_username, server.rest_password)
1793            _, _, success = cli.cluster_init(256, None, None, None, None, None, server.rest_username,
1794                                             server.rest_password, None)
1795            self.assertTrue(success, "Cluster initialization failed during test setup")
1796            if initial_servers != "":
1797                time.sleep(5)
1798                _, _, errored = cli.server_add(initial_servers, server.rest_username, server.rest_password, None, None,
1799                                               None)
1800                self.assertTrue(errored, "Unable to add initial servers")
1801                _, _, errored = cli.rebalance(None)
1802                self.assertTrue(errored, "Unable to complete initial rebalance")
1803            _, _, success = cli.bucket_create("bucket", "couchbase", 256, None, None, None, None,
1804                                                  None, None)
1805            self.assertTrue(success, "Bucket not created during test setup")
1806            time.sleep(10)
1807
1808        cli = CouchbaseCLI(server, username, password)
1809        stdout, _, _ = cli.failover(server_to_failover, force)
1810
1811        if not expect_error:
1812            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, "Server failed over"),
1813                            "Expected command to succeed")
1814            self.assertTrue(self.verifyActiveServers(server, num_initial_servers - 1),
1815                            "Servers not failed over")
1816            self.assertTrue(self.verifyFailedServers(server, 1),
1817                            "Not all servers failed over have `inactiveFailed` status")
1818        else:
1819            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, error_msg),
1820                            "Expected error message not found")
1821            if initialized:
1822                self.assertTrue(self.verifyActiveServers(server, num_initial_servers),
1823                                "Servers should not have been failed over")
1824
1825    def testUserManage(self):
1826        username = self.input.param("username", None)
1827        password = self.input.param("password", None)
1828        list = self.input.param("list", False)
1829        delete = self.input.param("delete", False)
1830        set = self.input.param("set", False)
1831        roles = self.input.param("roles", None)
1832        rbac_username = self.input.param("rbac-username", None)
1833        rbac_password = self.input.param("rbac-password", None)
1834        auth_domain = self.input.param("auth-domain", None)
1835        initialized = self.input.param("initialized", True)
1836        expect_error = self.input.param("expect-error")
1837        error_msg = self.input.param("error-msg", "")
1838
1839        init_rbac_username = self.input.param("init-rbac-username", None)
1840        init_rbac_password = self.input.param("init-rbac-password", None)
1841
1842        server = copy.deepcopy(self.servers[0])
1843
1844        rest = RestConnection(server)
1845        rest.force_eject_node()
1846
1847        payload = "name=%s&roles=%s" % (init_rbac_username, roles)
1848        if auth_domain == "local":
1849            init_user = "local/" + init_rbac_username
1850            rbac_user = "local/" + rbac_username
1851            payload = "name=%s&roles=%s&password=%s" \
1852                      % (init_rbac_username, roles, init_rbac_password)
1853        elif auth_domain == "external":
1854            init_user = "external/" + init_rbac_username
1855            rbac_user = "external/" + rbac_username
1856        else:
1857            self.fail("need to set auth-domain to run this test")
1858
1859        if initialized:
1860            cli = CouchbaseCLI(server, server.rest_username, server.rest_password)
1861            _, _, success = cli.cluster_init(256, 256, None, "data", None, None,
1862                                             server.rest_username,
1863                                             server.rest_password, None)
1864            self.assertTrue(success, "Cluster initialization failed during test setup")
1865            if init_rbac_username and init_rbac_password:
1866                rest.set_user_roles(init_user, payload)
1867                result = self.verifyUserRoles(server, init_rbac_username, roles)
1868                self.assertTrue(result,
1869                                "Setting initial rbac user failed")
1870
1871        time.sleep(5)
1872        cli = CouchbaseCLI(server, username, password)
1873        stdout, _, errored = cli.user_manage(delete, list, set, rbac_username, rbac_password,
1874                                             roles, auth_domain)
1875
1876        if not expect_error:
1877            if list:
1878                self.assertTrue(stdout[0] == init_rbac_username,
1879                                "Listed rbac user is not correct")
1880            else:
1881                self.assertTrue(errored, "Expected command to succeed")
1882            if set:
1883                self.assertTrue(self.verifyUserRoles(server, rbac_username, roles),
1884                                "Read only user was not set")
1885        else:
1886            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, error_msg),
1887                            "Expected error message not found")
1888            if initialized and init_rbac_username:
1889                self.assertTrue(self.verifyUserRoles(server, init_rbac_username, roles),
1890                                "Read only user was changed")
1891        self.log.info("Delete users after test")
1892        if init_rbac_username:
1893            rest.delete_user_roles(init_user)
1894        elif rbac_username:
1895            rest.delete_user_roles(rbac_user)
1896
1897    def testCollectLogStart(self):
1898        username = self.input.param("username", None)
1899        password = self.input.param("password", None)
1900        all_nodes = self.input.param("all-nodes", False)
1901        nodes = self.input.param("nodes", 0)
1902        upload = self.input.param("upload", None)
1903        upload_host = self.input.param("upload-host", None)
1904        upload_customer = self.input.param("customer", None)
1905        upload_ticket = self.input.param("ticket", None)
1906        invalid_node = self.input.param("invalid-node", None)
1907        initialized = self.input.param("initialized", True)
1908        expect_error = self.input.param("expect-error")
1909        error_msg = self.input.param("error-msg", "")
1910
1911        init_num_servers = self.input.param("init-num-servers", 1)
1912
1913        server = copy.deepcopy(self.servers[0])
1914        if len(self.servers) < 4:
1915            mesg = "***\n Collect logs start tests need minimum 4 servers to run\n***"
1916            RemoteMachineShellConnection(server).stop_current_python_running(mesg)
1917
1918        rest = RestConnection(server)
1919        rest.force_eject_node()
1920
1921        servers_to_add = list()
1922        for idx in range(init_num_servers-1):
1923            servers_to_add.append("%s:%s" % (self.servers[idx + 1].ip, self.servers[idx + 1].port))
1924        servers_to_add = ",".join(servers_to_add)
1925
1926        log_nodes = None
1927        if nodes > 0 or invalid_node is not None:
1928            log_nodes = list()
1929            for idx in range(nodes):
1930                log_nodes.append("%s:%s" % (self.servers[idx + 1].ip, self.servers[idx + 1].port))
1931            if invalid_node is not None:
1932                log_nodes.append("invalid:8091")
1933            log_nodes = ",".join(log_nodes)
1934
1935        if initialized:
1936            cli = CouchbaseCLI(server, server.rest_username, server.rest_password)
1937            _, _, success = cli.cluster_init(256, 256, None, "data", None, None, server.rest_username,
1938                                             server.rest_password, None)
1939            self.assertTrue(success, "Cluster initialization failed during test setup")
1940
1941            if init_num_servers > 1:
1942                time.sleep(5)
1943                _, _, errored = cli.server_add(servers_to_add, server.rest_username, server.rest_password, None, None, None)
1944                self.assertTrue(errored, "Could not add initial servers")
1945                _, _, errored = cli.rebalance(None)
1946                self.assertTrue(errored, "Unable to complete initial rebalance")
1947
1948        time.sleep(5)
1949        cli = CouchbaseCLI(server, username, password)
1950        stdout, _, errored = cli.collect_logs_start(all_nodes, log_nodes, upload, upload_host, upload_customer, upload_ticket)
1951
1952        if not expect_error:
1953            self.assertTrue(errored, "Expected command to succeed")
1954        else:
1955            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, error_msg),
1956                            "Expected error message not found")
1957
1958    def testCollectLogStop(self):
1959        username = self.input.param("username", None)
1960        password = self.input.param("password", None)
1961        initialized = self.input.param("initialized", True)
1962        expect_error = self.input.param("expect-error")
1963        error_msg = self.input.param("error-msg", "")
1964
1965        server = copy.deepcopy(self.servers[0])
1966
1967        rest = RestConnection(server)
1968        rest.force_eject_node()
1969
1970        if initialized:
1971            cli = CouchbaseCLI(server, server.rest_username, server.rest_password)
1972            _, _, success = cli.cluster_init(256, 256, None, "data", None, None, server.rest_username,
1973                                             server.rest_password, None)
1974            self.assertTrue(success, "Cluster initialization failed during test setup")
1975        time.sleep(5)
1976        cli = CouchbaseCLI(server, username, password)
1977        stdout, _, errored = cli.collect_logs_stop()
1978
1979        if not expect_error:
1980            self.assertTrue(errored, "Expected command to succeed")
1981        else:
1982            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, error_msg),
1983                            "Expected error message not found")
1984
1985    def test_mctimings_with_data_monitoring_role(self):
1986        """ This role only works from 5.1 and later
1987            params: sasl_buckets=2,default_bucket=False,nodes_init=2,
1988                    permission=self_bucket
1989            if permission=other_bucket, need to add should-fail=True
1990        """
1991        if 5.1 > float(self.cb_version[:3]):
1992            self.log.info("This test only work for version 5.1+")
1993            return
1994        if len(self.buckets) < 2:
1995            self.fail("This test requires minimum of 2 buckets")
1996
1997        permission = self.input.param("permission", "all")
1998        username = "data_monitoring"
1999        bucket_names = []
2000        bucket_name = ""
2001        rest = RestConnection(self.master)
2002        shell = RemoteMachineShellConnection(self.master)
2003        for bucket in self.buckets:
2004            bucket_names.append(bucket.name)
2005        if permission == "all":
2006            role = '*'
2007            bucket_name = bucket_names[random.randint(0,1)]
2008        elif permission == "self_bucket":
2009            role = "{0}".format(bucket_names[0])
2010            bucket_name = bucket_names[0]
2011        elif permission == "other_bucket":
2012            role = "{0}".format(bucket_names[1])
2013            bucket_name = bucket_names[0]
2014        testuser = [{"id": username,
2015                         "name": username,
2016                         "password": "password"}]
2017        rolelist = [{"id": username,
2018                         "name": username,
2019                         "roles": "data_monitoring[{0}]".format(role)}]
2020        kv_gen = BlobGenerator('create', 'create', self.value_size, end=self.num_items)
2021        self._load_all_buckets(self.master, kv_gen, "create",
2022                               self.expire_time, flag=self.item_flag)
2023        try:
2024            status = self.add_built_in_server_user(testuser, rolelist)
2025            if not status:
2026                self.fail("Failed to add user: {0} with role: {1} "\
2027                                             .format(username, role))
2028            cmd = self.cli_command_path + "mctimings" + self.cmd_ext
2029            cmd += " -h " + self.master.ip + ":11210 -u " + username
2030            cmd += " -P password -b " + bucket_name + " --verbose "
2031            output, _ = shell.execute_command(cmd)
2032            if not self.should_fail:
2033                self.assertTrue(self._check_output("The following data is collected", output))
2034            else:
2035                if self._check_output("The following data is collected", output):
2036                    self.fail("This user should not allow to monitor data in this bucket {0}"\
2037                                                                      .format(bucket_name))
2038                else:
2039                    self.log.info("Alright, user bucket A has no permission to check bucket B")
2040        except Exception as e:
2041            print e
2042        finally:
2043            shell.disconnect()
2044            if status:
2045                self.log.info("Remove user {0}".format(rolelist))
2046                RbacBase().remove_user_role(["data_monitoring"], rest)
2047
2048    def test_cmd_set_stats(self):
2049        """ When set any items, cmd_set should increase counting number.
2050            params: default_bucket=False,sasl_buckets=1
2051        /opt/couchbase/bin/cbstats localhost:11210 all -u Administrator -p password -b bucket0 | grep cmd_set
2052        cmd_set: 10011
2053        """
2054
2055        shell = RemoteMachineShellConnection(self.master)
2056        cmd = self.cli_command_path + "cbstats" + self.cmd_ext + " "
2057        cmd += self.master.ip + ":11210 all -u Administrator -p password "
2058        cmd += "-b bucket0 | grep cmd_set"
2059
2060        output, _ = shell.execute_command(cmd)
2061        self.assertTrue(self._check_output("0", output))
2062        kv_gen = BlobGenerator('create', 'create', self.value_size, end=1000)
2063        self._load_all_buckets(self.master, kv_gen, "create",
2064                               self.expire_time, flag=self.item_flag)
2065
2066        output, _ = shell.execute_command(cmd)
2067        self.assertTrue(self._check_output("1000", output))
2068        shell.disconnect()
2069
2070    def testNodeInit(self):
2071        username = self.input.param("username", None)
2072        password = self.input.param("password", None)
2073        data_path = self.input.param("data-path", None)
2074        index_path = self.input.param("index-path", None)
2075        hostname = self.input.param("hostname", None)
2076        initialized = self.input.param("initialized", False)
2077        expect_error = self.input.param("expect-error")
2078        error_msg = self.input.param("error-msg", "")
2079
2080        server = copy.deepcopy(self.servers[0])
2081
2082        rest = RestConnection(server)
2083        rest.force_eject_node()
2084
2085        node_settings = rest.get_nodes_self()
2086        if self.os == "windows":
2087            self.log_path = self.log_path.replace("/cygdrive/c/", "c:/")
2088
2089        if data_path is not None:
2090            if data_path == "valid":
2091                data_path = self.log_path
2092            elif self.os == "windows" and data_path[:1] == "/":
2093                data_path = "c:" + data_path
2094
2095        if index_path is not None:
2096            if index_path == "valid":
2097                index_path = self.log_path
2098            elif self.os == "windows" and index_path[:1] == "/":
2099                index_path = "c:" + index_path
2100
2101        if initialized:
2102            cli = CouchbaseCLI(server, server.rest_username, server.rest_password)
2103            _, _, success = cli.cluster_init(256, 256, None, "data", None, None,
2104                                             server.rest_username,
2105                                             server.rest_password, None)
2106            self.assertTrue(success, "Cluster initialization failed during test setup")
2107        time.sleep(5)
2108        cli = CouchbaseCLI(server, username, password)
2109
2110        stdout, _, errored = cli.node_init(data_path, index_path, hostname)
2111
2112        if not expect_error:
2113            self.assertTrue(errored, "Expected command to succeed")
2114            if data_path is None:
2115                data_path = node_settings.storage[0].path
2116            elif self.os == "windows":
2117                data_path = data_path.replace("\\", "")[:-1]
2118
2119            if index_path is None:
2120                index_path = node_settings.storage[0].index_path
2121            elif self.os == "windows":
2122                index_path = index_path.replace("\\", "")[:-1]
2123            self.assertTrue(self.verify_node_settings(server, data_path,
2124                                                      index_path, hostname),
2125                            "Node settings not changed")
2126        elif self.os != "windows":
2127            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, error_msg),
2128                                                    "Expected error message not found")
2129
2130    def testGroupManage(self):
2131        username = self.input.param("username", None)
2132        password = self.input.param("password", None)
2133        create = self.input.param("create", None)
2134        delete = self.input.param("delete", None)
2135        list = self.input.param("list", None)
2136        move = self.input.param("move-servers", 0)
2137        rename = self.input.param("rename", None)
2138        name = self.input.param("name", None)
2139        from_group = self.input.param("from-group", None)
2140        to_group = self.input.param("to-group", None)
2141        initialized = self.input.param("initialized", True)
2142        expect_error = self.input.param("expect-error")
2143        error_msg = self.input.param("error-msg", "")
2144
2145        init_group = self.input.param("init-group", None)
2146        init_num_servers = self.input.param("init-num-servers", 1)
2147        invalid_move_server = self.input.param("invalid-move-server", None)
2148
2149        server = copy.deepcopy(self.servers[0])
2150
2151        rest = RestConnection(server)
2152        rest.force_eject_node()
2153
2154        to_move = None
2155        if move > 0:
2156            to_move = []
2157            for idx in range(move):
2158                to_move.append("%s:%s" % (self.servers[idx].ip, self.servers[idx].port))
2159            to_move = ",".join(to_move)
2160
2161        if invalid_move_server:
2162            to_move = invalid_move_server
2163
2164        servers_to_add = []
2165        for idx in range(init_num_servers-1):
2166            servers_to_add.append("%s:%s" % (self.servers[idx + 1].ip, self.servers[idx + 1].port))
2167        servers_to_add = ",".join(servers_to_add)
2168
2169        if initialized:
2170            cli = CouchbaseCLI(server, server.rest_username, server.rest_password)
2171            _, _, success = cli.cluster_init(256, 256, None, "data", None, None, server.rest_username,
2172                                             server.rest_password, None)
2173            self.assertTrue(success, "Cluster initialization failed during test setup")
2174
2175            if init_num_servers > 1:
2176                time.sleep(5)
2177                _, _, errored = cli.server_add(servers_to_add, server.rest_username, server.rest_password, None, None, None)
2178                self.assertTrue(errored, "Could not add initial servers")
2179                _, _, errored = cli.rebalance(None)
2180                self.assertTrue(errored, "Unable to complete initial rebalance")
2181
2182            if init_group is not None:
2183                time.sleep(5)
2184                _, _, errored = cli.group_manage(True, False, False, None, None, init_group, None, None)
2185
2186        time.sleep(5)
2187        cli = CouchbaseCLI(server, username, password)
2188        stdout, _, errored = cli.group_manage(create, delete, list, to_move, rename, name, to_group, from_group)
2189
2190        if not expect_error:
2191            self.assertTrue(errored, "Expected command to succeed")
2192            if create:
2193                self.assertTrue(self.verifyGroupExists(server, name), "Group doesn't exist")
2194            elif delete:
2195                self.assertTrue(not self.verifyGroupExists(server, name), "Group doesn't exist")
2196            elif rename:
2197                self.assertTrue(self.verifyGroupExists(server, name), "Group not renamed")
2198            elif move > 0:
2199                _, _, errored = cli.group_manage(False, False, False, to_move, None, None, from_group, to_group)
2200                self.assertTrue(errored, "Group reset failed")
2201
2202        else:
2203            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, error_msg),
2204                            "Expected error message not found")
2205
2206    def testRecovery(self):
2207        username = self.input.param("username", None)
2208        password = self.input.param("password", None)
2209        servers = self.input.param("servers", 0)
2210        recovery_type = self.input.param("recovery-type", None)
2211        initialized = self.input.param("initialized", True)
2212        expect_error = self.input.param("expect-error")
2213        error_msg = self.input.param("error-msg", "")
2214
2215        skip_failover = self.input.param("skip-failover", False)
2216        init_num_servers = self.input.param("init-num-servers", 1)
2217        invalid_recover_server = self.input.param("invalid-recover-server", None)
2218
2219        server = copy.deepcopy(self.servers[0])
2220
2221        rest = RestConnection(server)
2222        rest.force_eject_node()
2223
2224        servers_to_recover = None
2225        if servers > 0:
2226            servers_to_recover = []
2227            for idx in range(servers):
2228                servers_to_recover.append("%s:%s" % (self.servers[idx+1].ip, self.servers[idx+1].port))
2229            servers_to_recover = ",".join(servers_to_recover)
2230
2231        if invalid_recover_server:
2232            servers_to_recover = invalid_recover_server
2233
2234        servers_to_add = []
2235        for idx in range(init_num_servers - 1):
2236            servers_to_add.append("%s:%s" % (self.servers[idx + 1].ip, self.servers[idx + 1].port))
2237        servers_to_add = ",".join(servers_to_add)
2238
2239        if initialized:
2240            cli = CouchbaseCLI(server, server.rest_username, server.rest_password)
2241            _, _, success = cli.cluster_init(256, 256, None, "data", None, None, server.rest_username,
2242                                             server.rest_password, None)
2243            self.assertTrue(success, "Cluster initialization failed during test setup")
2244
2245            if init_num_servers > 1:
2246                time.sleep(5)
2247                _, _, errored = cli.server_add(servers_to_add, server.rest_username, server.rest_password, None, None,
2248                                               None)
2249                self.assertTrue(errored, "Could not add initial servers")
2250                _, _, errored = cli.rebalance(None)
2251                self.assertTrue(errored, "Unable to complete initial rebalance")
2252
2253            if servers_to_recover and not skip_failover:
2254                for restore_server in servers_to_recover.split(","):
2255                    _, _, errored = cli.failover(restore_server, True)
2256                    self.assertTrue(errored, "Unable to failover servers")
2257
2258        time.sleep(5)
2259        cli = CouchbaseCLI(server, username, password)
2260        stdout, _, errored = cli.recovery(servers_to_recover, recovery_type)
2261
2262        if not expect_error:
2263            self.assertTrue(errored, "Expected command to succeed")
2264            self.assertTrue(self.verifyRecoveryType(server, servers_to_recover, recovery_type), "Servers not recovered")
2265        else:
2266            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, error_msg),
2267                            "Expected error message not found")
2268
2269    def testServerReadd(self):
2270        username = self.input.param("username", None)
2271        password = self.input.param("password", None)
2272        servers = self.input.param("servers", 0)
2273        initialized = self.input.param("initialized", True)
2274        expect_error = self.input.param("expect-error")
2275        error_msg = self.input.param("error-msg", "")
2276
2277        skip_failover = self.input.param("skip-failover", False)
2278        init_num_servers = self.input.param("init-num-servers", 1)
2279        invalid_recover_server = self.input.param("invalid-recover-server", None)
2280
2281        server = copy.deepcopy(self.servers[0])
2282        if len(self.servers) < 4:
2283            mesg = "***\n Sever readd tests need minimum 4 servers to run\n***"
2284            RemoteMachineShellConnection(server).stop_current_python_running(mesg)
2285
2286        rest = RestConnection(server)
2287        rest.force_eject_node()
2288
2289        servers_to_recover = None
2290        if servers > 0:
2291            servers_to_recover = []
2292            for idx in range(servers):
2293                servers_to_recover.append("%s:%s" % (self.servers[idx + 1].ip,
2294                                                     self.servers[idx + 1].port))
2295            servers_to_recover = ",".join(servers_to_recover)
2296
2297        if invalid_recover_server:
2298            servers_to_recover = invalid_recover_server
2299
2300        servers_to_add = []
2301        for idx in range(init_num_servers - 1):
2302            servers_to_add.append("%s:%s" % (self.servers[idx + 1].ip,
2303                                             self.servers[idx + 1].port))
2304        servers_to_add = ",".join(servers_to_add)
2305
2306        if initialized:
2307            cli = CouchbaseCLI(server, server.rest_username, server.rest_password)
2308            _, _, success = cli.cluster_init(256, 256, None, "data", None, None,
2309                                             server.rest_username,
2310                                             server.rest_password, None)
2311            self.assertTrue(success, "Cluster initialization failed during test setup")
2312
2313            if init_num_servers > 1:
2314                time.sleep(5)
2315                _, _, errored = cli.server_add(servers_to_add, server.rest_username,
2316                                               server.rest_password, None, None, None)
2317                self.assertTrue(errored, "Could not add initial servers")
2318                _, _, errored = cli.rebalance(None)
2319                self.assertTrue(errored, "Unable to complete initial rebalance")
2320
2321            if servers_to_recover and not skip_failover:
2322                for restore_server in servers_to_recover.split(","):
2323                    _, _, errored = cli.failover(restore_server, True)
2324                    self.assertTrue(errored, "Unable to failover servers")
2325
2326        time.sleep(5)
2327        cli = CouchbaseCLI(server, username, password)
2328        stdout, _, errored = cli.server_readd(servers_to_recover)
2329
2330        if not expect_error:
2331            self.assertTrue(errored, "Expected command to succeed")
2332            self.assertTrue(self.verifyRecoveryType(server, servers_to_recover, "full"),
2333                                                               "Servers not recovered")
2334        else:
2335            self.assertTrue(self.verifyCommandOutput(stdout, expect_error, error_msg),
2336                                                   "Expected error message not found")
2337
2338    def test_change_admin_password_with_read_only_account(self):
2339        """ this test automation for bug MB-20170.
2340            In the bug, when update password of admin, read only account is removed.
2341            This test is to maker sure read only account stay after update password
2342            of Administrator. """
2343        if self.cb_version[:5] in COUCHBASE_FROM_SHERLOCK:
2344            readonly_user = "readonlyuser_1"
2345            curr_passwd = "password"
2346            update_passwd = "password_1"
2347            try:
2348                self.log.info("remove any readonly user in cluster ")
2349                output, error = self.shell.execute_command("%scouchbase-cli "
2350                                             "user-manage --list -c %s:8091 "
2351                                             "-u Administrator -p %s "
2352                           % (self.cli_command_path, self.shell.ip, curr_passwd))
2353                self.log.info("read only user in this cluster %s" % output)
2354                if output:
2355                    for user in output:
2356                        output, error = self.shell.execute_command("%scouchbase-cli "
2357                                                    "user-manage --delete -c %s:8091 "
2358                                                    "--ro-username=%s "
2359                                                    "-u Administrator -p %s "
2360                               % (self.cli_command_path, self.shell.ip, user,
2361                                                                curr_passwd))
2362                        self.log.info("%s" % output)
2363                self.log.info("create a read only user account")
2364                output, error = self.shell.execute_command("%scouchbase-cli "
2365                                              "user-manage -c %s:8091 --set "
2366                                              "--ro-username=%s "
2367                                            "--ro-password=readonlypassword "
2368                                              "-u Administrator -p %s "
2369                                      % (self.cli_command_path, self.shell.ip,
2370                                                  readonly_user, curr_passwd))
2371                self.log.info("%s" % output)
2372                output, error = self.shell.execute_command("%scouchbase-cli "
2373                                             "user-manage --list -c %s:8091 "
2374                                              "-u Administrator -p %s "
2375                                                    % (self.cli_command_path,
2376                                                 self.shell.ip, curr_passwd))
2377                self.log.info("readonly user craeted in this cluster %s" % output)
2378                self.log.info("start update Administrator password")
2379                output, error = self.shell.execute_command("%scouchbase-cli "
2380                                                   "cluster-edit -c %s:8091 "
2381                                                    "-u Administrator -p %s "
2382                                              "--cluster-username=Administrator "
2383                                              "--cluster-password=%s"
2384                                         % (self.cli_command_path, self.shell.ip,
2385                                                     curr_passwd, update_passwd))
2386                self.log.info("%s" % output)
2387                self.log.info("verify if readonly user: %s still exist "
2388                                                                 % readonly_user )
2389                output, error = self.shell.execute_command("%scouchbase-cli "
2390                                             "user-manage --list -c %s:8091 "
2391                                              "-u Administrator -p %s "
2392                                         % (self.cli_command_path, self.shell.ip,
2393                                                                  update_passwd))
2394                self.log.info(" current readonly user in cluster: %s" % output)
2395                self.assertTrue(self._check_output("%s" % readonly_user, output))
2396            finally:
2397                self.log.info("reset password back to original in case test failed")
2398                output, error = self.shell.execute_command("%scouchbase-cli "
2399                                                   "cluster-edit -c %s:8091 "
2400                                                    "-u Administrator -p %s "
2401                                              "--cluster-username=Administrator "
2402                                              "--cluster-password=%s"
2403                                         % (self.cli_command_path, self.shell.ip,
2404                                                     update_passwd, curr_passwd))
2405        else:
2406            self.log.info("readonly account does not support on this version")
2407
2408    def test_directory_backup_structure(self):
2409        """ directory of backup stuctrure should be like
2410            /backup_path/date/date-mode/ as in
2411            /tmp/backup/2016-08-19T185902Z/2016-08-19T185902Z-full/
2412            automation test for bug in ticket MB-20021
2413
2414            default params to run:
2415                backup_cmd=cbbackup,load_all=false,num_sasl_buckets=1 """
2416        backup_cmd = self.input.param("backup_cmd", None)
2417        load_all = self.input.param("load_all", None)
2418        num_backup_bucket = self.input.param("num_backup_bucket", "all")
2419        num_sasl_buckets = self.input.param("num_sasl_buckets", 1)
2420        self.bucket_size = 200
2421        bucket_params=self._create_bucket_params(server=self.master, size=self.bucket_size,
2422                                                        replicas=self.num_replicas,
2423                                                        enable_replica_index=self.enable_replica_index,
2424                                                        eviction_policy=self.eviction_policy)
2425        self.cluster.create_default_bucket(bucket_params)
2426        self._create_sasl_buckets(self.master, num_sasl_buckets)
2427        self.buckets = RestConnection(self.master).get_buckets()
2428        if load_all is None:
2429            self.shell.execute_cbworkloadgen("Administrator", "password", 10000,
2430                                              0.95, "default", 125, " -j")
2431        elif str(load_all).lower() == "true":
2432            for bucket in self.buckets:
2433                self.log.info("load data to bucket %s " % bucket.name)
2434                self.shell.execute_cbworkloadgen("Administrator", "password", 10000,
2435                                                      0.95, bucket.name, 125, " -j")
2436        self.log.info("remove any backup data on node ")
2437        self.shell.execute_command("rm -rf %sbackup" % self.tmp_path)
2438        self.log.info("create backup data on node ")
2439        self.shell.execute_command("mkdir %sbackup " % self.tmp_path)
2440
2441        """ start backup data and check directory structure """
2442        output, error = self.shell.execute_command("date | grep -o '....$'")
2443        dir_start_with = output[0] + "-"
2444        backup_all_buckets = True
2445        partial_backup_buckets = []
2446        if self.cb_version[:5] not in COUCHBASE_FROM_SPOCK and \
2447                                       backup_cmd == "cbbackup":
2448            if num_backup_bucket == "all":
2449                self.shell.execute_cluster_backup()
2450            else:
2451                if str(num_backup_bucket).isdigit() and \
2452                       int(num_sasl_buckets) >= int(num_backup_bucket):
2453                    backup_all_buckets = False
2454                    for i in range(int(num_backup_bucket)):
2455                        partial_backup_buckets.append("bucket" + str(i))
2456                        option = "-b " + "bucket%s " % str(i)
2457                        self.shell.execute_cluster_backup(command_options=option,
2458                                                             delete_backup=False)
2459            output, error = self.shell.execute_command("ls %sbackup " % self.tmp_path)
2460            if output and len(output) == 1:
2461                if output[0].startswith(dir_start_with):
2462                    self.log.info("first level of backup dir is correct %s" % output[0])
2463                else:
2464                    self.fail("Incorrect directory name %s.  It should start with %s"
2465                                                       % (output[0], dir_start_with))
2466            elif output and len(output) > 1:
2467                self.fail("first backup dir level should only have one directory")
2468            elif not output:
2469                self.fail("backup command did not run.  Empty directory %s" % output)
2470            output, error = self.shell.execute_command("ls %sback