1import logger
2import unittest
3import copy
4import datetime
5import time
6import paramiko
7import os
8
9from couchbase_helper.cluster import Cluster
10from TestInput import TestInputSingleton
11from membase.api.rest_client import RestConnection, Bucket
12from couchbase_helper.documentgenerator import BlobGenerator, DocumentGenerator
13from remote.remote_util import RemoteMachineShellConnection, RemoteUtilHelper
14
15class NonRootTests(unittest.TestCase):
16    def setUp(self):
17        self.log = logger.Logger.get_logger()
18        self.input = TestInputSingleton.input
19        self._os = self.input.param("os","null");     #To allow centos, ubuntu, windows
20        self.build = self.input.param("build", "couchbase-server-enterprise_2.2.0-817-rel_x86_64.rpm")
21        self.num_items = self.input.param("items", 100000)
22        self.servers = self.input.servers
23        self.master = self.servers[0]
24        self.clean_up()
25        self.log.info("Begin setting up the couchbase on all server nodes...")
26        self.non_root_install()
27        self.log.info("Wait for 30 seconds after couchbase install over all servers...")
28        time.sleep(30)
29        self.log.info("==============  NonRootTests setUp was started ==============")
30
31    def tearDown(self):
32        """
33            Delete the non-root installation
34        """
35        self.log.info("==============  NonRootTests tearDown was started ==============")
36        for server in self.servers:
37            shell = RemoteMachineShellConnection(server)
38            if self._os == "centos" or self._os == "ubuntu":
39                command = "cd /home/{0}/opt/couchbase && ./bin/couchbase-server -k".format(server.ssh_username)
40                o, e = shell.execute_non_sudo_command(command)
41                shell.log_command_output(o, e)
42                command = "rm -rf etc/ opt/ usr/ {0}.*".format(self.build[:-4])
43                o, e = shell.execute_non_sudo_command(command)
44                shell.log_command_output(o, e)
45            else:
46                #Handling Windows?
47                pass
48            shell.disconnect()
49
50    def clean_up(self):
51        self.log.info("Cleaning up nodes, stopping previous couchbase instances if any ..")
52        for server in self.servers:
53            shell = RemoteMachineShellConnection(server)
54            if self._os == "centos" or self._os == "ubuntu":
55                command = "cd /home/{0}/opt/couchbase && ./bin/couchbase-server -k".format(server.ssh_username)
56                o, e = shell.execute_non_sudo_command(command)
57                shell.log_command_output(o, e)
58                command = "rm -rf etc/ opt/ usr/ {0}.*".format(self.build[:-4])
59                o, e = shell.execute_non_sudo_command(command)
60                shell.log_command_output(o, e)
61                command = "rm -rf backup/"
62                shell.log_command_output(o, e)
63            else:
64                #Handling Windows?
65                pass
66            shell.disconnect()
67
68    """
69        Method that sets up couchbase-server on the server list, without root privileges.
70    """
71    def non_root_install(self):
72        for server in self.servers:
73            shell = RemoteMachineShellConnection(server)
74            info = shell.extract_remote_info()
75            if self._os == "centos":
76                command0 = "rm -rf opt/ etc/ && rm -rf {0}".format(self.build)
77                command1 = "wget http://builds.hq.northscale.net/latestbuilds/{0}".format(self.build)
78                command2 = "rpm2cpio {0} | cpio --extract --make-directories --no-absolute-filenames".format(self.build)
79                command3 = "cd /home/{0}/opt/couchbase && ./bin/install/reloc.sh `pwd`".format(server.ssh_username)
80                command4 = "cd /home/{0}/opt/couchbase && ./bin/couchbase-server -- -noinput -detached".format(server.ssh_username)
81                command5 = "cd /home/{0}/opt/couchbase && ./bin/couchbase-server -k".format(server.ssh_username)
82                o, e = shell.execute_non_sudo_command(command0)
83                shell.log_command_output(o, e)
84                o, e = shell.execute_non_sudo_command(command1)
85                shell.log_command_output(o, e)
86                o, e = shell.execute_non_sudo_command(command2)
87                shell.log_command_output(o, e)
88                o, e = shell.execute_non_sudo_command(command3)
89                shell.log_command_output(o, e)
90                self.log.info("Starting couchbase server <non-root, non-sudo> ..")
91                o, e = shell.execute_non_sudo_command(command4)
92                shell.log_command_output(o, e)
93            elif self._os == "ubuntu":
94                command0 = "rm -rf opt/ etc/ && rm -rf {0}".format(self.build)
95                command1 = "wget http://builds.hq.northscale.net/latestbuilds/{0}".format(self.build)
96                command2 = "dpkg-deb -x {0} /home/{1}".format(self.build, server.ssh_username)
97                command3 = "cd /home/{0}/opt/couchbase && ./bin/install/reloc.sh `pwd`".format(server.ssh_username)
98                command4 = "cd /home/{0}/opt/couchbase && ./bin/couchbase-server -- -noinput -detached".format(server.ssh_username)
99                command5 = "cd /home/{0}/opt/couchbase && ./bin/couchbase-server -k".format(server.ssh_username)
100                o, e = shell.execute_non_sudo_command(command0)
101                shell.log_command_output(o, e)
102                o, e = shell.execute_non_sudo_command(command1)
103                shell.log_command_output(o, e)
104                o, e = shell.execute_non_sudo_command(command2)
105                shell.log_command_output(o, e)
106                o, e = shell.execute_non_sudo_command(command3)
107                shell.log_command_output(o, e)
108                self.log.info("Starting couchbase server <non-root, non-sudo> ..")
109                o, e = shell.execute_non_sudo_command(command4)
110                shell.log_command_output(o, e)
111            elif self._os == "windows":
112                self.fail("TODO: Add instructions for windows")
113            else:
114                self.fail("Enter valid os name, options: centos, ubuntu, windows; entered name: {0} - invalid.".format(self._os))
115
116
117    """
118        Method that initializes cluster, rebalances in nodes, and creates a standard bucket
119    """
120    def init_rebalance_cluster_create_testbucket(self, master, servers):
121        shell = RemoteMachineShellConnection(master)
122        if self._os == "centos" or self._os == "ubuntu":
123            _1 = "cd /home/{0}/opt/couchbase &&".format(master.ssh_username)
124            _2 = " ./bin/couchbase-cli cluster-init -c localhost:8091"
125            _3 = " --cluster-init-username={0} --cluster-init-password={1}".format(master.rest_username, master.rest_password)
126            _4 = " --cluster-init-port=8091 --cluster-init-ramsize=1000"
127            command_to_init = _1 + _2 + _3 + _4
128            o, e = shell.execute_non_sudo_command(command_to_init)
129            shell.log_command_output(o, e)
130            time.sleep(10)
131            for i in range(1, len(servers)):
132                _1 = "cd /home/{0}/opt/couchbase &&".format(master.ssh_username)
133                _2 = " ./bin/couchbase-cli rebalance -c {0}:8091".format(master.ip)
134                _3 = " --server-add={0}:8091".format(servers[i].ip)
135                _4 = " --server-add-username={0}".format(servers[i].rest_username)
136                _5 = " --server-add-password={0}".format(servers[i].rest_password)
137                _6 = " -u {0} -p {1}".format(servers[i].rest_username, servers[i].rest_password)
138                command_to_rebalance = _1 + _2 + _3 + _4 + _5 + _6
139                o, e = shell.execute_non_sudo_command(command_to_rebalance)
140                shell.log_command_output(o, e)
141                time.sleep(10)
142            if len(servers) < 2:
143                rep_count = 0
144            else:
145                rep_count = 1
146            self.log.info("Cluster set up, now creating a bucket ..")
147            _1 = "cd /home/{0}/opt/couchbase &&".format(master.ssh_username)
148            _2 = " ./bin/couchbase-cli bucket-create -c localhost:8091"
149            _3 = " --bucket=testbucket --bucket-type=couchbase --bucket-port=11211"
150            _4 = " --bucket-ramsize=500 --bucket-replica={0} --wait".format(rep_count)
151            _5 = " -u {0} -p {1}".format(master.rest_username, master.rest_password)
152            command_to_create_bucket = _1 + _2 + _3 + _4 + _5
153            o, e = shell.execute_non_sudo_command(command_to_create_bucket)
154            shell.log_command_output(o, e)
155            time.sleep(30)
156        elif self._os == "windows":
157            # TODO: Windows support
158            pass
159        shell.disconnect()
160
161#BEGIN TEST 1
162    """
163        Test loads a certain number of items on a standard bucket created
164        using couchbase-cli and later verifies if the number matches what's expected.
165    """
166    def test_create_bucket_test_load(self):
167        shell = RemoteMachineShellConnection(self.master)
168        self.init_rebalance_cluster_create_testbucket(self.master, self.servers)
169        if self._os == "centos" or self._os == "ubuntu":
170            self.log.info("Load {0} through cbworkloadgen ..".format(self.num_items))
171            _1 = "cd /home/{0}/opt/couchbase &&".format(self.master.ssh_username)
172            _2 = " ./bin/cbworkloadgen -n localhost:8091"
173            _3 = " -r .8 -i {0} -s 256 -b testbucket -t 1".format(self.num_items)
174            _4 = " -u {0} -p {1}".format(self.master.rest_username, self.master.rest_password)
175            command_to_load = _1 + _2 + _3 + _4
176            o, e = shell.execute_non_sudo_command(command_to_load)
177            shell.log_command_output(o, e)
178            time.sleep(20)
179            rest = RestConnection(self.master)
180            item_count = rest.fetch_bucket_stats(bucket="testbucket")["op"]["samples"]["curr_items"][-1]
181            if (item_count == self.num_items):
182                self.log.info("Item count matched, {0}={1}".format(item_count, self.num_items))
183            else:
184                self.fail("Item count: Not what's expected, {0}!={1}".format(item_count, self.num_items))
185            self.log.info("Deleting testbucket ..");
186            _1 = "cd /home/{0}/opt/couchbase &&".format(self.master.ssh_username)
187            _2 = " ./bin/couchbase-cli bucket-delete -c localhost:8091"
188            _3 = " --bucket=testbucket"
189            _4 = " -u {0} -p {1}".format(self.master.rest_username, self.master.rest_password)
190            command_to_delete_bucket = _1 + _2 + _3 + _4
191            o, e = shell.execute_non_sudo_command(command_to_delete_bucket)
192            shell.log_command_output(o, e)
193            time.sleep(10)
194        elif self._os == "windows":
195            # TODO: Windows support
196            self.log.info("Yet to add support for windows!")
197            pass
198        shell.disconnect()
199#END TEST 1
200
201#BEGIN TEST 2
202    """
203        Test that loads a certain number of items, backs up, deletes bucket,
204        recreates bucket, restores, and verifies if count matched.
205    """
206    def test_bucket_backup_restore(self):
207        shell = RemoteMachineShellConnection(self.master)
208        self.init_rebalance_cluster_create_testbucket(self.master, self.servers)
209        if self._os == "centos" or self._os == "ubuntu":
210            self.log.info("Load {0} through cbworkloadgen ..".format(self.num_items))
211            _1 = "cd /home/{0}/opt/couchbase &&".format(self.master.ssh_username)
212            _2 = " ./bin/cbworkloadgen -n localhost:8091"
213            _3 = " -r .8 -i {0} -s 256 -b testbucket -t 1".format(self.num_items)
214            _4 = " -u {0} -p {1}".format(self.master.rest_username, self.master.rest_password)
215            command_to_load = _1 + _2 + _3 + _4
216            o, e = shell.execute_non_sudo_command(command_to_load)
217            shell.log_command_output(o, e)
218            time.sleep(20)
219            rest = RestConnection(self.master)
220            ini_item_count = rest.fetch_bucket_stats(bucket="testbucket")["op"]["samples"]["curr_items"][-1]
221            self.log.info("Backing up bucket 'testbucket' ..")
222            _1 = "cd /home/{0}/opt/couchbase &&".format(self.master.ssh_username)
223            _2 = " ./bin/cbbackup http://localhost:8091"
224            _3 = " /home/{0}/backup".format(self.master.ssh_username)
225            _4 = " -u {0} -p {1}".format(self.master.rest_username, self.master.rest_password)
226            command_to_backup = _1 + _2 + _3 + _4
227            o, e = shell.execute_non_sudo_command(command_to_backup)
228            shell.log_command_output(o, e)
229            time.sleep(10)
230            self.log.info("Deleting bucket ..")
231            _1 = "cd /home/{0}/opt/couchbase &&".format(self.master.ssh_username)
232            _2 = " ./bin/couchbase-cli bucket-delete -c localhost:8091"
233            _3 = " --bucket=testbucket"
234            _4 = " -u {0} -p {1}".format(self.master.rest_username, self.master.rest_password)
235            command_to_delete_bucket = _1 + _2 + _3 + _4
236            o, e = shell.execute_non_sudo_command(command_to_delete_bucket)
237            shell.log_command_output(o, e)
238            time.sleep(20)
239            if len(self.servers) < 2:
240                rep_count = 0
241            else:
242                rep_count = 1
243            self.log.info("Recreating bucket ..")
244            _1 = "cd /home/{0}/opt/couchbase &&".format(self.master.ssh_username)
245            _2 = " ./bin/couchbase-cli bucket-create -c localhost:8091"
246            _3 = " --bucket=testbucket --bucket-type=couchbase --bucket-port=11211"
247            _4 = " --bucket-ramsize=500 --bucket-replica={0} --wait".format(rep_count)
248            _5 = " -u {0} -p {1}".format(self.master.rest_username, self.master.rest_password)
249            command_to_create_bucket = _1 + _2 + _3 + _4 + _5
250            o, e = shell.execute_non_sudo_command(command_to_create_bucket)
251            shell.log_command_output(o, e)
252            time.sleep(20)
253            self.log.info("Restoring bucket 'testbucket' ..")
254            _1 = "cd /home/{0}/opt/couchbase &&".format(self.master.ssh_username)
255            _2 = " ./bin/cbrestore /home/{0}/backup http://localhost:8091".format(self.master.ssh_username)
256            _3 = " -b testbucket -B testbucket"
257            _4 = " -u {0} -p {1}".format(self.master.rest_username, self.master.rest_password)
258            command_to_restore = _1 + _2 + _3 + _4
259            o, e = shell.execute_non_sudo_command(command_to_restore)
260            shell.log_command_output(o, e)
261            time.sleep(10)
262            rest = RestConnection(self.master)
263            fin_item_count = rest.fetch_bucket_stats(bucket="testbucket")["op"]["samples"]["curr_items"][-1]
264            self.log.info("Removing backed-up folder ..")
265            command_to_remove_folder = "rm -rf /home/{0}/backup".format(self.master.ssh_username)
266            o, e = shell.execute_non_sudo_command(command_to_remove_folder)
267            shell.log_command_output(o, e)
268            if (fin_item_count == ini_item_count):
269                self.log.info("Item count before and after deleting with backup/restore matched, {0}={1}".format(
270                    fin_item_count, ini_item_count))
271            else:
272                self.fail("Item count didnt match - backup/restore, {0}!={1}".format(fin_item_count, ini_item_count))
273            self.log.info("Deleting testbucket ..");
274            _1 = "cd /home/{0}/opt/couchbase &&".format(self.master.ssh_username)
275            _2 = " ./bin/couchbase-cli bucket-delete -c localhost:8091"
276            _3 = " --bucket=testbucket"
277            _4 = " -u {0} -p {1}".format(self.master.rest_username, self.master.rest_password)
278            command_to_delete_bucket = _1 + _2 + _3 + _4
279            o, e = shell.execute_non_sudo_command(command_to_delete_bucket)
280            shell.log_command_output(o, e)
281            time.sleep(10)
282        elif self._os == "windows":
283            # TODO: Windows support
284            self.log.info("Yet to add support for windows!")
285            pass
286        shell.disconnect()
287#END TEST 2
288
289    """
290        Method to setup XDCR, and start replication(s)
291    """
292    def setup_xdcr_start_replication(self, src, dest, rep_type, bidirectional):
293        shell1 = RemoteMachineShellConnection(src)
294        shell2 = RemoteMachineShellConnection(dest)
295        if self._os == "centos" or self._os == "ubuntu":
296            self.log.info("Setting up XDCR from source cluster to destination cluster ..")
297            _1 = "cd /home/{0}/opt/couchbase &&".format(src.ssh_username)
298            _2 = " ./bin/couchbase-cli xdcr-setup -c localhost:8091"
299            _3 = " --create --xdcr-cluster-name=_dest --xdcr-hostname={0}:8091".format(dest.ip)
300            _4 = " --xdcr-username={0} --xdcr-password={1}".format(dest.rest_username, dest.rest_password)
301            _5 = " -u {0} -p {1}".format(src.rest_username, src.rest_password)
302            command_to_setup_xdcr = _1 + _2 + _3 + _4 + _5
303            o, e = shell1.execute_non_sudo_command(command_to_setup_xdcr)
304            shell1.log_command_output(o, e)
305            if bidirectional:
306                self.log.info("Setting up XDCR from destination cluster to source cluster ..")
307                _1 = "cd /home/{0}/opt/couchbase &&".format(dest.ssh_username)
308                _2 = " ./bin/couchbase-cli xdcr-setup -c localhost:8091"
309                _3 = " --create --xdcr-cluster-name=_src --xdcr-hostname={0}:8091".format(src.ip)
310                _4 = " --xdcr-username={0} --xdcr-password={1}".format(src.rest_username, src.rest_password)
311                _5 = " -u {0} -p {1}".format(dest.rest_username, dest.rest_password)
312                command_to_setup_xdcr = _1 + _2 + _3 + _4 + _5
313                o, e = shell2.execute_non_sudo_command(command_to_setup_xdcr)
314                shell2.log_command_output(o, e)
315            time.sleep(10)
316            self.log.info("Starting replication from source to destination ..")
317            _1 = "cd /home/{0}/opt/couchbase &&".format(src.ssh_username)
318            _2 = " ./bin/couchbase-cli xdcr-replicate -c localhost:8091"
319            _3 = " --create --xdcr-cluster-name=_dest --xdcr-from-bucket=testbucket"
320            _4 = " --xdcr-to-bucket=testbucket --xdcr-replication-mode={0}".format(rep_type)
321            _5 = " -u {0} -p {1}".format(src.rest_username, src.rest_password)
322            command_to_setup_xdcr = _1 + _2 + _3 + _4 + _5
323            o, e = shell1.execute_non_sudo_command(command_to_setup_xdcr)
324            shell1.log_command_output(o, e)
325            if bidirectional:
326                self.log.info("Starting replication from destination to source ..")
327                _1 = "cd /home/{0}/opt/couchbase &&".format(dest.ssh_username)
328                _2 = " ./bin/couchbase-cli xdcr-replicate -c localhost:8091"
329                _3 = " --create --xdcr-cluster-name=_src --xdcr-from-bucket=testbucket"
330                _4 = " --xdcr-to-bucket=testbucket --xdcr-replication-mode={0}".format(rep_type)
331                _5 = " -u {0} -p {1}".format(dest.rest_username, dest.rest_password)
332                command_to_setup_xdcr = _1 + _2 + _3 + _4 + _5
333                o, e = shell2.execute_non_sudo_command(command_to_setup_xdcr)
334                shell2.log_command_output(o, e)
335            time.sleep(10)
336        elif self._os == "windows":
337            # TODO: WIndows support
338            pass
339        shell1.disconnect()
340        shell2.disconnect()
341
342    """
343        Method to wait for replication to catch up
344    """
345    def wait_for_replication_to_catchup(self, src, dest, timeout, _str_):
346        rest1 = RestConnection(src)
347        rest2 = RestConnection(dest)
348        # 20 minutes by default
349        end_time = time.time() + timeout
350
351        _count1 = rest1.fetch_bucket_stats(bucket="testbucket")["op"]["samples"]["curr_items"][-1]
352        _count2 = rest2.fetch_bucket_stats(bucket="testbucket")["op"]["samples"]["curr_items"][-1]
353        while _count1 != _count2 and (time.time() - end_time) < 0:
354            self.log.info("Waiting for replication to catch up ..")
355            time.sleep(60)
356            _count1 = rest1.fetch_bucket_stats(bucket="testbucket")["op"]["samples"]["curr_items"][-1]
357            _count2 = rest2.fetch_bucket_stats(bucket="testbucket")["op"]["samples"]["curr_items"][-1]
358        if _count1 != _count2:
359            self.fail("not all items replicated in {0} sec for bucket: testbucket. on source cluster:{1}, on dest:{2}".\
360                                   format(timeout, _count1, _count2))
361        self.log.info("Replication caught up at {0}, for testbucket".format(_str_))
362
363#BEGIN TEST 3
364    """
365        Test that enagages clusters specified in XDCR,
366        verifies whether data went through as expected.
367    """
368    def test_xdcr(self):
369        _rep_type = self.input.param("replication_type", "capi")    # capi or xmem
370        _bixdcr = self.input.param("bidirectional", "false")
371        _clusters_dic = self.input.clusters
372        _src_nodes = copy.copy(_clusters_dic[0])
373        _src_master = _src_nodes[0]
374        _dest_nodes = copy.copy(_clusters_dic[1])
375        _dest_master = _dest_nodes[0]
376
377        # Build source cluster
378        self.init_rebalance_cluster_create_testbucket(_src_master, _src_nodes)
379        # Build destination cluster
380        self.init_rebalance_cluster_create_testbucket(_dest_master, _dest_nodes)
381
382        # Setting up XDCR
383        self.setup_xdcr_start_replication(_src_master, _dest_master, _rep_type, _bixdcr)
384
385        shell1 = RemoteMachineShellConnection(_src_master)
386        shell2 = RemoteMachineShellConnection(_dest_master)
387        src_item_count = 0
388        dest_item_count = 0
389        if self._os == "centos" or self._os == "ubuntu":
390            self.log.info("Load {0} through cbworkloadgen at src..".format(self.num_items))
391            _1 = "cd /home/{0}/opt/couchbase &&".format(_src_master.ssh_username)
392            _2 = " ./bin/cbworkloadgen -n localhost:8091 --prefix=s_"
393            _3 = " -r .8 -i {0} -s 256 -b testbucket -t 1".format(self.num_items)
394            _4 = " -u {0} -p {1}".format(_src_master.rest_username, _src_master.rest_password)
395            command_to_load = _1 + _2 + _3 + _4
396            o, e = shell1.execute_non_sudo_command(command_to_load)
397            shell1.log_command_output(o, e)
398            time.sleep(20)
399            rest = RestConnection(_src_master)
400            src_item_count = rest.fetch_bucket_stats(bucket="testbucket")["op"]["samples"]["curr_items"][-1]
401            if _bixdcr:
402                self.log.info("Load {0} through cbworkloadgen at src..".format(self.num_items))
403                _1 = "cd /home/{0}/opt/couchbase &&".format(_dest_master.ssh_username)
404                _2 = " ./bin/cbworkloadgen -n localhost:8091 --prefix=d_"
405                _3 = " -r .8 -i {0} -s 256 -b testbucket -t 1".format(self.num_items)
406                _4 = " -u {0} -p {1}".format(_dest_master.rest_username, _dest_master.rest_password)
407                command_to_load = _1 + _2 + _3 + _4
408                o, e = shell2.execute_non_sudo_command(command_to_load)
409                shell2.log_command_output(o, e)
410                time.sleep(20)
411                rest = RestConnection(_dest_master)
412                dest_item_count = rest.fetch_bucket_stats(bucket="testbucket")["op"]["samples"]["curr_items"][-1]
413            self.wait_for_replication_to_catchup(_src_master, _dest_master, 1200, "destination")
414            if _bixdcr:
415                self.wait_for_replication_to_catchup(_dest_master, _src_master, 1200, "source")
416            self.log.info("XDC REPLICATION caught up")
417            rest1 = RestConnection(_src_master)
418            rest2 = RestConnection(_dest_master)
419            curr_count_on_src = rest1.fetch_bucket_stats(bucket="testbucket")["op"]["samples"]["curr_items"][-1]
420            curr_count_on_dest = rest2.fetch_bucket_stats(bucket="testbucket")["op"]["samples"]["curr_items"][-1]
421            assert(curr_count_on_src==(src_item_count + dest_item_count), "ItemCount on source not what's expected")
422            assert(curr_count_on_dest==(src_item_count + dest_item_count), "ItemCount on destination not what's expected")
423        elif self._os == "windows":
424            # TODO: Windows support
425            self.log.info("Yet to add support for windows!")
426            pass
427        shell1.disconnect()
428        shell2.disconnect()
429#END TEST 3
430