1import logger 2import unittest 3import copy 4import datetime 5import time 6import paramiko 7import os 8 9from couchbase_helper.cluster import Cluster 10from TestInput import TestInputSingleton 11from membase.api.rest_client import RestConnection, Bucket 12from couchbase_helper.documentgenerator import BlobGenerator, DocumentGenerator 13from remote.remote_util import RemoteMachineShellConnection, RemoteUtilHelper 14 15class NonRootTests(unittest.TestCase): 16 def setUp(self): 17 self.log = logger.Logger.get_logger() 18 self.input = TestInputSingleton.input 19 self._os = self.input.param("os","null"); #To allow centos, ubuntu, windows 20 self.build = self.input.param("build", "couchbase-server-enterprise_2.2.0-817-rel_x86_64.rpm") 21 self.num_items = self.input.param("items", 100000) 22 self.servers = self.input.servers 23 self.master = self.servers[0] 24 self.clean_up() 25 self.log.info("Begin setting up the couchbase on all server nodes...") 26 self.non_root_install() 27 self.log.info("Wait for 30 seconds after couchbase install over all servers...") 28 time.sleep(30) 29 self.log.info("============== NonRootTests setUp was started ==============") 30 31 def tearDown(self): 32 """ 33 Delete the non-root installation 34 """ 35 self.log.info("============== NonRootTests tearDown was started ==============") 36 for server in self.servers: 37 shell = RemoteMachineShellConnection(server) 38 if self._os == "centos" or self._os == "ubuntu": 39 command = "cd /home/{0}/opt/couchbase && ./bin/couchbase-server -k".format(server.ssh_username) 40 o, e = shell.execute_non_sudo_command(command) 41 shell.log_command_output(o, e) 42 command = "rm -rf etc/ opt/ usr/ {0}.*".format(self.build[:-4]) 43 o, e = shell.execute_non_sudo_command(command) 44 shell.log_command_output(o, e) 45 else: 46 #Handling Windows? 47 pass 48 shell.disconnect() 49 50 def clean_up(self): 51 self.log.info("Cleaning up nodes, stopping previous couchbase instances if any ..") 52 for server in self.servers: 53 shell = RemoteMachineShellConnection(server) 54 if self._os == "centos" or self._os == "ubuntu": 55 command = "cd /home/{0}/opt/couchbase && ./bin/couchbase-server -k".format(server.ssh_username) 56 o, e = shell.execute_non_sudo_command(command) 57 shell.log_command_output(o, e) 58 command = "rm -rf etc/ opt/ usr/ {0}.*".format(self.build[:-4]) 59 o, e = shell.execute_non_sudo_command(command) 60 shell.log_command_output(o, e) 61 command = "rm -rf backup/" 62 shell.log_command_output(o, e) 63 else: 64 #Handling Windows? 65 pass 66 shell.disconnect() 67 68 """ 69 Method that sets up couchbase-server on the server list, without root privileges. 70 """ 71 def non_root_install(self): 72 for server in self.servers: 73 shell = RemoteMachineShellConnection(server) 74 info = shell.extract_remote_info() 75 if self._os == "centos": 76 command0 = "rm -rf opt/ etc/ && rm -rf {0}".format(self.build) 77 command1 = "wget http://builds.hq.northscale.net/latestbuilds/{0}".format(self.build) 78 command2 = "rpm2cpio {0} | cpio --extract --make-directories --no-absolute-filenames".format(self.build) 79 command3 = "cd /home/{0}/opt/couchbase && ./bin/install/reloc.sh `pwd`".format(server.ssh_username) 80 command4 = "cd /home/{0}/opt/couchbase && ./bin/couchbase-server -- -noinput -detached".format(server.ssh_username) 81 command5 = "cd /home/{0}/opt/couchbase && ./bin/couchbase-server -k".format(server.ssh_username) 82 o, e = shell.execute_non_sudo_command(command0) 83 shell.log_command_output(o, e) 84 o, e = shell.execute_non_sudo_command(command1) 85 shell.log_command_output(o, e) 86 o, e = shell.execute_non_sudo_command(command2) 87 shell.log_command_output(o, e) 88 o, e = shell.execute_non_sudo_command(command3) 89 shell.log_command_output(o, e) 90 self.log.info("Starting couchbase server <non-root, non-sudo> ..") 91 o, e = shell.execute_non_sudo_command(command4) 92 shell.log_command_output(o, e) 93 elif self._os == "ubuntu": 94 command0 = "rm -rf opt/ etc/ && rm -rf {0}".format(self.build) 95 command1 = "wget http://builds.hq.northscale.net/latestbuilds/{0}".format(self.build) 96 command2 = "dpkg-deb -x {0} /home/{1}".format(self.build, server.ssh_username) 97 command3 = "cd /home/{0}/opt/couchbase && ./bin/install/reloc.sh `pwd`".format(server.ssh_username) 98 command4 = "cd /home/{0}/opt/couchbase && ./bin/couchbase-server -- -noinput -detached".format(server.ssh_username) 99 command5 = "cd /home/{0}/opt/couchbase && ./bin/couchbase-server -k".format(server.ssh_username) 100 o, e = shell.execute_non_sudo_command(command0) 101 shell.log_command_output(o, e) 102 o, e = shell.execute_non_sudo_command(command1) 103 shell.log_command_output(o, e) 104 o, e = shell.execute_non_sudo_command(command2) 105 shell.log_command_output(o, e) 106 o, e = shell.execute_non_sudo_command(command3) 107 shell.log_command_output(o, e) 108 self.log.info("Starting couchbase server <non-root, non-sudo> ..") 109 o, e = shell.execute_non_sudo_command(command4) 110 shell.log_command_output(o, e) 111 elif self._os == "windows": 112 self.fail("TODO: Add instructions for windows") 113 else: 114 self.fail("Enter valid os name, options: centos, ubuntu, windows; entered name: {0} - invalid.".format(self._os)) 115 116 117 """ 118 Method that initializes cluster, rebalances in nodes, and creates a standard bucket 119 """ 120 def init_rebalance_cluster_create_testbucket(self, master, servers): 121 shell = RemoteMachineShellConnection(master) 122 if self._os == "centos" or self._os == "ubuntu": 123 _1 = "cd /home/{0}/opt/couchbase &&".format(master.ssh_username) 124 _2 = " ./bin/couchbase-cli cluster-init -c localhost:8091" 125 _3 = " --cluster-init-username={0} --cluster-init-password={1}".format(master.rest_username, master.rest_password) 126 _4 = " --cluster-init-port=8091 --cluster-init-ramsize=1000" 127 command_to_init = _1 + _2 + _3 + _4 128 o, e = shell.execute_non_sudo_command(command_to_init) 129 shell.log_command_output(o, e) 130 time.sleep(10) 131 for i in range(1, len(servers)): 132 _1 = "cd /home/{0}/opt/couchbase &&".format(master.ssh_username) 133 _2 = " ./bin/couchbase-cli rebalance -c {0}:8091".format(master.ip) 134 _3 = " --server-add={0}:8091".format(servers[i].ip) 135 _4 = " --server-add-username={0}".format(servers[i].rest_username) 136 _5 = " --server-add-password={0}".format(servers[i].rest_password) 137 _6 = " -u {0} -p {1}".format(servers[i].rest_username, servers[i].rest_password) 138 command_to_rebalance = _1 + _2 + _3 + _4 + _5 + _6 139 o, e = shell.execute_non_sudo_command(command_to_rebalance) 140 shell.log_command_output(o, e) 141 time.sleep(10) 142 if len(servers) < 2: 143 rep_count = 0 144 else: 145 rep_count = 1 146 self.log.info("Cluster set up, now creating a bucket ..") 147 _1 = "cd /home/{0}/opt/couchbase &&".format(master.ssh_username) 148 _2 = " ./bin/couchbase-cli bucket-create -c localhost:8091" 149 _3 = " --bucket=testbucket --bucket-type=couchbase --bucket-port=11211" 150 _4 = " --bucket-ramsize=500 --bucket-replica={0} --wait".format(rep_count) 151 _5 = " -u {0} -p {1}".format(master.rest_username, master.rest_password) 152 command_to_create_bucket = _1 + _2 + _3 + _4 + _5 153 o, e = shell.execute_non_sudo_command(command_to_create_bucket) 154 shell.log_command_output(o, e) 155 time.sleep(30) 156 elif self._os == "windows": 157 # TODO: Windows support 158 pass 159 shell.disconnect() 160 161#BEGIN TEST 1 162 """ 163 Test loads a certain number of items on a standard bucket created 164 using couchbase-cli and later verifies if the number matches what's expected. 165 """ 166 def test_create_bucket_test_load(self): 167 shell = RemoteMachineShellConnection(self.master) 168 self.init_rebalance_cluster_create_testbucket(self.master, self.servers) 169 if self._os == "centos" or self._os == "ubuntu": 170 self.log.info("Load {0} through cbworkloadgen ..".format(self.num_items)) 171 _1 = "cd /home/{0}/opt/couchbase &&".format(self.master.ssh_username) 172 _2 = " ./bin/cbworkloadgen -n localhost:8091" 173 _3 = " -r .8 -i {0} -s 256 -b testbucket -t 1".format(self.num_items) 174 _4 = " -u {0} -p {1}".format(self.master.rest_username, self.master.rest_password) 175 command_to_load = _1 + _2 + _3 + _4 176 o, e = shell.execute_non_sudo_command(command_to_load) 177 shell.log_command_output(o, e) 178 time.sleep(20) 179 rest = RestConnection(self.master) 180 item_count = rest.fetch_bucket_stats(bucket="testbucket")["op"]["samples"]["curr_items"][-1] 181 if (item_count == self.num_items): 182 self.log.info("Item count matched, {0}={1}".format(item_count, self.num_items)) 183 else: 184 self.fail("Item count: Not what's expected, {0}!={1}".format(item_count, self.num_items)) 185 self.log.info("Deleting testbucket .."); 186 _1 = "cd /home/{0}/opt/couchbase &&".format(self.master.ssh_username) 187 _2 = " ./bin/couchbase-cli bucket-delete -c localhost:8091" 188 _3 = " --bucket=testbucket" 189 _4 = " -u {0} -p {1}".format(self.master.rest_username, self.master.rest_password) 190 command_to_delete_bucket = _1 + _2 + _3 + _4 191 o, e = shell.execute_non_sudo_command(command_to_delete_bucket) 192 shell.log_command_output(o, e) 193 time.sleep(10) 194 elif self._os == "windows": 195 # TODO: Windows support 196 self.log.info("Yet to add support for windows!") 197 pass 198 shell.disconnect() 199#END TEST 1 200 201#BEGIN TEST 2 202 """ 203 Test that loads a certain number of items, backs up, deletes bucket, 204 recreates bucket, restores, and verifies if count matched. 205 """ 206 def test_bucket_backup_restore(self): 207 shell = RemoteMachineShellConnection(self.master) 208 self.init_rebalance_cluster_create_testbucket(self.master, self.servers) 209 if self._os == "centos" or self._os == "ubuntu": 210 self.log.info("Load {0} through cbworkloadgen ..".format(self.num_items)) 211 _1 = "cd /home/{0}/opt/couchbase &&".format(self.master.ssh_username) 212 _2 = " ./bin/cbworkloadgen -n localhost:8091" 213 _3 = " -r .8 -i {0} -s 256 -b testbucket -t 1".format(self.num_items) 214 _4 = " -u {0} -p {1}".format(self.master.rest_username, self.master.rest_password) 215 command_to_load = _1 + _2 + _3 + _4 216 o, e = shell.execute_non_sudo_command(command_to_load) 217 shell.log_command_output(o, e) 218 time.sleep(20) 219 rest = RestConnection(self.master) 220 ini_item_count = rest.fetch_bucket_stats(bucket="testbucket")["op"]["samples"]["curr_items"][-1] 221 self.log.info("Backing up bucket 'testbucket' ..") 222 _1 = "cd /home/{0}/opt/couchbase &&".format(self.master.ssh_username) 223 _2 = " ./bin/cbbackup http://localhost:8091" 224 _3 = " /home/{0}/backup".format(self.master.ssh_username) 225 _4 = " -u {0} -p {1}".format(self.master.rest_username, self.master.rest_password) 226 command_to_backup = _1 + _2 + _3 + _4 227 o, e = shell.execute_non_sudo_command(command_to_backup) 228 shell.log_command_output(o, e) 229 time.sleep(10) 230 self.log.info("Deleting bucket ..") 231 _1 = "cd /home/{0}/opt/couchbase &&".format(self.master.ssh_username) 232 _2 = " ./bin/couchbase-cli bucket-delete -c localhost:8091" 233 _3 = " --bucket=testbucket" 234 _4 = " -u {0} -p {1}".format(self.master.rest_username, self.master.rest_password) 235 command_to_delete_bucket = _1 + _2 + _3 + _4 236 o, e = shell.execute_non_sudo_command(command_to_delete_bucket) 237 shell.log_command_output(o, e) 238 time.sleep(20) 239 if len(self.servers) < 2: 240 rep_count = 0 241 else: 242 rep_count = 1 243 self.log.info("Recreating bucket ..") 244 _1 = "cd /home/{0}/opt/couchbase &&".format(self.master.ssh_username) 245 _2 = " ./bin/couchbase-cli bucket-create -c localhost:8091" 246 _3 = " --bucket=testbucket --bucket-type=couchbase --bucket-port=11211" 247 _4 = " --bucket-ramsize=500 --bucket-replica={0} --wait".format(rep_count) 248 _5 = " -u {0} -p {1}".format(self.master.rest_username, self.master.rest_password) 249 command_to_create_bucket = _1 + _2 + _3 + _4 + _5 250 o, e = shell.execute_non_sudo_command(command_to_create_bucket) 251 shell.log_command_output(o, e) 252 time.sleep(20) 253 self.log.info("Restoring bucket 'testbucket' ..") 254 _1 = "cd /home/{0}/opt/couchbase &&".format(self.master.ssh_username) 255 _2 = " ./bin/cbrestore /home/{0}/backup http://localhost:8091".format(self.master.ssh_username) 256 _3 = " -b testbucket -B testbucket" 257 _4 = " -u {0} -p {1}".format(self.master.rest_username, self.master.rest_password) 258 command_to_restore = _1 + _2 + _3 + _4 259 o, e = shell.execute_non_sudo_command(command_to_restore) 260 shell.log_command_output(o, e) 261 time.sleep(10) 262 rest = RestConnection(self.master) 263 fin_item_count = rest.fetch_bucket_stats(bucket="testbucket")["op"]["samples"]["curr_items"][-1] 264 self.log.info("Removing backed-up folder ..") 265 command_to_remove_folder = "rm -rf /home/{0}/backup".format(self.master.ssh_username) 266 o, e = shell.execute_non_sudo_command(command_to_remove_folder) 267 shell.log_command_output(o, e) 268 if (fin_item_count == ini_item_count): 269 self.log.info("Item count before and after deleting with backup/restore matched, {0}={1}".format( 270 fin_item_count, ini_item_count)) 271 else: 272 self.fail("Item count didnt match - backup/restore, {0}!={1}".format(fin_item_count, ini_item_count)) 273 self.log.info("Deleting testbucket .."); 274 _1 = "cd /home/{0}/opt/couchbase &&".format(self.master.ssh_username) 275 _2 = " ./bin/couchbase-cli bucket-delete -c localhost:8091" 276 _3 = " --bucket=testbucket" 277 _4 = " -u {0} -p {1}".format(self.master.rest_username, self.master.rest_password) 278 command_to_delete_bucket = _1 + _2 + _3 + _4 279 o, e = shell.execute_non_sudo_command(command_to_delete_bucket) 280 shell.log_command_output(o, e) 281 time.sleep(10) 282 elif self._os == "windows": 283 # TODO: Windows support 284 self.log.info("Yet to add support for windows!") 285 pass 286 shell.disconnect() 287#END TEST 2 288 289 """ 290 Method to setup XDCR, and start replication(s) 291 """ 292 def setup_xdcr_start_replication(self, src, dest, rep_type, bidirectional): 293 shell1 = RemoteMachineShellConnection(src) 294 shell2 = RemoteMachineShellConnection(dest) 295 if self._os == "centos" or self._os == "ubuntu": 296 self.log.info("Setting up XDCR from source cluster to destination cluster ..") 297 _1 = "cd /home/{0}/opt/couchbase &&".format(src.ssh_username) 298 _2 = " ./bin/couchbase-cli xdcr-setup -c localhost:8091" 299 _3 = " --create --xdcr-cluster-name=_dest --xdcr-hostname={0}:8091".format(dest.ip) 300 _4 = " --xdcr-username={0} --xdcr-password={1}".format(dest.rest_username, dest.rest_password) 301 _5 = " -u {0} -p {1}".format(src.rest_username, src.rest_password) 302 command_to_setup_xdcr = _1 + _2 + _3 + _4 + _5 303 o, e = shell1.execute_non_sudo_command(command_to_setup_xdcr) 304 shell1.log_command_output(o, e) 305 if bidirectional: 306 self.log.info("Setting up XDCR from destination cluster to source cluster ..") 307 _1 = "cd /home/{0}/opt/couchbase &&".format(dest.ssh_username) 308 _2 = " ./bin/couchbase-cli xdcr-setup -c localhost:8091" 309 _3 = " --create --xdcr-cluster-name=_src --xdcr-hostname={0}:8091".format(src.ip) 310 _4 = " --xdcr-username={0} --xdcr-password={1}".format(src.rest_username, src.rest_password) 311 _5 = " -u {0} -p {1}".format(dest.rest_username, dest.rest_password) 312 command_to_setup_xdcr = _1 + _2 + _3 + _4 + _5 313 o, e = shell2.execute_non_sudo_command(command_to_setup_xdcr) 314 shell2.log_command_output(o, e) 315 time.sleep(10) 316 self.log.info("Starting replication from source to destination ..") 317 _1 = "cd /home/{0}/opt/couchbase &&".format(src.ssh_username) 318 _2 = " ./bin/couchbase-cli xdcr-replicate -c localhost:8091" 319 _3 = " --create --xdcr-cluster-name=_dest --xdcr-from-bucket=testbucket" 320 _4 = " --xdcr-to-bucket=testbucket --xdcr-replication-mode={0}".format(rep_type) 321 _5 = " -u {0} -p {1}".format(src.rest_username, src.rest_password) 322 command_to_setup_xdcr = _1 + _2 + _3 + _4 + _5 323 o, e = shell1.execute_non_sudo_command(command_to_setup_xdcr) 324 shell1.log_command_output(o, e) 325 if bidirectional: 326 self.log.info("Starting replication from destination to source ..") 327 _1 = "cd /home/{0}/opt/couchbase &&".format(dest.ssh_username) 328 _2 = " ./bin/couchbase-cli xdcr-replicate -c localhost:8091" 329 _3 = " --create --xdcr-cluster-name=_src --xdcr-from-bucket=testbucket" 330 _4 = " --xdcr-to-bucket=testbucket --xdcr-replication-mode={0}".format(rep_type) 331 _5 = " -u {0} -p {1}".format(dest.rest_username, dest.rest_password) 332 command_to_setup_xdcr = _1 + _2 + _3 + _4 + _5 333 o, e = shell2.execute_non_sudo_command(command_to_setup_xdcr) 334 shell2.log_command_output(o, e) 335 time.sleep(10) 336 elif self._os == "windows": 337 # TODO: WIndows support 338 pass 339 shell1.disconnect() 340 shell2.disconnect() 341 342 """ 343 Method to wait for replication to catch up 344 """ 345 def wait_for_replication_to_catchup(self, src, dest, timeout, _str_): 346 rest1 = RestConnection(src) 347 rest2 = RestConnection(dest) 348 # 20 minutes by default 349 end_time = time.time() + timeout 350 351 _count1 = rest1.fetch_bucket_stats(bucket="testbucket")["op"]["samples"]["curr_items"][-1] 352 _count2 = rest2.fetch_bucket_stats(bucket="testbucket")["op"]["samples"]["curr_items"][-1] 353 while _count1 != _count2 and (time.time() - end_time) < 0: 354 self.log.info("Waiting for replication to catch up ..") 355 time.sleep(60) 356 _count1 = rest1.fetch_bucket_stats(bucket="testbucket")["op"]["samples"]["curr_items"][-1] 357 _count2 = rest2.fetch_bucket_stats(bucket="testbucket")["op"]["samples"]["curr_items"][-1] 358 if _count1 != _count2: 359 self.fail("not all items replicated in {0} sec for bucket: testbucket. on source cluster:{1}, on dest:{2}".\ 360 format(timeout, _count1, _count2)) 361 self.log.info("Replication caught up at {0}, for testbucket".format(_str_)) 362 363#BEGIN TEST 3 364 """ 365 Test that enagages clusters specified in XDCR, 366 verifies whether data went through as expected. 367 """ 368 def test_xdcr(self): 369 _rep_type = self.input.param("replication_type", "capi") # capi or xmem 370 _bixdcr = self.input.param("bidirectional", "false") 371 _clusters_dic = self.input.clusters 372 _src_nodes = copy.copy(_clusters_dic[0]) 373 _src_master = _src_nodes[0] 374 _dest_nodes = copy.copy(_clusters_dic[1]) 375 _dest_master = _dest_nodes[0] 376 377 # Build source cluster 378 self.init_rebalance_cluster_create_testbucket(_src_master, _src_nodes) 379 # Build destination cluster 380 self.init_rebalance_cluster_create_testbucket(_dest_master, _dest_nodes) 381 382 # Setting up XDCR 383 self.setup_xdcr_start_replication(_src_master, _dest_master, _rep_type, _bixdcr) 384 385 shell1 = RemoteMachineShellConnection(_src_master) 386 shell2 = RemoteMachineShellConnection(_dest_master) 387 src_item_count = 0 388 dest_item_count = 0 389 if self._os == "centos" or self._os == "ubuntu": 390 self.log.info("Load {0} through cbworkloadgen at src..".format(self.num_items)) 391 _1 = "cd /home/{0}/opt/couchbase &&".format(_src_master.ssh_username) 392 _2 = " ./bin/cbworkloadgen -n localhost:8091 --prefix=s_" 393 _3 = " -r .8 -i {0} -s 256 -b testbucket -t 1".format(self.num_items) 394 _4 = " -u {0} -p {1}".format(_src_master.rest_username, _src_master.rest_password) 395 command_to_load = _1 + _2 + _3 + _4 396 o, e = shell1.execute_non_sudo_command(command_to_load) 397 shell1.log_command_output(o, e) 398 time.sleep(20) 399 rest = RestConnection(_src_master) 400 src_item_count = rest.fetch_bucket_stats(bucket="testbucket")["op"]["samples"]["curr_items"][-1] 401 if _bixdcr: 402 self.log.info("Load {0} through cbworkloadgen at src..".format(self.num_items)) 403 _1 = "cd /home/{0}/opt/couchbase &&".format(_dest_master.ssh_username) 404 _2 = " ./bin/cbworkloadgen -n localhost:8091 --prefix=d_" 405 _3 = " -r .8 -i {0} -s 256 -b testbucket -t 1".format(self.num_items) 406 _4 = " -u {0} -p {1}".format(_dest_master.rest_username, _dest_master.rest_password) 407 command_to_load = _1 + _2 + _3 + _4 408 o, e = shell2.execute_non_sudo_command(command_to_load) 409 shell2.log_command_output(o, e) 410 time.sleep(20) 411 rest = RestConnection(_dest_master) 412 dest_item_count = rest.fetch_bucket_stats(bucket="testbucket")["op"]["samples"]["curr_items"][-1] 413 self.wait_for_replication_to_catchup(_src_master, _dest_master, 1200, "destination") 414 if _bixdcr: 415 self.wait_for_replication_to_catchup(_dest_master, _src_master, 1200, "source") 416 self.log.info("XDC REPLICATION caught up") 417 rest1 = RestConnection(_src_master) 418 rest2 = RestConnection(_dest_master) 419 curr_count_on_src = rest1.fetch_bucket_stats(bucket="testbucket")["op"]["samples"]["curr_items"][-1] 420 curr_count_on_dest = rest2.fetch_bucket_stats(bucket="testbucket")["op"]["samples"]["curr_items"][-1] 421 assert(curr_count_on_src==(src_item_count + dest_item_count), "ItemCount on source not what's expected") 422 assert(curr_count_on_dest==(src_item_count + dest_item_count), "ItemCount on destination not what's expected") 423 elif self._os == "windows": 424 # TODO: Windows support 425 self.log.info("Yet to add support for windows!") 426 pass 427 shell1.disconnect() 428 shell2.disconnect() 429#END TEST 3 430