1import json
2
3from base_2i import BaseSecondaryIndexingTests
4from membase.api.rest_client import RestConnection, RestHelper
5import random
6from lib import testconstants
7from lib.memcached.helper.data_helper import MemcachedClientHelper
8from lib.remote.remote_util import RemoteMachineShellConnection
9from threading import Thread
10from pytests.query_tests_helper import QueryHelperTests
11
12
13class GSIReplicaIndexesTests(BaseSecondaryIndexingTests, QueryHelperTests):
14    def setUp(self):
15        super(GSIReplicaIndexesTests, self).setUp()
16        self.rest = RestConnection(self.servers[0])
17        self.n1ql_server = self.get_nodes_from_services_map(
18            service_type="n1ql")
19        self.create_primary_index = False
20        shell = RemoteMachineShellConnection(self.servers[0])
21        info = shell.extract_remote_info().type.lower()
22        if info == 'linux':
23            self.cli_command_location = testconstants.LINUX_COUCHBASE_BIN_PATH
24            self.backup_path = testconstants.LINUX_BACKUP_PATH
25        elif info == 'windows':
26            self.cmd_ext = ".exe"
27            self.cli_command_location = testconstants.WIN_COUCHBASE_BIN_PATH_RAW
28            self.backup_path = testconstants.WIN_BACKUP_C_PATH
29        elif info == 'mac':
30            self.cli_command_location = testconstants.MAC_COUCHBASE_BIN_PATH
31            self.backup_path = testconstants.LINUX_BACKUP_PATH
32        else:
33            raise Exception("OS not supported.")
34
35        self.expected_err_msg = self.input.param("expected_err_msg", None)
36        self.nodes = self.input.param("nodes", None)
37        self.override_default_num_replica_with_num = self.input.param(
38            "override_with_num", 0)
39        self.override_default_num_replica_with_nodes = self.input.param(
40            "override_with_nodes", None)
41        if self.override_default_num_replica_with_nodes:
42            self.nodes = self.override_default_num_replica_with_nodes
43        self.node_out = self.input.param("node_out", 0)
44        self.server_grouping = self.input.param("server_grouping", None)
45        self.eq_index_node = self.input.param("eq_index_node", None)
46        self.recovery_type = self.input.param("recovery_type", None)
47        self.dest_node = self.input.param("dest_node", None)
48        self.rand = random.randint(1, 1000000000)
49        self.expected_nodes = self.input.param("expected_nodes", None)
50        self.failure_in_node = self.input.param("failure_in_node", None)
51        self.alter_index = self.input.param("alter_index",None)
52
53    def tearDown(self):
54        super(GSIReplicaIndexesTests, self).tearDown()
55
56    def test_create_replica_index_with_num_replica(self):
57        index_name_prefix = "random_index_" + str(
58            random.randint(100000, 999999))
59        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'num_replica': {0}}};".format(
60            self.num_index_replicas)
61        try:
62            self.n1ql_helper.run_cbq_query(query=create_index_query,
63                                           server=self.n1ql_node)
64        except Exception, ex:
65            self.log.info(str(ex))
66            if self.expected_err_msg not in str(ex):
67                self.fail(
68                    "index creation did not fail with expected error : {0}".format(
69                        str(ex)))
70            else:
71                self.log.info("Index creation failed as expected")
72        self.sleep(30)
73        index_map = self.get_index_map()
74        self.log.info(index_map)
75        if not self.expected_err_msg:
76            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
77                                                    index_map,
78                                                    self.num_index_replicas)
79
80    def test_create_replica_index_one_failed_node_num_replica(self):
81        node_out = self.servers[self.node_out]
82        failover_task = self.cluster.async_failover(
83            self.servers[:self.nodes_init],
84            [node_out],
85            self.graceful, wait_for_pending=180)
86
87        failover_task.result()
88
89        index_name_prefix = "random_index_" + str(
90            random.randint(100000, 999999))
91        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'num_replica': {0}}};".format(
92            self.num_index_replicas)
93        try:
94            self.n1ql_helper.run_cbq_query(query=create_index_query,
95                                           server=self.n1ql_node)
96        except Exception, ex:
97            self.log.info(str(ex))
98            if self.expected_err_msg not in str(ex):
99                self.fail(
100                    "index creation did not fail with expected error : {0}".format(
101                        str(ex)))
102            else:
103                self.log.info("Index creation failed as expected")
104        self.sleep(30)
105        index_map = self.get_index_map()
106        self.log.info(index_map)
107        if not self.expected_err_msg:
108            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
109                                                    index_map,
110                                                    self.num_index_replicas)
111
112    def test_failover_during_create_index_with_replica(self):
113        node_out = self.servers[self.node_out]
114        index_name_prefix = "random_index_" + str(
115            random.randint(100000, 999999))
116        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'num_replica': {0}, 'defer_build': true}};".format(
117            self.num_index_replicas)
118
119        threads = [
120            Thread(target=self.n1ql_helper.run_cbq_query, name="run_query",
121                   args=(create_index_query, 10, self.n1ql_node)),
122            Thread(target=self.cluster.async_failover, name="failover", args=(
123                self.servers[:self.nodes_init], [node_out], self.graceful))]
124        for thread in threads:
125            thread.start()
126            thread.join()
127        self.sleep(30)
128        index_map = self.get_index_map()
129        self.log.info(index_map)
130
131        try:
132            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
133                                                    index_map,
134                                                    self.num_index_replicas)
135        except Exception, ex:
136            self.log.info(str(ex))
137            if self.expected_err_msg not in str(ex):
138                self.fail(
139                    "index creation did not fail with expected error : {0}".format(
140                        str(ex)))
141            else:
142                self.log.info("Index creation failed as expected")
143
144    def test_create_replica_index_with_node_list(self):
145        nodes = self._get_node_list()
146        self.log.info(nodes)
147        index_name_prefix = "random_index_" + str(
148            random.randint(100000, 999999))
149        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'nodes': {0}}};".format(
150            nodes)
151        self.log.info(create_index_query)
152        try:
153            self.n1ql_helper.run_cbq_query(query=create_index_query,
154                                           server=self.n1ql_node)
155        except Exception, ex:
156            self.log.info(str(ex))
157            if self.expected_err_msg not in str(ex):
158                self.fail(
159                    "index creation did not fail with expected error : {0}".format(
160                        str(ex)))
161            else:
162                self.log.info("Index creation failed as expected")
163        self.sleep(30)
164        index_map = self.get_index_map()
165        self.log.info(index_map)
166        if not self.expected_err_msg:
167            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
168                                                    index_map,
169                                                    len(nodes) - 1, nodes)
170
171    '''Due to a bug you could force the indexer to ignore the default replica settings, ensure that the nodelist
172       must contain enough nodes for the min number of replicas set'''
173    def test_neg_create_replica_index_with_node_list(self):
174        node_1 = self.servers[1].ip + ":" + self.servers[1].port
175        node_2 = self.servers[2].ip + ":" + self.servers[2].port
176
177        rest = RestConnection(self.servers[1])
178        rest.set_index_settings({"indexer.settings.num_replica":2})
179        self.sleep(10)
180
181        index_name_prefix = "random_index_" + str(
182            random.randint(100000, 999999))
183        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'nodes':['{0}','{1}']}};".format(node_1,node_2)
184        self.log.info(create_index_query)
185        try:
186            self.n1ql_helper.run_cbq_query(query=create_index_query,
187                                           server=self.n1ql_node)
188        except Exception, ex:
189            self.log.info(str(ex))
190            if self.expected_err_msg not in str(ex):
191                self.fail(
192                    "index creation did not fail with expected error : {0}".format(
193                        str(ex)))
194            else:
195                self.log.info("Index creation failed as expected")
196
197    def test_create_replica_index_one_failed_node_with_node_list(self):
198        node_out = self.servers[self.node_out]
199        failover_task = self.cluster.async_failover(
200            self.servers[:self.nodes_init],
201            [node_out],
202            self.graceful, wait_for_pending=180)
203
204        failover_task.result()
205
206        nodes = self._get_node_list()
207        self.log.info(nodes)
208        index_name_prefix = "random_index_" + str(
209            random.randint(100000, 999999))
210        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'nodes': {0}}};".format(
211            nodes)
212        self.log.info(create_index_query)
213        try:
214            self.n1ql_helper.run_cbq_query(query=create_index_query,
215                                           server=self.n1ql_node)
216        except Exception, ex:
217            self.log.info(str(ex))
218            if self.expected_err_msg not in str(ex):
219                self.fail(
220                    "index creation did not fail with expected error : {0}".format(
221                        str(ex)))
222            else:
223                self.log.info("Index creation failed as expected")
224        self.sleep(30)
225        index_map = self.get_index_map()
226        self.log.info(index_map)
227        if not self.expected_err_msg:
228            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
229                                                    index_map,
230                                                    len(nodes) - 1, nodes)
231
232    def test_create_replica_index_with_num_replicas_and_node_list(self):
233        nodes = self._get_node_list()
234        self.log.info(nodes)
235        index_name_prefix = "random_index_" + str(
236            random.randint(100000, 999999))
237        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'num_replica': {0},'nodes': {1}}};".format(
238            self.num_index_replicas, nodes)
239        self.log.info(create_index_query)
240        try:
241            self.n1ql_helper.run_cbq_query(query=create_index_query,
242                                           server=self.n1ql_node)
243        except Exception, ex:
244            self.log.info(str(ex))
245            if self.expected_err_msg not in str(ex):
246                self.fail(
247                    "index creation did not fail with expected error : {0}".format(
248                        str(ex)))
249            else:
250                self.log.info("Index creation failed as expected")
251        self.sleep(30)
252        index_map = self.get_index_map()
253        self.log.info(index_map)
254        if not self.expected_err_msg:
255            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
256                                                    index_map,
257                                                    self.num_index_replicas,
258                                                    nodes)
259
260    def test_create_replica_index_with_server_groups(self):
261        nodes = self._get_node_list()
262        self.log.info(nodes)
263
264        self._create_server_groups()
265        self.sleep(5)
266        index_name_prefix = "random_index_" + str(
267            random.randint(100000, 999999))
268        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'num_replica': {0}}};".format(
269            self.num_index_replicas)
270        self.log.info(create_index_query)
271        try:
272            self.n1ql_helper.run_cbq_query(query=create_index_query,
273                                           server=self.n1ql_node)
274        except Exception, ex:
275            self.log.info(str(ex))
276            self.fail("Index creation Failed : %s", str(ex))
277
278        self.sleep(30)
279        index_map = self.get_index_map()
280
281        self.log.info(index_map)
282        if not self.expected_err_msg:
283            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
284                                                    index_map,
285                                                    self.num_index_replicas)
286
287    def test_create_index_while_another_index_building(self):
288        index_name_age = "age_index_" + str(
289            random.randint(100000, 999999))
290        index_name_name = "name_index_" + str(
291            random.randint(100000, 999999))
292        create_index_query_age = "CREATE INDEX " + index_name_age + " ON default(age) USING GSI  WITH {{'num_replica': {0}, 'defer_build': true}};".format(
293            self.num_index_replicas)
294        create_index_query_name = "CREATE INDEX " + index_name_name + " ON default(name) USING GSI  WITH {{'num_replica': {0}}};".format(
295            self.num_index_replicas)
296
297        try:
298            self.n1ql_helper.run_cbq_query(query=create_index_query_age,
299                                           server=self.n1ql_node)
300        except Exception, ex:
301            self.log.info(str(ex))
302            self.fail("Index creation Failed : %s", str(ex))
303
304        build_index_query_age = "BUILD INDEX on `default`(" + index_name_age + ")"
305
306        threads = [
307            Thread(target=self.n1ql_helper.run_cbq_query, name="run_query",
308                   args=(build_index_query_age, 10, self.n1ql_node)),
309            Thread(target=self.n1ql_helper.run_cbq_query, name="run_query",
310                   args=(create_index_query_name, 10, self.n1ql_node))]
311
312        for thread in threads:
313            thread.start()
314            self.sleep(1)
315
316        for thread in threads:
317            thread.join()
318
319        self.sleep(120)
320        index_map = self.get_index_map()
321        self.log.info(index_map)
322
323        try:
324            self.n1ql_helper.verify_replica_indexes([index_name_age],
325                                                    index_map,
326                                                    self.num_index_replicas)
327            self.n1ql_helper.verify_replica_indexes([index_name_name],
328                                                    index_map,
329                                                    self.num_index_replicas)
330        except Exception, ex:
331            self.log.info(str(ex))
332            if self.expected_err_msg not in str(ex):
333                self.fail(
334                    "index creation did not fail with expected error : {0}".format(
335                        str(ex)))
336            else:
337                self.log.info("Index creation failed as expected")
338
339    def test_default_num_indexes(self):
340        self.rest.set_indexer_num_replica(self.num_index_replicas)
341        nodes = self._get_node_list()
342        self.log.info(nodes)
343        index_name_prefix = "random_index_" + str(
344            random.randint(100000, 999999))
345        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age)"
346
347        if self.override_default_num_replica_with_nodes and self.override_default_num_replica_with_num:
348            create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'num_replica': {0},'nodes': {1}}};".format(
349                self.override_default_num_replica_with_num, nodes)
350
351        elif self.override_default_num_replica_with_nodes:
352            create_index_query += "USING GSI  WITH {{'nodes': {0}}};".format(
353                nodes)
354
355        elif self.override_default_num_replica_with_num:
356            create_index_query += "USING GSI  WITH {{'num_replica': {0}}};".format(
357                self.override_default_num_replica_with_num)
358
359        self.log.info(create_index_query)
360        try:
361            self.n1ql_helper.run_cbq_query(query=create_index_query,
362                                           server=self.n1ql_node)
363        except Exception, ex:
364            self.log.info(str(ex))
365            if self.expected_err_msg not in str(ex):
366                self.fail(
367                    "index creation did not fail with expected error : {0}".format(
368                        str(ex)))
369            else:
370                self.log.info("Index creation failed as expected")
371        self.sleep(30)
372        index_map = self.get_index_map()
373        self.log.info(index_map)
374        expected_num_replicas = self.num_index_replicas
375        if self.override_default_num_replica_with_num > 0:
376            expected_num_replicas = self.override_default_num_replica_with_num
377        if not self.expected_err_msg:
378            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
379                                                    index_map,
380                                                    expected_num_replicas,
381                                                    nodes)
382
383        # Reset the default value for num_replica
384        self.rest.set_indexer_num_replica(0)
385
386    def test_build_index_with_replica(self):
387        index_name_prefix = "random_index_" + str(
388            random.randint(100000, 999999))
389        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'num_replica': {0}, 'defer_build': true}};".format(
390            self.num_index_replicas)
391        try:
392            self.n1ql_helper.run_cbq_query(query=create_index_query,
393                                           server=self.n1ql_node)
394        except Exception, ex:
395            self.log.info(str(ex))
396            if self.expected_err_msg not in str(ex):
397                self.fail(
398                    "index creation did not fail with expected error : {0}".format(
399                        str(ex)))
400            else:
401                self.log.info("Index creation failed as expected")
402        self.sleep(30)
403        index_map = self.get_index_map()
404        self.log.info(index_map)
405        if not self.expected_err_msg:
406            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
407                                                    index_map,
408                                                    self.num_index_replicas)
409
410        self.n1ql_helper.verify_replica_indexes_build_status(index_map,
411                                                             self.num_index_replicas,
412                                                             defer_build=True)
413
414        build_index_query = "BUILD INDEX on `default`(" + index_name_prefix + ")"
415
416        try:
417            self.n1ql_helper.run_cbq_query(query=build_index_query,
418                                           server=self.n1ql_node)
419        except Exception, ex:
420            self.log.info(str(ex))
421            if self.expected_err_msg not in str(ex):
422                self.fail(
423                    "index building did not fail with expected error : {0}".format(
424                        str(ex)))
425            else:
426                self.log.info("Index building failed as expected")
427
428        self.sleep(30)
429        index_map = self.get_index_map()
430        self.n1ql_helper.verify_replica_indexes_build_status(index_map,
431                                                             self.num_index_replicas,
432                                                             defer_build=False)
433
434    def test_build_index_with_replica_one_failed_node(self):
435        nodes = self._get_node_list()
436        self.log.info(nodes)
437        index_name_prefix = "random_index_" + str(
438            random.randint(100000, 999999))
439        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'nodes': {0}, 'defer_build': true}};".format(
440            nodes)
441        self.log.info(create_index_query)
442        try:
443            self.n1ql_helper.run_cbq_query(query=create_index_query,
444                                           server=self.n1ql_node)
445        except Exception, ex:
446            self.log.info(str(ex))
447            if self.expected_err_msg not in str(ex):
448                self.fail(
449                    "index creation did not fail with expected error : {0}".format(
450                        str(ex)))
451            else:
452                self.log.info("Index creation failed as expected")
453        self.sleep(30)
454        index_map = self.get_index_map()
455        self.log.info(index_map)
456        if not self.expected_err_msg:
457            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
458                                                    index_map,
459                                                    len(nodes) - 1, nodes)
460
461        self.n1ql_helper.verify_replica_indexes_build_status(index_map,
462                                                             len(nodes) - 1,
463                                                             defer_build=True)
464
465        node_out = self.servers[self.node_out]
466        failover_task = self.cluster.async_failover(
467            self.servers[:self.nodes_init],
468            [node_out],
469            self.graceful, wait_for_pending=180)
470
471        failover_task.result()
472
473        build_index_query = "BUILD INDEX on `default`(" + index_name_prefix + ")"
474
475        try:
476            self.n1ql_helper.run_cbq_query(query=build_index_query,
477                                           server=self.n1ql_node)
478        except Exception, ex:
479            self.log.info(str(ex))
480            if self.expected_err_msg not in str(ex):
481                self.fail(
482                    "index building did not fail with expected error : {0}".format(
483                        str(ex)))
484            else:
485                self.log.info("Index building failed as expected")
486
487        self.sleep(30)
488        index_map = self.get_index_map()
489        try:
490            self.n1ql_helper.verify_replica_indexes_build_status(index_map,
491                                                                 len(nodes) - 1,
492                                                                 defer_build=False)
493        except Exception, ex:
494            self.log.info(str(ex))
495            if self.expected_err_msg not in str(ex):
496                self.fail(
497                    "index building did not fail with expected error : {0}".format(
498                        str(ex)))
499            else:
500                self.log.info("Index building failed as expected")
501
502    def test_failover_during_build_index(self):
503        nodes = self._get_node_list()
504        self.log.info(nodes)
505        index_name_prefix = "random_index_" + str(
506            random.randint(100000, 999999))
507        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'nodes': {0}, 'defer_build': true}};".format(
508            nodes)
509        self.log.info(create_index_query)
510        try:
511            self.n1ql_helper.run_cbq_query(query=create_index_query,
512                                           server=self.n1ql_node)
513        except Exception, ex:
514            self.log.info(str(ex))
515            if self.expected_err_msg not in str(ex):
516                self.fail(
517                    "index creation did not fail with expected error : {0}".format(
518                        str(ex)))
519            else:
520                self.log.info("Index creation failed as expected")
521        self.sleep(30)
522        index_map = self.get_index_map()
523        self.log.info(index_map)
524        if not self.expected_err_msg:
525            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
526                                                    index_map,
527                                                    len(nodes) - 1, nodes)
528
529        self.n1ql_helper.verify_replica_indexes_build_status(index_map,
530                                                             len(nodes) - 1,
531                                                             defer_build=True)
532
533        node_out = self.servers[self.node_out]
534        build_index_query = "BUILD INDEX on `default`(" + index_name_prefix + ")"
535        threads = []
536        threads.append(
537            Thread(target=self.n1ql_helper.run_cbq_query, name="run_query",
538                   args=(build_index_query, 10, self.n1ql_node)))
539        threads.append(
540            Thread(target=self.cluster.async_failover, name="failover", args=(
541                self.servers[:self.nodes_init], [node_out], self.graceful)))
542        for thread in threads:
543            thread.start()
544            thread.join()
545        self.sleep(30)
546
547        index_map = self.get_index_map()
548        try:
549            self.n1ql_helper.verify_replica_indexes_build_status(index_map,
550                                                                 len(nodes) - 1,
551                                                                 defer_build=False)
552        except Exception, ex:
553            self.log.info(str(ex))
554            if self.expected_err_msg not in str(ex):
555                self.fail(
556                    "index building did not fail with expected error : {0}".format(
557                        str(ex)))
558            else:
559                self.log.info("Index building failed as expected")
560
561    def test_build_index_with_replica_failover_addback(self):
562        nodes = self._get_node_list()
563        self.log.info(nodes)
564        index_name_prefix = "random_index_" + str(
565            random.randint(100000, 999999))
566        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'nodes': {0}, 'defer_build': true}};".format(
567            nodes)
568        self.log.info(create_index_query)
569        try:
570            self.n1ql_helper.run_cbq_query(query=create_index_query,
571                                           server=self.n1ql_node)
572        except Exception, ex:
573            self.log.info(str(ex))
574            if self.expected_err_msg not in str(ex):
575                self.fail(
576                    "index creation did not fail with expected error : {0}".format(
577                        str(ex)))
578            else:
579                self.log.info("Index creation failed as expected")
580        self.sleep(30)
581        index_map = self.get_index_map()
582        self.log.info(index_map)
583        if not self.expected_err_msg:
584            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
585                                                    index_map,
586                                                    len(nodes) - 1, nodes)
587
588        self.n1ql_helper.verify_replica_indexes_build_status(index_map,
589                                                             len(nodes) - 1,
590                                                             defer_build=True)
591
592        node_out = self.servers[self.node_out]
593        failover_task = self.cluster.async_failover(
594            self.servers[:self.nodes_init],
595            [node_out],
596            self.graceful, wait_for_pending=180)
597
598        failover_task.result()
599
600        build_index_query = "BUILD INDEX on `default`(" + index_name_prefix + ")"
601
602        try:
603            self.n1ql_helper.run_cbq_query(query=build_index_query,
604                                           server=self.n1ql_node)
605        except Exception, ex:
606            self.log.info(str(ex))
607            if self.expected_err_msg not in str(ex):
608                self.fail(
609                    "index building did not fail with expected error : {0}".format(
610                        str(ex)))
611            else:
612                self.log.info("Index building failed as expected")
613
614        self.sleep(30)
615
616        nodes_all = self.rest.node_statuses()
617        for node in nodes_all:
618            if node.ip == node_out.ip:
619                break
620
621        self.rest.set_recovery_type(node.id, self.recovery_type)
622        self.rest.add_back_node(node.id)
623
624        rebalance = self.cluster.async_rebalance(self.servers[:self.nodes_init],
625                                                 [], [])
626        reached = RestHelper(self.rest).rebalance_reached()
627        self.assertTrue(reached, "rebalance failed, stuck or did not complete")
628        rebalance.result()
629        self.sleep(180)
630
631        index_map = self.get_index_map()
632        self.n1ql_helper.verify_replica_indexes_build_status(index_map,
633                                                             len(nodes) - 1,
634                                                             defer_build=False)
635
636    def test_build_index_with_network_partitioning(self):
637        nodes = self._get_node_list()
638        self.log.info(nodes)
639        index_name_prefix = "random_index_" + str(
640            random.randint(100000, 999999))
641        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'nodes': {0}, 'defer_build': true}};".format(
642            nodes)
643        self.log.info(create_index_query)
644        try:
645            self.n1ql_helper.run_cbq_query(query=create_index_query,
646                                           server=self.n1ql_node)
647        except Exception, ex:
648            self.log.info(str(ex))
649            if self.expected_err_msg not in str(ex):
650                self.fail(
651                    "index creation did not fail with expected error : {0}".format(
652                        str(ex)))
653            else:
654                self.log.info("Index creation failed as expected")
655        self.sleep(30)
656        index_map = self.get_index_map()
657        self.log.info(index_map)
658        if not self.expected_err_msg:
659            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
660                                                    index_map,
661                                                    len(nodes) - 1, nodes)
662
663        self.n1ql_helper.verify_replica_indexes_build_status(index_map,
664                                                             len(nodes) - 1,
665                                                             defer_build=True)
666
667        node_out = self.servers[self.node_out]
668        build_index_query = "BUILD INDEX on `default`(" + index_name_prefix + ")"
669
670        try:
671            self.start_firewall_on_node(node_out)
672            self.sleep(10)
673            self.n1ql_helper.run_cbq_query(query=build_index_query,
674                                           server=self.n1ql_node)
675        except Exception, ex:
676            self.log.info(str(ex))
677            if self.expected_err_msg not in str(ex):
678                self.fail(
679                    "index building did not fail with expected error : {0}".format(
680                        str(ex)))
681            else:
682                self.log.info("Index building failed as expected")
683
684        finally:
685            # Heal network partition and wait for some time to allow indexes
686            # to get built automatically on that node
687            self.stop_firewall_on_node(node_out)
688            self.sleep(360)
689
690            index_map = self.get_index_map()
691            self.n1ql_helper.verify_replica_indexes_build_status(index_map,
692                                                                 len(nodes) - 1,
693                                                                 defer_build=False)
694
695
696    def test_build_index_while_another_index_building(self):
697        index_name_age = "age_index_" + str(
698            random.randint(100000, 999999))
699        index_name_name = "name_index_" + str(
700            random.randint(100000, 999999))
701        create_index_query_age = "CREATE INDEX " + index_name_age + " ON default(age) USING GSI  WITH {{'num_replica': {0}, 'defer_build':true}};".format(
702            self.num_index_replicas)
703        create_index_query_name = "CREATE INDEX " + index_name_name + " ON default(name) USING GSI  WITH {{'num_replica': {0}, 'defer_build':true}};".format(
704            self.num_index_replicas)
705
706        self.n1ql_helper.run_cbq_query(query=create_index_query_age,
707                                       server=self.n1ql_node)
708
709        self.n1ql_helper.run_cbq_query(query=create_index_query_name,
710                                       server=self.n1ql_node)
711
712        self.sleep(30)
713        index_map = self.get_index_map()
714        self.log.info(index_map)
715
716        try:
717            self.n1ql_helper.verify_replica_indexes([index_name_age],
718                                                    index_map,
719                                                    self.num_index_replicas)
720            self.n1ql_helper.verify_replica_indexes([index_name_name],
721                                                    index_map,
722                                                    self.num_index_replicas)
723        except Exception, ex:
724            self.log.info(str(ex))
725            if self.expected_err_msg not in str(ex):
726                self.fail(
727                    "index creation did not fail with expected error : {0}".format(
728                        str(ex)))
729            else:
730                self.log.info("Index creation failed as expected")
731
732        build_index_query_age = "BUILD INDEX on `default`(" + index_name_age + ")"
733        build_index_query_name = "BUILD INDEX on `default`(" + index_name_name + ")"
734
735        threads = [
736            Thread(target=self.n1ql_helper.run_cbq_query, name="run_query",
737                   args=(build_index_query_age, 10, self.n1ql_node)),
738            Thread(target=self.n1ql_helper.run_cbq_query, name="run_query",
739                   args=(build_index_query_name, 10, self.n1ql_node))]
740
741        for thread in threads:
742            thread.start()
743            self.sleep(1)
744
745        for thread in threads:
746            thread.join()
747
748        # Index building in the background can take longer, so wait for some time
749        self.sleep(360)
750
751        try:
752            index_map = self.get_index_map()
753            self.log.info(index_map)
754            self.n1ql_helper.verify_replica_indexes_build_status(index_map,
755                                                                 self.num_index_replicas,
756                                                                 defer_build=False)
757        except Exception, ex:
758            self.log.info(str(ex))
759            if self.expected_err_msg not in str(ex):
760                self.fail(
761                    "index building did not fail with expected error : {0}".format(
762                        str(ex)))
763            else:
764                self.log.info("Index building failed as expected")
765
766    def test_drop_index_with_replica(self):
767        index_name_prefix = "random_index_" + str(
768            random.randint(100000, 999999))
769        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'num_replica': {0}, 'defer_build': {1}}};".format(
770            self.num_index_replicas, self.defer_build)
771        try:
772            self.n1ql_helper.run_cbq_query(query=create_index_query,
773                                           server=self.n1ql_node)
774        except Exception, ex:
775            self.log.info(str(ex))
776            if self.expected_err_msg not in str(ex):
777                self.fail(
778                    "index creation did not fail with expected error : {0}".format(
779                        str(ex)))
780            else:
781                self.log.info("Index creation failed as expected")
782        self.sleep(30)
783        index_map = self.get_index_map()
784        self.log.info(index_map)
785        if not self.expected_err_msg:
786            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
787                                                    index_map,
788                                                    self.num_index_replicas)
789
790        drop_index_query = "DROP INDEX `default`." + index_name_prefix
791
792        try:
793            self.n1ql_helper.run_cbq_query(query=drop_index_query,
794                                           server=self.n1ql_node)
795        except Exception, ex:
796            self.log.info(str(ex))
797            if self.expected_err_msg not in str(ex):
798                self.fail(
799                    "Drop index did not fail with expected error : {0}".format(
800                        str(ex)))
801            else:
802                self.log.info("Drop index failed as expected")
803
804        self.sleep(30)
805        index_map = self.get_index_map()
806        self.log.info("Index map after drop index: %s", index_map)
807        if not index_map == {}:
808            self.fail("Indexes not dropped correctly")
809
810    def test_drop_index_with_replica_one_failed_node(self):
811        index_name_prefix = "random_index_" + str(
812            random.randint(100000, 999999))
813        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'num_replica': {0}, 'defer_build': {1}}};".format(
814            self.num_index_replicas, self.defer_build)
815        try:
816            self.n1ql_helper.run_cbq_query(query=create_index_query,
817                                           server=self.n1ql_node)
818        except Exception, ex:
819            self.log.info(str(ex))
820            if self.expected_err_msg not in str(ex):
821                self.fail(
822                    "index creation did not fail with expected error : {0}".format(
823                        str(ex)))
824            else:
825                self.log.info("Index creation failed as expected")
826        self.sleep(30)
827        index_map = self.get_index_map()
828        self.log.info(index_map)
829        if not self.expected_err_msg:
830            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
831                                                    index_map,
832                                                    self.num_index_replicas)
833
834        node_out = self.servers[self.node_out]
835        failover_task = self.cluster.async_failover(
836            self.servers[:self.nodes_init],
837            [node_out],
838            self.graceful, wait_for_pending=180)
839
840        failover_task.result()
841
842        drop_index_query = "DROP INDEX `default`." + index_name_prefix
843
844        try:
845            self.n1ql_helper.run_cbq_query(query=drop_index_query,
846                                           server=self.n1ql_node)
847        except Exception, ex:
848            self.log.info(str(ex))
849            if self.expected_err_msg not in str(ex):
850                self.fail(
851                    "Drop index did not fail with expected error : {0}".format(
852                        str(ex)))
853            else:
854                self.log.info("Drop index failed as expected")
855
856        self.sleep(30)
857        index_map = self.get_index_map()
858        self.log.info("Index map after drop index: %s", index_map)
859        if not index_map == {}:
860            self.fail("Indexes not dropped correctly")
861
862    def test_failover_during_drop_index(self):
863        index_name_prefix = "random_index_" + str(
864            random.randint(100000, 999999))
865        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'num_replica': {0}}};".format(
866            self.num_index_replicas)
867        try:
868            self.n1ql_helper.run_cbq_query(query=create_index_query,
869                                           server=self.n1ql_node)
870        except Exception, ex:
871            self.log.info(str(ex))
872            if self.expected_err_msg not in str(ex):
873                self.fail(
874                    "index creation did not fail with expected error : {0}".format(
875                        str(ex)))
876            else:
877                self.log.info("Index creation failed as expected")
878        self.sleep(30)
879        index_map = self.get_index_map()
880        self.log.info(index_map)
881        if not self.expected_err_msg:
882            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
883                                                    index_map,
884                                                    self.num_index_replicas)
885
886        node_out = self.servers[self.node_out]
887        drop_index_query = "DROP INDEX `default`." + index_name_prefix
888
889        threads = []
890        threads.append(
891            Thread(target=self.n1ql_helper.run_cbq_query, name="run_query",
892                   args=(drop_index_query, 10, self.n1ql_node)))
893        threads.append(
894            Thread(target=self.cluster.async_failover, name="failover", args=(
895                self.servers[:self.nodes_init], [node_out], self.graceful)))
896
897        for thread in threads:
898            thread.start()
899        self.sleep(5)
900        for thread in threads:
901            thread.join()
902
903        self.sleep(30)
904        index_map = self.get_index_map()
905        self.log.info("Index map after drop index: %s", index_map)
906        if not index_map == {}:
907            self.fail("Indexes not dropped correctly")
908
909    def test_drop_index_with_replica_failover_addback(self):
910        index_name_prefix = "random_index_" + str(
911            random.randint(100000, 999999))
912        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'num_replica': {0}, 'defer_build': {1}}};".format(
913            self.num_index_replicas, self.defer_build)
914        try:
915            self.n1ql_helper.run_cbq_query(query=create_index_query,
916                                           server=self.n1ql_node)
917        except Exception, ex:
918            self.log.info(str(ex))
919            if self.expected_err_msg not in str(ex):
920                self.fail(
921                    "index creation did not fail with expected error : {0}".format(
922                        str(ex)))
923            else:
924                self.log.info("Index creation failed as expected")
925        self.sleep(30)
926        index_map = self.get_index_map()
927        self.log.info(index_map)
928        if not self.expected_err_msg:
929            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
930                                                    index_map,
931                                                    self.num_index_replicas)
932
933        node_out = self.servers[self.node_out]
934        failover_task = self.cluster.async_failover(
935            self.servers[:self.nodes_init],
936            [node_out],
937            self.graceful, wait_for_pending=180)
938
939        failover_task.result()
940
941        drop_index_query = "DROP INDEX `default`." + index_name_prefix
942
943        try:
944            self.n1ql_helper.run_cbq_query(query=drop_index_query,
945                                           server=self.n1ql_node)
946        except Exception, ex:
947            self.log.info(str(ex))
948            if self.expected_err_msg not in str(ex):
949                self.fail(
950                    "Drop index did not fail with expected error : {0}".format(
951                        str(ex)))
952            else:
953                self.log.info("Drop index failed as expected")
954
955        self.sleep(30)
956        index_map = self.get_index_map()
957        self.log.info("Index map after drop index: %s", index_map)
958        if not index_map == {}:
959            self.fail("Indexes not dropped correctly")
960
961        nodes_all = self.rest.node_statuses()
962        for node in nodes_all:
963            if node.ip == node_out.ip:
964                break
965
966        self.rest.set_recovery_type(node.id, self.recovery_type)
967        self.rest.add_back_node(node.id)
968
969        rebalance = self.cluster.async_rebalance(self.servers[:self.nodes_init],
970                                                 [], [])
971        reached = RestHelper(self.rest).rebalance_reached()
972        self.assertTrue(reached, "rebalance failed, stuck or did not complete")
973        rebalance.result()
974        self.sleep(180)
975
976        index_map = self.get_index_map()
977        self.log.info("Index map after drop index: %s", index_map)
978        if not index_map == {}:
979            self.fail("Indexes not dropped correctly")
980
981    def test_drop_index_with_network_partitioning(self):
982        index_name_prefix = "random_index_" + str(
983            random.randint(100000, 999999))
984        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'num_replica': {0}, 'defer_build': {1}}};".format(
985            self.num_index_replicas, self.defer_build)
986        try:
987            self.n1ql_helper.run_cbq_query(query=create_index_query,
988                                           server=self.n1ql_node)
989        except Exception, ex:
990            self.log.info(str(ex))
991            if self.expected_err_msg not in str(ex):
992                self.fail(
993                    "index creation did not fail with expected error : {0}".format(
994                        str(ex)))
995            else:
996                self.log.info("Index creation failed as expected")
997        self.sleep(30)
998        index_map = self.get_index_map()
999        self.log.info(index_map)
1000        if not self.expected_err_msg:
1001            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
1002                                                    index_map,
1003                                                    self.num_index_replicas)
1004
1005        node_out = self.servers[self.node_out]
1006        drop_index_query = "DROP INDEX `default`." + index_name_prefix
1007
1008        try:
1009            self.start_firewall_on_node(node_out)
1010            self.sleep(10)
1011            self.n1ql_helper.run_cbq_query(query=drop_index_query,
1012                                           server=self.n1ql_node)
1013        except Exception, ex:
1014            self.log.info(str(ex))
1015            if self.expected_err_msg not in str(ex):
1016                self.fail(
1017                    "Drop index did not fail with expected error : {0}".format(
1018                        str(ex)))
1019            else:
1020                self.log.info("Drop index failed as expected")
1021
1022        finally:
1023            # Heal network partition and wait for some time to allow indexes
1024            # on the node to get dropped automatically
1025            self.stop_firewall_on_node(node_out)
1026            self.sleep(180)
1027
1028            index_map = self.get_index_map()
1029            self.log.info("Index map after drop index: %s", index_map)
1030            if not index_map == {}:
1031                self.fail("Indexes not dropped correctly")
1032
1033    def test_replica_movement_with_rebalance_out(self):
1034        nodes = self._get_node_list()
1035        self.log.info(nodes)
1036        index_name_prefix = "random_index_" + str(
1037            random.randint(100000, 999999))
1038        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'nodes': {0}}};".format(
1039            nodes)
1040        self.log.info(create_index_query)
1041        try:
1042            self.n1ql_helper.run_cbq_query(query=create_index_query,
1043                                           server=self.n1ql_node)
1044        except Exception, ex:
1045            self.log.info(str(ex))
1046            self.fail("Index creation Failed : %s", str(ex))
1047
1048        self.sleep(30)
1049        index_map_before_rebalance = self.get_index_map()
1050        stats_map_before_rebalance = self.get_index_stats(perNode=False)
1051
1052        self.log.info(index_map_before_rebalance)
1053        if not self.expected_err_msg:
1054            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
1055                                                    index_map_before_rebalance,
1056                                                    len(nodes) - 1, nodes)
1057
1058        node_out = self.servers[self.node_out]
1059        rebalance = self.cluster.async_rebalance(self.servers[:self.nodes_init],
1060                                                 [], [node_out])
1061        reached = RestHelper(self.rest).rebalance_reached()
1062        self.assertTrue(reached, "rebalance failed, stuck or did not complete")
1063        rebalance.result()
1064        self.sleep(30)
1065
1066        index_map_after_rebalance = self.get_index_map()
1067        stats_map_after_rebalance = self.get_index_stats(perNode=False)
1068
1069        try:
1070            self.n1ql_helper.verify_indexes_redistributed(
1071                index_map_before_rebalance,
1072                index_map_after_rebalance,
1073                stats_map_before_rebalance,
1074                stats_map_after_rebalance,
1075                [],
1076                [node_out])
1077        except Exception, ex:
1078            self.log.info(str(ex))
1079            if self.expected_err_msg not in str(ex):
1080                self.fail(
1081                    "Error in index distribution post rebalance : ".format(
1082                        str(ex)))
1083            else:
1084                self.log.info(str(ex))
1085
1086    def test_dropped_replica_add_new_node(self):
1087        nodes = self._get_node_list()
1088        self.log.info(nodes)
1089        index_name_prefix = "random_index_" + str(
1090            random.randint(100000, 999999))
1091        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'nodes': {0}}};".format(
1092            nodes)
1093        self.log.info(create_index_query)
1094        try:
1095            self.n1ql_helper.run_cbq_query(query=create_index_query,
1096                                           server=self.n1ql_node)
1097        except Exception, ex:
1098            self.log.info(str(ex))
1099            self.fail("Index creation Failed : %s", str(ex))
1100
1101        self.sleep(30)
1102        index_map_before_rebalance = self.get_index_map()
1103        stats_map_before_rebalance = self.get_index_stats(perNode=False)
1104
1105        self.log.info(index_map_before_rebalance)
1106        if not self.expected_err_msg:
1107            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
1108                                                    index_map_before_rebalance,
1109                                                    len(nodes) - 1, nodes)
1110
1111        node_out = self.servers[self.node_out]
1112        rebalance = self.cluster.async_rebalance(self.servers[:self.nodes_init],
1113                                                 [], [node_out])
1114        reached = RestHelper(self.rest).rebalance_reached()
1115        self.assertTrue(reached, "rebalance failed, stuck or did not complete")
1116        rebalance.result()
1117        self.sleep(30)
1118
1119        index_map_after_rebalance1 = self.get_index_map()
1120        stats_map_after_rebalance1 = self.get_index_stats(perNode=False)
1121
1122        try:
1123            self.n1ql_helper.verify_indexes_redistributed(
1124                index_map_before_rebalance,
1125                index_map_after_rebalance1,
1126                stats_map_before_rebalance,
1127                stats_map_after_rebalance1,
1128                [],
1129                [node_out])
1130        except Exception, ex:
1131            self.log.info(str(ex))
1132            if "some indexes are missing after rebalance" not in str(ex):
1133                self.fail(
1134                    "Error in index distribution post rebalance : ".format(
1135                        str(ex)))
1136            else:
1137                self.log.info(str(ex))
1138
1139        node_in = self.servers[self.node_out]
1140        rebalance = self.cluster.async_rebalance(self.servers[:self.nodes_init],
1141                                                 [node_in], [],
1142                                                 services=["index"])
1143        reached = RestHelper(self.rest).rebalance_reached()
1144        self.assertTrue(reached, "rebalance failed, stuck or did not complete")
1145        rebalance.result()
1146        self.sleep(30)
1147
1148        index_map_after_rebalance2 = self.get_index_map()
1149        stats_map_after_rebalance2 = self.get_index_stats(perNode=False)
1150
1151        try:
1152            self.n1ql_helper.verify_indexes_redistributed(
1153                index_map_before_rebalance,
1154                index_map_after_rebalance2,
1155                stats_map_before_rebalance,
1156                stats_map_after_rebalance2,
1157                [node_in],
1158                [])
1159        except Exception, ex:
1160            self.log.info(str(ex))
1161            if self.expected_err_msg not in str(ex):
1162                self.fail(
1163                    "Error in index distribution post rebalance : ".format(
1164                        str(ex)))
1165            else:
1166                self.log.info(str(ex))
1167
1168    # This test is for MB-25609 : Fix wait for index build in rebalancer for replica repair
1169    def test_rebalance_in_out_same_node_with_deferred_and_non_deferred_indexes(self):
1170        nodes = self._get_node_list()
1171        self.log.info(nodes)
1172        index_name_prefix = "random_index_" + str(
1173            random.randint(100000, 999999))
1174        create_non_deferred_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'nodes': {0}}};".format(
1175            nodes)
1176        self.log.info(create_non_deferred_index_query)
1177        try:
1178            self.n1ql_helper.run_cbq_query(query=create_non_deferred_index_query,
1179                                           server=self.n1ql_node)
1180        except Exception, ex:
1181            self.log.info(str(ex))
1182            self.fail("Index creation Failed : %s", str(ex))
1183
1184        create_deferred_index_query = "CREATE INDEX " + index_name_prefix + "_deferred ON default(age) USING GSI  WITH {{'nodes': {0},'defer_build':true}};".format(
1185            nodes)
1186        self.log.info(create_deferred_index_query)
1187        try:
1188            self.n1ql_helper.run_cbq_query(
1189                query=create_deferred_index_query,
1190                server=self.n1ql_node)
1191        except Exception, ex:
1192            self.log.info(str(ex))
1193            self.fail("Index creation Failed : %s", str(ex))
1194
1195        self.sleep(30)
1196        index_map_before_rebalance = self.get_index_map()
1197        stats_map_before_rebalance = self.get_index_stats(perNode=False)
1198
1199        self.log.info(index_map_before_rebalance)
1200        if not self.expected_err_msg:
1201            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
1202                                                    index_map_before_rebalance,
1203                                                    len(nodes) - 1, nodes)
1204
1205        node_out = self.servers[self.node_out]
1206        rebalance = self.cluster.async_rebalance(self.servers[:self.nodes_init],
1207                                                 [], [node_out])
1208        reached = RestHelper(self.rest).rebalance_reached()
1209        self.assertTrue(reached, "rebalance failed, stuck or did not complete")
1210        rebalance.result()
1211        self.sleep(30)
1212
1213        index_map_after_rebalance1 = self.get_index_map()
1214        stats_map_after_rebalance1 = self.get_index_stats(perNode=False)
1215
1216        try:
1217            self.n1ql_helper.verify_indexes_redistributed(
1218                index_map_before_rebalance,
1219                index_map_after_rebalance1,
1220                stats_map_before_rebalance,
1221                stats_map_after_rebalance1,
1222                [],
1223                [node_out])
1224        except Exception, ex:
1225            self.log.info(str(ex))
1226            if "some indexes are missing after rebalance" not in str(ex):
1227                self.fail(
1228                    "Error in index distribution post rebalance : ".format(
1229                        str(ex)))
1230            else:
1231                self.log.info(str(ex))
1232
1233        node_in = self.servers[self.node_out]
1234        rebalance = self.cluster.async_rebalance(self.servers[:self.nodes_init],
1235                                                 [node_in], [],
1236                                                 services=["index"])
1237        reached = RestHelper(self.rest).rebalance_reached()
1238        self.assertTrue(reached, "rebalance failed, stuck or did not complete")
1239        rebalance.result()
1240        self.sleep(30)
1241
1242        index_map_after_rebalance2 = self.get_index_map()
1243        stats_map_after_rebalance2 = self.get_index_stats(perNode=False)
1244
1245        try:
1246            self.n1ql_helper.verify_indexes_redistributed(
1247                index_map_before_rebalance,
1248                index_map_after_rebalance2,
1249                stats_map_before_rebalance,
1250                stats_map_after_rebalance2,
1251                [node_in],
1252                [])
1253        except Exception, ex:
1254            self.log.info(str(ex))
1255            if self.expected_err_msg not in str(ex):
1256                self.fail(
1257                    "Error in index distribution post rebalance : ".format(
1258                        str(ex)))
1259            else:
1260                self.log.info(str(ex))
1261
1262    def test_replica_movement_with_rebalance_out_and_server_groups(self):
1263        nodes = self._get_node_list()
1264        self.log.info(nodes)
1265
1266        self._create_server_groups()
1267
1268        index_name_prefix = "random_index_" + str(
1269            random.randint(100000, 999999))
1270        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'nodes': {0}}};".format(
1271            nodes)
1272        self.log.info(create_index_query)
1273        try:
1274            self.n1ql_helper.run_cbq_query(query=create_index_query,
1275                                           server=self.n1ql_node)
1276        except Exception, ex:
1277            self.log.info(str(ex))
1278            self.fail("Index creation Failed : %s", str(ex))
1279
1280        self.sleep(30)
1281        index_map_before_rebalance = self.get_index_map()
1282        stats_map_before_rebalance = self.get_index_stats(perNode=False)
1283
1284        self.log.info(index_map_before_rebalance)
1285        if not self.expected_err_msg:
1286            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
1287                                                    index_map_before_rebalance,
1288                                                    len(nodes) - 1, nodes)
1289
1290        node_out = self.servers[self.node_out]
1291        rebalance = self.cluster.async_rebalance(self.servers[:self.nodes_init],
1292                                                 [], [node_out])
1293        reached = RestHelper(self.rest).rebalance_reached()
1294        self.assertTrue(reached, "rebalance failed, stuck or did not complete")
1295        rebalance.result()
1296        self.sleep(30)
1297
1298        index_map_after_rebalance = self.get_index_map()
1299        stats_map_after_rebalance = self.get_index_stats(perNode=False)
1300
1301        try:
1302            self.n1ql_helper.verify_indexes_redistributed(
1303                index_map_before_rebalance,
1304                index_map_after_rebalance,
1305                stats_map_before_rebalance,
1306                stats_map_after_rebalance,
1307                [],
1308                [node_out])
1309        except Exception, ex:
1310            self.log.info(str(ex))
1311            if self.expected_err_msg not in str(ex):
1312                self.fail(
1313                    "Error in index distribution post rebalance : ".format(
1314                        str(ex)))
1315            else:
1316                self.log.info(str(ex))
1317
1318    def test_dropped_replica_add_new_node_with_server_group(self):
1319        # Remove the last node from the cluster
1320        node_out = self.servers[self.nodes_init - 1]
1321        rebalance = self.cluster.async_rebalance(self.servers[:self.nodes_init],
1322                                                 [], [node_out])
1323        reached = RestHelper(self.rest).rebalance_reached()
1324        self.assertTrue(reached, "rebalance failed, stuck or did not complete")
1325        rebalance.result()
1326        self.sleep(30)
1327
1328        nodes = self._get_node_list()
1329        self.log.info(nodes)
1330
1331        # Default source zone
1332        zones = self.rest.get_zone_names().keys()
1333        source_zone = zones[0]
1334        self._create_server_groups()
1335
1336        index_name_prefix = "random_index_" + str(
1337            random.randint(100000, 999999))
1338        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'nodes': {0}}};".format(
1339            nodes)
1340        self.log.info(create_index_query)
1341        try:
1342            self.n1ql_helper.run_cbq_query(query=create_index_query,
1343                                           server=self.n1ql_node)
1344        except Exception, ex:
1345            self.log.info(str(ex))
1346            self.fail("Index creation Failed : %s", str(ex))
1347
1348        self.sleep(30)
1349        index_map_before_rebalance = self.get_index_map()
1350        stats_map_before_rebalance = self.get_index_stats(perNode=False)
1351
1352        self.log.info(index_map_before_rebalance)
1353        if not self.expected_err_msg:
1354            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
1355                                                    index_map_before_rebalance,
1356                                                    len(nodes) - 1, nodes)
1357
1358        node_out = self.servers[self.node_out]
1359        rebalance = self.cluster.async_rebalance(self.servers[:self.nodes_init],
1360                                                 [], [node_out])
1361        reached = RestHelper(self.rest).rebalance_reached()
1362        self.assertTrue(reached, "rebalance failed, stuck or did not complete")
1363        rebalance.result()
1364        self.sleep(30)
1365
1366        index_map_after_rebalance1 = self.get_index_map()
1367        stats_map_after_rebalance1 = self.get_index_stats(perNode=False)
1368
1369        try:
1370            self.n1ql_helper.verify_indexes_redistributed(
1371                index_map_before_rebalance,
1372                index_map_after_rebalance1,
1373                stats_map_before_rebalance,
1374                stats_map_after_rebalance1,
1375                [],
1376                [node_out])
1377        except Exception, ex:
1378            self.log.info(str(ex))
1379            if "some indexes are missing after rebalance" not in str(ex):
1380                self.fail(
1381                    "Error in index distribution post rebalance : ".format(
1382                        str(ex)))
1383            else:
1384                self.log.info(str(ex))
1385
1386        self.rest.add_zone("server_group_3")
1387
1388        # Add back the node that was recently removed.
1389        node_in = [self.servers[self.node_out]]
1390        rebalance1 = self.cluster.async_rebalance(
1391            self.servers[:self.nodes_init],
1392            node_in, [],
1393            services=["index"])
1394
1395        # Add back the node that was previously removed.
1396        node_in = [self.servers[self.nodes_init - 1]]
1397        rebalance2 = self.cluster.async_rebalance(
1398            self.servers[:self.nodes_init],
1399            node_in, [],
1400            services=["index"])
1401
1402        # Add nodes to server groups
1403        self.rest.shuffle_nodes_in_zones([self.servers[self.node_out].ip],
1404                                         source_zone, "server_group_1")
1405        self.rest.shuffle_nodes_in_zones([self.servers[3].ip],
1406                                         source_zone, "server_group_3")
1407
1408        reached = RestHelper(self.rest).rebalance_reached()
1409        self.assertTrue(reached, "rebalance failed, stuck or did not complete")
1410        rebalance1.result()
1411        rebalance2.result()
1412        self.sleep(30)
1413
1414        index_map_after_rebalance2 = self.get_index_map()
1415        stats_map_after_rebalance2 = self.get_index_stats(perNode=False)
1416
1417        try:
1418            self.n1ql_helper.verify_indexes_redistributed(
1419                index_map_before_rebalance,
1420                index_map_after_rebalance2,
1421                stats_map_before_rebalance,
1422                stats_map_after_rebalance2,
1423                [node_in],
1424                [])
1425        except Exception, ex:
1426            self.log.info(str(ex))
1427            if self.expected_err_msg not in str(ex):
1428                self.fail(
1429                    "Error in index distribution post rebalance : ".format(
1430                        str(ex)))
1431            else:
1432                self.log.info(str(ex))
1433
1434    def test_replica_movement_with_rebalance_out_and_equivalent_index(self):
1435        nodes = self._get_node_list()
1436        self.log.info(nodes)
1437
1438        eq_index_node = self.servers[int(self.eq_index_node)].ip + ":" + \
1439                        self.servers[int(self.eq_index_node)].port
1440
1441        # Create Equivalent Index
1442        equivalent_index_query = "CREATE INDEX eq_index ON default(age) USING GSI  WITH {{'nodes': '{0}'}};".format(
1443            eq_index_node)
1444        self.log.info(equivalent_index_query)
1445        try:
1446            self.n1ql_helper.run_cbq_query(query=equivalent_index_query,
1447                                           server=self.n1ql_node)
1448        except Exception, ex:
1449            self.log.info(str(ex))
1450            self.fail("Index creation Failed : %s", str(ex))
1451
1452        index_name_prefix = "random_index_" + str(
1453            random.randint(100000, 999999))
1454        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'nodes': {0}}};".format(
1455            nodes)
1456        self.log.info(create_index_query)
1457        try:
1458            self.n1ql_helper.run_cbq_query(query=create_index_query,
1459                                           server=self.n1ql_node)
1460        except Exception, ex:
1461            self.log.info(str(ex))
1462            self.fail("Index creation Failed : %s", str(ex))
1463
1464        self.sleep(30)
1465        index_map_before_rebalance = self.get_index_map()
1466        stats_map_before_rebalance = self.get_index_stats(perNode=False)
1467
1468        self.log.info(index_map_before_rebalance)
1469        if not self.expected_err_msg:
1470            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
1471                                                    index_map_before_rebalance,
1472                                                    len(nodes) - 1, nodes)
1473
1474        node_out = self.servers[self.node_out]
1475        rebalance = self.cluster.async_rebalance(self.servers[:self.nodes_init],
1476                                                 [], [node_out])
1477        reached = RestHelper(self.rest).rebalance_reached()
1478        self.assertTrue(reached, "rebalance failed, stuck or did not complete")
1479        rebalance.result()
1480        self.sleep(30)
1481
1482        index_map_after_rebalance = self.get_index_map()
1483        stats_map_after_rebalance = self.get_index_stats(perNode=False)
1484
1485        try:
1486            self.n1ql_helper.verify_indexes_redistributed(
1487                index_map_before_rebalance,
1488                index_map_after_rebalance,
1489                stats_map_before_rebalance,
1490                stats_map_after_rebalance,
1491                [],
1492                [node_out])
1493        except Exception, ex:
1494            self.log.info(str(ex))
1495            if self.expected_err_msg not in str(ex):
1496                self.fail(
1497                    "Error in index distribution post rebalance : ".format(
1498                        str(ex)))
1499            else:
1500                self.log.info(str(ex))
1501
1502    def test_replica_movement_with_failover(self):
1503        nodes = self._get_node_list()
1504        node_out = self.servers[self.node_out]
1505        self.log.info(nodes)
1506        index_name_prefix = "random_index_" + str(
1507            random.randint(100000, 999999))
1508        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'nodes': {0}}};".format(
1509            nodes)
1510        self.log.info(create_index_query)
1511        try:
1512            self.n1ql_helper.run_cbq_query(query=create_index_query,
1513                                           server=self.n1ql_node)
1514        except Exception, ex:
1515            self.log.info(str(ex))
1516            self.fail("Index creation Failed : %s", str(ex))
1517
1518        self.sleep(30)
1519        index_map_before_failover = self.get_index_map()
1520        stats_map_before_failover = self.get_index_stats(perNode=False)
1521
1522        self.log.info(index_map_before_failover)
1523        if not self.expected_err_msg:
1524            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
1525                                                    index_map_before_failover,
1526                                                    len(nodes) - 1, nodes)
1527
1528        failover_task = self.cluster.async_failover(
1529            self.servers[:self.nodes_init],
1530            [node_out],
1531            self.graceful, wait_for_pending=180)
1532
1533        failover_task.result()
1534
1535        rebalance = self.cluster.async_rebalance(self.servers[:self.nodes_init],
1536                                                 [], [node_out])
1537        reached = RestHelper(self.rest).rebalance_reached()
1538        self.assertTrue(reached, "rebalance failed, stuck or did not complete")
1539        rebalance.result()
1540        self.sleep(30)
1541
1542        index_map_after_failover = self.get_index_map()
1543        stats_map_after_failover = self.get_index_stats(perNode=False)
1544
1545        try:
1546            self.n1ql_helper.verify_indexes_redistributed(
1547                index_map_before_failover,
1548                index_map_after_failover,
1549                stats_map_before_failover,
1550                stats_map_after_failover,
1551                [],
1552                [node_out])
1553        except Exception, ex:
1554            self.log.info(str(ex))
1555            if self.expected_err_msg not in str(ex):
1556                self.fail(
1557                    "Error in index distribution post failover : ".format(
1558                        str(ex)))
1559            else:
1560                self.log.info(str(ex))
1561
1562    def test_replica_movement_with_failover_and_addback(self):
1563        nodes = self._get_node_list()
1564        node_out = self.servers[self.node_out]
1565        self.log.info(nodes)
1566        index_name_prefix = "random_index_" + str(
1567            random.randint(100000, 999999))
1568        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'nodes': {0}}};".format(
1569            nodes)
1570        self.log.info(create_index_query)
1571        try:
1572            self.n1ql_helper.run_cbq_query(query=create_index_query,
1573                                           server=self.n1ql_node)
1574        except Exception, ex:
1575            self.log.info(str(ex))
1576            self.fail("Index creation Failed : %s", str(ex))
1577
1578        self.sleep(30)
1579        index_map_before_failover = self.get_index_map()
1580        stats_map_before_failover = self.get_index_stats(perNode=False)
1581
1582        self.log.info(index_map_before_failover)
1583        if not self.expected_err_msg:
1584            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
1585                                                    index_map_before_failover,
1586                                                    len(nodes) - 1, nodes)
1587
1588        nodes_all = self.rest.node_statuses()
1589        for node in nodes_all:
1590            if node.ip == node_out.ip:
1591                break
1592
1593        failover_task = self.cluster.async_failover(
1594            self.servers[:self.nodes_init],
1595            [node_out],
1596            self.graceful, wait_for_pending=180)
1597
1598        failover_task.result()
1599
1600        self.sleep(60)
1601
1602        self.rest.set_recovery_type(node.id, self.recovery_type)
1603        self.rest.add_back_node(node.id)
1604
1605        rebalance = self.cluster.async_rebalance(self.servers[:self.nodes_init],
1606                                                 [], [])
1607        reached = RestHelper(self.rest).rebalance_reached()
1608        self.assertTrue(reached, "rebalance failed, stuck or did not complete")
1609        rebalance.result()
1610        self.sleep(30)
1611
1612        index_map_after_failover = self.get_index_map()
1613        stats_map_after_failover = self.get_index_stats(perNode=False)
1614
1615        try:
1616            self.n1ql_helper.verify_indexes_redistributed(
1617                index_map_before_failover,
1618                index_map_after_failover,
1619                stats_map_before_failover,
1620                stats_map_after_failover,
1621                [],
1622                [])
1623        except Exception, ex:
1624            self.log.info(str(ex))
1625            if self.expected_err_msg not in str(ex):
1626                self.fail(
1627                    "Error in index distribution post failover : ".format(
1628                        str(ex)))
1629            else:
1630                self.log.info(str(ex))
1631
1632    def test_rebalance_out_with_replica_with_concurrent_querying(self):
1633        index_server = self.get_nodes_from_services_map(service_type="index",
1634                                                        get_all_nodes=False)
1635        self.run_operation(phase="before")
1636        self.sleep(30)
1637        map_before_rebalance, stats_map_before_rebalance = self._return_maps()
1638
1639        # start querying
1640        t1 = Thread(target=self.run_async_index_operations,
1641                    args=("query",))
1642        t1.start()
1643        # rebalance out a indexer node when querying is in progress
1644        rebalance = self.cluster.async_rebalance(self.servers[:self.nodes_init],
1645                                                 [], [index_server])
1646        reached = RestHelper(self.rest).rebalance_reached()
1647        self.assertTrue(reached, "rebalance failed, stuck or did not complete")
1648        rebalance.result()
1649        t1.join()
1650        map_after_rebalance, stats_map_after_rebalance = self._return_maps()
1651        # validate the results
1652        try:
1653            self.n1ql_helper.verify_indexes_redistributed(
1654                map_before_rebalance,
1655                map_after_rebalance,
1656                stats_map_before_rebalance,
1657                stats_map_after_rebalance,
1658                [],
1659                [index_server])
1660        except Exception, ex:
1661            self.log.info(str(ex))
1662            if "some indexes are missing after rebalance" not in str(ex):
1663                self.fail(
1664                    "Error in index distribution post rebalance : ".format(
1665                        str(ex)))
1666            else:
1667                self.log.info(str(ex))
1668
1669        self.run_operation(phase="after")
1670
1671    def test_failure_in_rebalance(self):
1672        nodes = self._get_node_list()
1673        self.log.info(nodes)
1674        index_name_prefix = "random_index_" + str(
1675            random.randint(100000, 999999))
1676        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'nodes': {0}}};".format(
1677            nodes)
1678        self.log.info(create_index_query)
1679        try:
1680            self.n1ql_helper.run_cbq_query(query=create_index_query,
1681                                           server=self.n1ql_node)
1682        except Exception, ex:
1683            self.log.info(str(ex))
1684            self.fail("Index creation Failed : %s", str(ex))
1685
1686        self.sleep(30)
1687        index_map_before_rebalance = self.get_index_map()
1688        stats_map_before_rebalance = self.get_index_stats(perNode=False)
1689
1690        self.log.info(index_map_before_rebalance)
1691        if not self.expected_err_msg:
1692            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
1693                                                    index_map_before_rebalance,
1694                                                    len(nodes) - 1, nodes)
1695
1696        node_out = self.servers[self.node_out]
1697        try:
1698            rebalance = self.cluster.async_rebalance(
1699                self.servers[:self.nodes_init],
1700                [], [node_out])
1701
1702            remote = RemoteMachineShellConnection(
1703                self.servers[self.failure_in_node])
1704            remote.stop_server()
1705            self.sleep(30)
1706            remote.start_server()
1707            rebalance.result()
1708
1709        except Exception, ex:
1710            if "Rebalance failed. See logs for detailed reason. You can try again" not in str(
1711                    ex):
1712                self.fail(
1713                    "rebalance failed with some unexpected error : {0}".format(
1714                        str(ex)))
1715        else:
1716            self.fail("rebalance did not fail")
1717
1718        self.sleep(60)
1719        index_servers = self.get_nodes_from_services_map(service_type="index",
1720                                                         get_all_nodes=True)
1721        # run a /cleanupRebalance after a rebalance failure
1722        for index_server in index_servers:
1723            output = self.rest.cleanup_indexer_rebalance(server=index_server)
1724            self.log.info(output)
1725        self.log.info("Retrying rebalance")
1726        rebalance1 = self.cluster.async_rebalance(self.servers[:self.nodes_init],
1727                                                 [], [node_out])
1728        self.sleep(30)
1729        reached = RestHelper(self.rest).rebalance_reached()
1730        self.assertTrue(reached, "rebalance failed, stuck or did not complete")
1731        rebalance1.result()
1732        self.sleep(30)
1733
1734        index_map_after_rebalance = self.get_index_map()
1735        stats_map_after_rebalance = self.get_index_stats(perNode=False)
1736
1737        try:
1738            self.n1ql_helper.verify_indexes_redistributed(
1739                index_map_before_rebalance,
1740                index_map_after_rebalance,
1741                stats_map_before_rebalance,
1742                stats_map_after_rebalance,
1743                [],
1744                [node_out])
1745        except Exception, ex:
1746            self.log.info(str(ex))
1747            if self.expected_err_msg not in str(ex):
1748                self.fail(
1749                    "Error in index distribution post rebalance : ".format(
1750                        str(ex)))
1751            else:
1752                self.log.info(str(ex))
1753
1754    def test_failover_with_replica_with_concurrent_querying(self):
1755        index_server = self.get_nodes_from_services_map(service_type="index",
1756                                                        get_all_nodes=False)
1757        self.run_operation(phase="before")
1758        self.sleep(30)
1759        map_before_rebalance, stats_map_before_rebalance = self._return_maps()
1760
1761        # start querying
1762        t1 = Thread(target=self.run_async_index_operations,
1763                    args=("query",))
1764        t1.start()
1765        node_out = self.servers[self.node_out]
1766        failover_task = self.cluster.async_failover(
1767            self.servers[:self.nodes_init], [node_out], self.graceful,
1768            wait_for_pending=180)
1769        failover_task.result()
1770        t1.join()
1771        map_after_rebalance, stats_map_after_rebalance = self._return_maps()
1772        # validate the results
1773        try:
1774            self.n1ql_helper.verify_indexes_redistributed(
1775                map_before_rebalance,
1776                map_after_rebalance,
1777                stats_map_before_rebalance,
1778                stats_map_after_rebalance,
1779                [],
1780                [index_server])
1781        except Exception, ex:
1782            self.log.info(str(ex))
1783            if "some indexes are missing after rebalance" not in str(ex):
1784                self.fail(
1785                    "Error in index distribution post rebalance : ".format(
1786                        str(ex)))
1787            else:
1788                self.log.info(str(ex))
1789
1790        self.run_operation(phase="after")
1791
1792    def test_load_balancing_amongst_replicas(self):
1793        index_name_prefix = "random_index_" + str(
1794            random.randint(100000, 999999))
1795        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'num_replica': {0}}};".format(
1796            self.num_index_replicas)
1797        select_query = "SELECT count(age) from default"
1798        try:
1799            self.n1ql_helper.run_cbq_query(query=create_index_query,
1800                                           server=self.n1ql_node)
1801        except Exception, ex:
1802            self.log.info(str(ex))
1803            if self.expected_err_msg not in str(ex):
1804                self.fail(
1805                    "index creation did not fail with expected error : {0}".format(
1806                        str(ex)))
1807            else:
1808                self.log.info("Index creation failed as expected")
1809        self.sleep(30)
1810        index_map = self.get_index_map()
1811        self.log.info(index_map)
1812        if not self.expected_err_msg:
1813            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
1814                                                    index_map,
1815                                                    self.num_index_replicas)
1816
1817        # Run select query 100 times
1818        for i in range(0, 100):
1819            self.n1ql_helper.run_cbq_query(query=select_query,
1820                                           server=self.n1ql_node)
1821
1822        index_stats = self.get_index_stats(perNode=True)
1823
1824        load_balanced = True
1825        for i in range(0, self.num_index_replicas + 1):
1826            if i == 0:
1827                index_name = index_name_prefix
1828            else:
1829                index_name = index_name_prefix + " (replica {0})".format(str(i))
1830
1831            hostname, _ = self.n1ql_helper.get_index_details_using_index_name(
1832                index_name, index_map)
1833            num_request_served = index_stats[hostname]['default'][index_name][
1834                "num_completed_requests"]
1835            self.log.info("# Requests served by %s = %s" % (
1836                index_name, num_request_served))
1837            if num_request_served == 0:
1838                load_balanced = False
1839
1840        if not load_balanced:
1841            self.fail("Load is not balanced amongst index replicas")
1842
1843    def test_load_balancing_with_use_index_clause(self):
1844        index_name_prefix = "random_index_" + str(
1845            random.randint(100000, 999999))
1846        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'num_replica': {0}}};".format(
1847            self.num_index_replicas)
1848        select_query = "SELECT count(age) from default USE INDEX ({0} USING GSI) where age > 10".format(
1849            index_name_prefix)
1850        try:
1851            self.n1ql_helper.run_cbq_query(query=create_index_query,
1852                                           server=self.n1ql_node)
1853        except Exception, ex:
1854            self.log.info(str(ex))
1855            if self.expected_err_msg not in str(ex):
1856                self.fail(
1857                    "index creation did not fail with expected error : {0}".format(
1858                        str(ex)))
1859            else:
1860                self.log.info("Index creation failed as expected")
1861        self.sleep(30)
1862        index_map = self.get_index_map()
1863        self.log.info(index_map)
1864        if not self.expected_err_msg:
1865            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
1866                                                    index_map,
1867                                                    self.num_index_replicas)
1868
1869        # Run select query 100 times
1870        for i in range(0, 100):
1871            self.n1ql_helper.run_cbq_query(query=select_query,
1872                                           server=self.n1ql_node)
1873
1874        index_stats = self.get_index_stats(perNode=True)
1875
1876        load_balanced = True
1877        for i in range(0, self.num_index_replicas + 1):
1878            if i == 0:
1879                index_name = index_name_prefix
1880            else:
1881                index_name = index_name_prefix + " (replica {0})".format(str(i))
1882
1883            hostname, _ = self.n1ql_helper.get_index_details_using_index_name(
1884                index_name, index_map)
1885            num_request_served = index_stats[hostname]['default'][index_name][
1886                "num_completed_requests"]
1887            self.log.info("# Requests served by %s = %s" % (
1888                index_name, num_request_served))
1889            if num_request_served == 0:
1890                load_balanced = False
1891
1892        if not load_balanced:
1893            self.fail("Load is not balanced amongst index replicas")
1894
1895    def test_load_balancing_with_prepared_statements(self):
1896        index_name_prefix = "random_index_" + str(
1897            random.randint(100000, 999999))
1898        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'num_replica': {0}}};".format(
1899            self.num_index_replicas)
1900        prepared_statement = "PREPARE prep_stmt AS SELECT count(age) from default USE INDEX ({0} USING GSI) where age > 10".format(
1901            index_name_prefix)
1902        try:
1903            self.n1ql_helper.run_cbq_query(query=create_index_query,
1904                                           server=self.n1ql_node)
1905        except Exception, ex:
1906            self.log.info(str(ex))
1907            if self.expected_err_msg not in str(ex):
1908                self.fail(
1909                    "index creation did not fail with expected error : {0}".format(
1910                        str(ex)))
1911            else:
1912                self.log.info("Index creation failed as expected")
1913        self.sleep(30)
1914        index_map = self.get_index_map()
1915        self.log.info(index_map)
1916        if not self.expected_err_msg:
1917            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
1918                                                    index_map,
1919                                                    self.num_index_replicas)
1920
1921        prepared_statement = "PREPARE prep_stmt AS SELECT count(age) from default USE INDEX ({0} USING GSI) where age > 10".format(
1922            index_name_prefix)
1923        self.n1ql_helper.run_cbq_query(query=prepared_statement,
1924                                       server=self.n1ql_node)
1925        execute_prepared_query = "EXECUTE prep_stmt"
1926
1927        # Run select query 100 times
1928        for i in range(0, 100):
1929            self.n1ql_helper.run_cbq_query(query=execute_prepared_query,
1930                                           server=self.n1ql_node)
1931
1932        index_stats = self.get_index_stats(perNode=True)
1933
1934        load_balanced = True
1935        for i in range(0, self.num_index_replicas + 1):
1936            if i == 0:
1937                index_name = index_name_prefix
1938            else:
1939                index_name = index_name_prefix + " (replica {0})".format(str(i))
1940
1941            hostname, _ = self.n1ql_helper.get_index_details_using_index_name(
1942                index_name, index_map)
1943            num_request_served = index_stats[hostname]['default'][index_name][
1944                "num_completed_requests"]
1945            self.log.info("# Requests served by %s = %s" % (
1946                index_name, num_request_served))
1947            if num_request_served == 0:
1948                load_balanced = False
1949
1950        if not load_balanced:
1951            self.fail("Load is not balanced amongst index replicas")
1952
1953    def test_move_index(self):
1954        nodes = self._get_node_list()
1955        self.log.info(nodes)
1956        index_name_prefix = "random_index_" + str(
1957            random.randint(100000, 999999))
1958        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'nodes': {0}}};".format(
1959            nodes)
1960        self.log.info(create_index_query)
1961        try:
1962            self.n1ql_helper.run_cbq_query(query=create_index_query,
1963                                           server=self.n1ql_node)
1964        except Exception, ex:
1965            self.log.info(str(ex))
1966            if self.expected_err_msg not in str(ex):
1967                self.fail(
1968                    "index creation did not fail with expected error : {0}".format(
1969                        str(ex)))
1970            else:
1971                self.log.info("Index creation failed as expected")
1972        self.sleep(30)
1973        index_map = self.get_index_map()
1974        self.log.info(index_map)
1975        if not self.expected_err_msg:
1976            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
1977                                                    index_map,
1978                                                    len(nodes) - 1, nodes)
1979        dest_nodes = self._get_node_list(self.dest_node)
1980        self.log.info(dest_nodes)
1981        expect_failure = False
1982        if self.expected_err_msg:
1983            expect_failure = True
1984
1985
1986        output, error = self._cbindex_move(src_node=self.servers[0],
1987                                       node_list=dest_nodes,
1988                                       index_list=index_name_prefix,
1989                                       expect_failure=expect_failure,
1990                                           alter_index=self.alter_index)
1991        self.sleep(30)
1992        if self.expected_err_msg:
1993          if self.expected_err_msg not in error[0]:
1994            self.fail("Move index failed with unexpected error")
1995        else:
1996          index_map = self.get_index_map()
1997          self.n1ql_helper.verify_replica_indexes([index_name_prefix],
1998                                                index_map,
1999                                                len(dest_nodes) - 1,
2000                                                dest_nodes)
2001
2002    def test_move_index_failed_node(self):
2003        node_out = self.servers[self.node_out]
2004        failover_task = self.cluster.async_failover(
2005            self.servers[:self.nodes_init],
2006            [node_out],
2007            self.graceful)
2008
2009        failover_task.result()
2010        nodes = self._get_node_list()
2011        self.log.info(nodes)
2012        index_name_prefix = "random_index_" + str(
2013            random.randint(100000, 999999))
2014        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'nodes': {0}}};".format(
2015            nodes)
2016        self.log.info(create_index_query)
2017        try:
2018            self.n1ql_helper.run_cbq_query(query=create_index_query,
2019                                           server=self.n1ql_node)
2020        except Exception, ex:
2021            self.log.info(str(ex))
2022            if self.expected_err_msg not in str(ex):
2023                self.fail(
2024                    "index creation did not fail with expected error : {0}".format(
2025                        str(ex)))
2026            else:
2027                self.log.info("Index creation failed as expected")
2028        self.sleep(30)
2029        index_map = self.get_index_map()
2030        self.log.info(index_map)
2031        if not self.expected_err_msg:
2032            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
2033                                                    index_map,
2034                                                    len(nodes) - 1, nodes)
2035
2036        dest_nodes = self._get_node_list(self.dest_node)
2037        self.log.info(dest_nodes)
2038        expect_failure = False
2039        if self.expected_err_msg:
2040            expect_failure = True
2041
2042        output, error = self._cbindex_move(src_node=self.servers[0],
2043                                           node_list=dest_nodes,
2044                                           index_list=index_name_prefix,
2045                                           expect_failure=expect_failure, alter_index=self.alter_index)
2046        self.sleep(30)
2047        if self.expected_err_msg:
2048            if self.expected_err_msg not in error[0]:
2049               self.fail("Move index failed with unexpected error")
2050            else:
2051               index_map = self.get_index_map()
2052               self.n1ql_helper.verify_replica_indexes([index_name_prefix],
2053                                                    index_map,
2054                                                    len(dest_nodes) - 1,
2055                                                    dest_nodes)
2056
2057    def test_dest_node_fails_during_move_index(self):
2058        node_out = self.servers[self.node_out]
2059
2060        nodes = self._get_node_list()
2061        self.log.info(nodes)
2062        index_name_prefix = "random_index_" + str(
2063            random.randint(100000, 999999))
2064        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'nodes': {0}}};".format(
2065            nodes)
2066        self.log.info(create_index_query)
2067        try:
2068            self.n1ql_helper.run_cbq_query(query=create_index_query,
2069                                           server=self.n1ql_node)
2070        except Exception, ex:
2071            self.log.info(str(ex))
2072            if self.expected_err_msg not in str(ex):
2073                self.fail(
2074                    "index creation did not fail with expected error : {0}".format(
2075                        str(ex)))
2076            else:
2077                self.log.info("Index creation failed as expected")
2078        self.sleep(30)
2079        index_map = self.get_index_map()
2080        self.log.info(index_map)
2081        if not self.expected_err_msg:
2082            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
2083                                                    index_map,
2084                                                    len(nodes) - 1, nodes)
2085
2086        dest_nodes = self._get_node_list(self.dest_node)
2087        self.log.info(dest_nodes)
2088        expect_failure = False
2089        if self.expected_err_msg:
2090            expect_failure = True
2091
2092        threads = []
2093        if self.alter_index:
2094            alter_index_query = 'ALTER INDEX default.' + index_name_prefix + ' WITH {{"action":"move","nodes": ["{0}","{1}"]}}'.format(
2095            dest_nodes[0],dest_nodes[1])
2096            threads.append(
2097            Thread(target=self.n1ql_helper.run_cbq_query(query=alter_index_query,server=self.n1ql_node), name="alter_index",
2098                   args=(self.servers[0], dest_nodes, index_name_prefix,
2099                         expect_failure)))
2100        else:
2101            threads.append(
2102            Thread(target=self._cbindex_move, name="move_index",
2103                   args=(self.servers[0], dest_nodes, index_name_prefix,
2104                         expect_failure)))
2105        threads.append(
2106            Thread(target=self.cluster.async_failover, name="failover", args=(
2107                self.servers[:self.nodes_init], [node_out], self.graceful)))
2108
2109        try:
2110            for thread in threads:
2111                thread.start()
2112            for thread in threads:
2113                thread.join()
2114            self.sleep(30)
2115
2116        except Exception, ex:
2117            self.log.info("***** Exception : %s", str(ex))
2118
2119    def test_index_metadata_replicated(self):
2120        index_name_prefix = "random_index_" + str(
2121            random.randint(100000, 999999))
2122        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'num_replica': {0}}};".format(
2123            self.num_index_replicas)
2124        try:
2125            self.n1ql_helper.run_cbq_query(query=create_index_query,
2126                                           server=self.n1ql_node)
2127        except Exception, ex:
2128            self.log.info(str(ex))
2129            self.fail("index creation failed")
2130
2131        self.sleep(30)
2132        index_map = self.get_index_map()
2133        self.log.info(index_map)
2134
2135        self.n1ql_helper.verify_replica_indexes([index_name_prefix], index_map,
2136                                                self.num_index_replicas)
2137
2138        index_metadata = None
2139        for i in range(0, self.nodes_init-1):
2140            node_index_metadata = RestConnection(
2141                self.servers[i]).get_indexer_metadata()
2142            self.log.info("Index metadata for %s : %s" % (
2143            self.servers[i].ip, node_index_metadata))
2144            if index_metadata:
2145                if node_index_metadata != index_metadata:
2146                    self.fail("Index metadata not replicated properly")
2147            else:
2148                index_metadata = node_index_metadata
2149
2150    def test_replica_for_different_index_types(self):
2151        self.run_operation(phase="before")
2152
2153        self.sleep(30)
2154        index_map = self.get_index_map()
2155        self.log.info(index_map)
2156
2157        index_names = self.n1ql_helper.get_index_names()
2158
2159        for index_name in index_names:
2160            self.n1ql_helper.verify_replica_indexes([index_name], index_map,
2161                                                    self.num_index_replicas)
2162        self.run_operation(phase="after")
2163
2164    def test_replica_for_primary_index(self):
2165        create_index_query = "CREATE PRIMARY INDEX primary_index on default USING GSI  WITH {{'num_replica': {0}}};".format(
2166            self.num_index_replicas)
2167        try:
2168            self.n1ql_helper.run_cbq_query(query=create_index_query,
2169                                           server=self.n1ql_node)
2170        except Exception, ex:
2171            self.log.info(str(ex))
2172            if self.expected_err_msg not in str(ex):
2173                self.fail(
2174                    "index creation did not fail with expected error : {0}".format(
2175                        str(ex)))
2176            else:
2177                self.log.info("Index creation failed as expected")
2178        self.sleep(30)
2179        index_map = self.get_index_map()
2180        self.log.info(index_map)
2181        if not self.expected_err_msg:
2182            self.n1ql_helper.verify_replica_indexes(["primary_index"],
2183                                                    index_map,
2184                                                    self.num_index_replicas)
2185
2186    def test_replica_for_dynamic_index(self):
2187        create_index_query = "CREATE INDEX dynamic ON default(DISTINCT PAIRS({{name, age}})) USING GSI  WITH {{'num_replica': {0}}};".format(
2188            self.num_index_replicas)
2189        try:
2190            self.n1ql_helper.run_cbq_query(query=create_index_query,
2191                                           server=self.n1ql_node)
2192        except Exception, ex:
2193            self.log.info(str(ex))
2194            if self.expected_err_msg not in str(ex):
2195                self.fail(
2196                    "index creation did not fail with expected error : {0}".format(
2197                        str(ex)))
2198            else:
2199                self.log.info("Index creation failed as expected")
2200        self.sleep(30)
2201        index_map = self.get_index_map()
2202        self.log.info(index_map)
2203        if not self.expected_err_msg:
2204            self.n1ql_helper.verify_replica_indexes(["dynamic"],
2205                                                    index_map,
2206                                                    self.num_index_replicas)
2207
2208    def test_rollback_to_zero_with_replicas(self):
2209        index_name_prefix = "random_index_" + str(
2210            random.randint(100000, 999999))
2211        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'num_replica': {0}}};".format(
2212            self.num_index_replicas)
2213        try:
2214            self.n1ql_helper.run_cbq_query(query=create_index_query,
2215                                           server=self.n1ql_node)
2216        except Exception, ex:
2217            self.log.info(str(ex))
2218            if self.expected_err_msg not in str(ex):
2219                self.fail(
2220                    "index creation did not fail with expected error : {0}".format(
2221                        str(ex)))
2222            else:
2223                self.log.info("Index creation failed as expected")
2224        self.sleep(30)
2225        index_map = self.get_index_map()
2226        self.log.info(index_map)
2227        if not self.expected_err_msg:
2228            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
2229                                                    index_map,
2230                                                    self.num_index_replicas)
2231
2232        self.cluster.bucket_flush(self.master)
2233        self.sleep(60)
2234
2235        index_stats = self.get_index_stats(perNode=True)
2236
2237        for i in range(0, self.num_index_replicas + 1):
2238            if i == 0:
2239                index_name = index_name_prefix
2240            else:
2241                index_name = index_name_prefix + " (replica {0})".format(str(i))
2242
2243            hostname, _ = self.n1ql_helper.get_index_details_using_index_name(
2244                index_name, index_map)
2245            num_docs_processed = index_stats[hostname]['default'][index_name][
2246                "num_docs_processed"]
2247            self.log.info("# Docs processed by %s = %s" % (
2248                index_name, num_docs_processed))
2249            if num_docs_processed != 0:
2250                self.fail("Rollback to zero fails")
2251
2252    def test_partial_rollback_with_replicas(self):
2253        index_name_prefix = "random_index_" + str(
2254            random.randint(100000, 999999))
2255        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'num_replica': {0}}};".format(
2256            self.num_index_replicas)
2257        try:
2258            self.n1ql_helper.run_cbq_query(query=create_index_query,
2259                                           server=self.n1ql_node)
2260        except Exception, ex:
2261            self.log.info(str(ex))
2262            if self.expected_err_msg not in str(ex):
2263                self.fail(
2264                    "index creation did not fail with expected error : {0}".format(
2265                        str(ex)))
2266            else:
2267                self.log.info("Index creation failed as expected")
2268        self.sleep(30)
2269        index_map = self.get_index_map()
2270        self.log.info(index_map)
2271        if not self.expected_err_msg:
2272            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
2273                                                    index_map,
2274                                                    self.num_index_replicas)
2275
2276        # Stop Persistence on Node A & Node B
2277        self.log.info("Stopping persistence on NodeA & NodeB")
2278        mem_client = MemcachedClientHelper.direct_client(self.servers[0],
2279                                                         "default")
2280        mem_client.stop_persistence()
2281        mem_client = MemcachedClientHelper.direct_client(self.servers[1],
2282                                                         "default")
2283        mem_client.stop_persistence()
2284
2285        self.run_doc_ops()
2286
2287        self.sleep(10)
2288
2289        # Get count before rollback
2290        bucket_count_before_rollback = self.get_item_count(self.servers[0],
2291                                                           "default")
2292        self.log.info("# Items in bucket before rollback = %s",
2293                      bucket_count_before_rollback)
2294        index_stats = self.get_index_stats(perNode=True)
2295        num_docs_processed_before_rollback = {}
2296
2297        for i in range(0, self.num_index_replicas + 1):
2298            if i == 0:
2299                index_name = index_name_prefix
2300            else:
2301                index_name = index_name_prefix + " (replica {0})".format(str(i))
2302
2303            hostname, _ = self.n1ql_helper.get_index_details_using_index_name(
2304                index_name, index_map)
2305            num_docs_processed_before_rollback[index_name] = \
2306            index_stats[hostname]['default'][index_name][
2307                "items_count"]
2308            self.log.info("# Before Rollback Docs processed by %s = %s" % (
2309                index_name, num_docs_processed_before_rollback[index_name]))
2310
2311        # Kill memcached on Node A so that Node B becomes master
2312        self.log.info("Kill Memcached process on NodeA")
2313        shell = RemoteMachineShellConnection(self.master)
2314        shell.kill_memcached()
2315
2316        # Start persistence on Node B
2317        self.log.info("Starting persistence on NodeB")
2318        mem_client = MemcachedClientHelper.direct_client(
2319            self.input.servers[1], "default")
2320        mem_client.start_persistence()
2321
2322        # Failover Node B
2323        self.log.info("Failing over NodeB")
2324        self.sleep(10)
2325        failover_task = self.cluster.async_failover(
2326            self.servers[:self.nodes_init], [self.servers[1]], self.graceful,
2327            wait_for_pending=120)
2328
2329        failover_task.result()
2330
2331        # Wait for a couple of mins to allow rollback to complete
2332        # self.sleep(120)
2333
2334        # Get count after rollback
2335        bucket_count_after_rollback = self.get_item_count(self.servers[0],
2336                                                          "default")
2337
2338        self.log.info("# Items in bucket after rollback = %s",
2339                      bucket_count_after_rollback)
2340        index_stats = self.get_index_stats(perNode=True)
2341        num_docs_processed_after_rollback = {}
2342
2343        if bucket_count_before_rollback == bucket_count_after_rollback:
2344            self.log.info("Looks like KV rollback did not happen at all.")
2345
2346        for i in range(0, self.num_index_replicas + 1):
2347            if i == 0:
2348                index_name = index_name_prefix
2349            else:
2350                index_name = index_name_prefix + " (replica {0})".format(str(i))
2351
2352            hostname, _ = self.n1ql_helper.get_index_details_using_index_name(
2353                index_name, index_map)
2354            num_docs_processed_after_rollback[index_name] = \
2355                index_stats[hostname]['default'][index_name][
2356                    "items_count"]
2357            self.log.info("# After rollback Docs processed by %s = %s" % (
2358                index_name, num_docs_processed_after_rollback[index_name]))
2359            if num_docs_processed_after_rollback[
2360                index_name] != bucket_count_after_rollback:
2361                self.fail("# items in index do not match # items in bucket")
2362
2363    def test_backup_restore_with_replica(self):
2364        index_name_prefix = "random_index_" + str(
2365            random.randint(100000, 999999))
2366        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'num_replica': {0}}};".format(
2367            self.num_index_replicas)
2368        try:
2369            self.n1ql_helper.run_cbq_query(query=create_index_query,
2370                                           server=self.n1ql_node)
2371        except Exception, ex:
2372            self.log.info(str(ex))
2373            if self.expected_err_msg not in str(ex):
2374                self.fail(
2375                    "index creation did not fail with expected error : {0}".format(
2376                        str(ex)))
2377            else:
2378                self.log.info("Index creation failed as expected")
2379        self.sleep(30)
2380        index_map_before_backup = self.get_index_map()
2381        self.log.info(index_map_before_backup)
2382        if not self.expected_err_msg:
2383            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
2384                                                    index_map_before_backup,
2385                                                    self.num_index_replicas)
2386
2387        kv_node = self.get_nodes_from_services_map(service_type="kv",
2388                                                   get_all_nodes=False)
2389        self._create_backup(kv_node)
2390        self._create_restore(kv_node)
2391
2392        index_map_after_restore = self.get_index_map()
2393        self.log.info(index_map_after_restore)
2394        if not self.expected_err_msg:
2395            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
2396                                                    index_map_after_restore,
2397                                                    self.num_index_replicas)
2398
2399    def test_backup_restore_with_replica_one_node_less(self):
2400        index_name_prefix = "random_index_" + str(
2401            random.randint(100000, 999999))
2402        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'num_replica': {0}}};".format(
2403            self.num_index_replicas)
2404        try:
2405            self.n1ql_helper.run_cbq_query(query=create_index_query,
2406                                           server=self.n1ql_node)
2407        except Exception, ex:
2408            self.log.info(str(ex))
2409            if self.expected_err_msg not in str(ex):
2410                self.fail(
2411                    "index creation did not fail with expected error : {0}".format(
2412                        str(ex)))
2413            else:
2414                self.log.info("Index creation failed as expected")
2415        self.sleep(30)
2416        index_map_before_backup = self.get_index_map()
2417        self.log.info(index_map_before_backup)
2418        if not self.expected_err_msg:
2419            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
2420                                                    index_map_before_backup,
2421                                                    self.num_index_replicas)
2422
2423        kv_node = self.get_nodes_from_services_map(service_type="kv",
2424                                                   get_all_nodes=False)
2425        self._create_backup(kv_node)
2426
2427        index_map_before_rebalance = self.get_index_map()
2428        stats_map_before_rebalance = self.get_index_stats(perNode=False)
2429
2430        node_out = self.servers[self.node_out]
2431        rebalance = self.cluster.async_rebalance(self.servers[:self.nodes_init],
2432                                                 [], [node_out])
2433        reached = RestHelper(self.rest).rebalance_reached()
2434        self.assertTrue(reached, "rebalance failed, stuck or did not complete")
2435        rebalance.result()
2436        self.sleep(30)
2437
2438        self._create_restore(kv_node)
2439
2440        index_map_after_rebalance = self.get_index_map()
2441        stats_map_after_rebalance = self.get_index_stats(perNode=False)
2442
2443        try:
2444            self.n1ql_helper.verify_indexes_redistributed(
2445                index_map_before_rebalance,
2446                index_map_after_rebalance,
2447                stats_map_before_rebalance,
2448                stats_map_after_rebalance,
2449                [],
2450                [node_out])
2451        except Exception, ex:
2452            self.log.info(str(ex))
2453            if self.expected_err_msg not in str(ex):
2454                self.fail(
2455                    "Error in index distribution post rebalance : ".format(
2456                        str(ex)))
2457            else:
2458                self.log.info(str(ex))
2459
2460    def test_backup_restore_add_back_dropped_replica(self):
2461        index_name_prefix = "random_index_" + str(
2462            random.randint(100000, 999999))
2463        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'num_replica': {0}}};".format(
2464            self.num_index_replicas)
2465        try:
2466            self.n1ql_helper.run_cbq_query(query=create_index_query,
2467                                           server=self.n1ql_node)
2468        except Exception, ex:
2469            self.log.info(str(ex))
2470            if self.expected_err_msg not in str(ex):
2471                self.fail(
2472                    "index creation did not fail with expected error : {0}".format(
2473                        str(ex)))
2474            else:
2475                self.log.info("Index creation failed as expected")
2476        self.sleep(30)
2477        index_map_before_backup = self.get_index_map()
2478        self.log.info(index_map_before_backup)
2479        if not self.expected_err_msg:
2480            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
2481                                                    index_map_before_backup,
2482                                                    self.num_index_replicas)
2483
2484        # Rebalance out one node so that one index replica drops
2485        node_out = self.servers[self.node_out]
2486        rebalance = self.cluster.async_rebalance(self.servers[:self.nodes_init],
2487                                                 [], [node_out])
2488        reached = RestHelper(self.rest).rebalance_reached()
2489        self.assertTrue(reached, "rebalance failed, stuck or did not complete")
2490        rebalance.result()
2491        self.sleep(30)
2492
2493        # Take a backup of the cluster
2494        kv_node = self.get_nodes_from_services_map(service_type="kv",
2495                                                   get_all_nodes=False)
2496        self._create_backup(kv_node)
2497
2498        index_map_before_rebalance = self.get_index_map()
2499        stats_map_before_rebalance = self.get_index_stats(perNode=False)
2500
2501        # Add back the node to the cluster
2502        node_out = self.servers[self.node_out]
2503        rebalance = self.cluster.async_rebalance(self.servers[:self.nodes_init],
2504                                                 [node_out], [])
2505        reached = RestHelper(self.rest).rebalance_reached()
2506        self.assertTrue(reached, "rebalance failed, stuck or did not complete")
2507        rebalance.result()
2508        self.sleep(30)
2509
2510        # Restore from the backup
2511        self._create_restore(kv_node)
2512
2513        index_map_after_rebalance = self.get_index_map()
2514        stats_map_after_rebalance = self.get_index_stats(perNode=False)
2515
2516        try:
2517            self.n1ql_helper.verify_indexes_redistributed(
2518                index_map_before_rebalance,
2519                index_map_after_rebalance,
2520                stats_map_before_rebalance,
2521                stats_map_after_rebalance,
2522                [node_out],
2523                [])
2524        except Exception, ex:
2525            self.log.info(str(ex))
2526            if self.expected_err_msg not in str(ex):
2527                self.fail(
2528                    "Error in index distribution post rebalance : ".format(
2529                        str(ex)))
2530            else:
2531                self.log.info(str(ex))
2532
2533    def test_backup_restore_with_server_groups(self):
2534        nodes = self._get_node_list()
2535        self.log.info(nodes)
2536
2537        self._create_server_groups()
2538        self.sleep(5)
2539        index_name_prefix = "random_index_" + str(
2540            random.randint(100000, 999999))
2541        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'num_replica': {0}}};".format(
2542            self.num_index_replicas)
2543        self.log.info(create_index_query)
2544        try:
2545            self.n1ql_helper.run_cbq_query(query=create_index_query,
2546                                           server=self.n1ql_node)
2547        except Exception, ex:
2548            self.log.info(str(ex))
2549            self.fail("Index creation Failed : %s", str(ex))
2550
2551        self.sleep(30)
2552        index_map = self.get_index_map()
2553
2554        self.log.info(index_map)
2555        if not self.expected_err_msg:
2556            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
2557                                                    index_map,
2558                                                    self.num_index_replicas)
2559
2560        kv_node = self.get_nodes_from_services_map(service_type="kv",
2561                                                   get_all_nodes=False)
2562
2563        self._create_backup(kv_node)
2564        self._create_restore(kv_node)
2565
2566        index_map_after_restore = self.get_index_map()
2567        self.log.info(index_map_after_restore)
2568        if not self.expected_err_msg:
2569            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
2570                                                    index_map_after_restore,
2571                                                    self.num_index_replicas)
2572
2573    def test_backup_restore_with_server_groups_one_node_less(self):
2574        nodes = self._get_node_list()
2575        self.log.info(nodes)
2576
2577        self._create_server_groups()
2578
2579        index_name_prefix = "random_index_" + str(
2580            random.randint(100000, 999999))
2581        create_index_query = "CREATE INDEX " + index_name_prefix + " ON default(age) USING GSI  WITH {{'nodes': {0}}};".format(
2582            nodes)
2583        self.log.info(create_index_query)
2584        try:
2585            self.n1ql_helper.run_cbq_query(query=create_index_query,
2586                                           server=self.n1ql_node)
2587        except Exception, ex:
2588            self.log.info(str(ex))
2589            self.fail("Index creation Failed : %s", str(ex))
2590
2591        self.sleep(30)
2592
2593        kv_node = self.get_nodes_from_services_map(service_type="kv",
2594                                                   get_all_nodes=False)
2595        self._create_backup(kv_node)
2596
2597        index_map_before_rebalance = self.get_index_map()
2598        stats_map_before_rebalance = self.get_index_stats(perNode=False)
2599
2600        self.log.info(index_map_before_rebalance)
2601        if not self.expected_err_msg:
2602            self.n1ql_helper.verify_replica_indexes([index_name_prefix],
2603                                                    index_map_before_rebalance,
2604                                                    len(nodes) - 1, nodes)
2605
2606        node_out = self.servers[self.node_out]
2607        rebalance = self.cluster.async_rebalance(self.servers[:self.nodes_init],
2608                                                 [], [node_out])
2609        reached = RestHelper(self.rest).rebalance_reached()
2610        self.assertTrue(reached, "rebalance failed, stuck or did not complete")
2611        rebalance.result()
2612        self.sleep(30)
2613
2614        self._create_restore(kv_node)
2615
2616        index_map_after_rebalance = self.get_index_map()
2617        stats_map_after_rebalance = self.get_index_stats(perNode=False)
2618
2619        try:
2620            self.n1ql_helper.verify_indexes_redistributed(
2621                index_map_before_rebalance,
2622                index_map_after_rebalance,
2623                stats_map_before_rebalance,
2624                stats_map_after_rebalance,
2625                [],
2626                [node_out])
2627        except Exception, ex:
2628            self.log.info(str(ex))
2629            if self.expected_err_msg not in str(ex):
2630                self.fail(
2631                    "Error in index distribution post rebalance : ".format(
2632                        str(ex)))
2633            else:
2634                self.log.info(str(ex))
2635
2636    def test_cbindexplan(self):
2637        if self.server_grouping:
2638            self._create_server_groups()
2639
2640        if self.eq_index_node is not None:
2641            eq_index_node = self.servers[int(self.eq_index_node)].ip + ":" + \
2642                            self.servers[int(self.eq_index_node)].port
2643
2644            # Create Equivalent Index
2645            equivalent_index_query = "CREATE INDEX eq_index ON default(age) USING GSI  WITH {{'nodes': '{0}'}};".format(
2646                eq_index_node)
2647            self.log.info(equivalent_index_query)
2648            try:
2649                self.n1ql_helper.run_cbq_query(query=equivalent_index_query,
2650                                               server=self.n1ql_node)
2651            except Exception, ex:
2652                self.log.info(str(ex))
2653                self.fail("Index creation Failed : %s", str(ex))
2654
2655        output, error, json = self._cbindexplan_plan(self.servers[0],
2656                                                     self.num_index_replicas,
2657                                                     "age")
2658
2659        if error:
2660            self.fail("cbindexplan errored out")
2661        else:
2662            expected_node_list = None
2663            if self.expected_nodes:
2664                expected_node_list = self._get_node_list(self.expected_nodes)
2665            if json:
2666                self._validate_cbindexplan_result(json, "plan",
2667                                                  expected_node_list)
2668
2669
2670    def test_failover_with_replica_with_concurrent_querying_using_use_index(self):
2671        self.run_operation(phase="before")
2672        index_server = self.get_nodes_from_services_map(service_type="index", get_all_nodes=False)
2673        index_name_prefix = "random_index_" + str(random.randint(100000, 999999))
2674        create_index_query = "CREATE INDEX " + index_name_prefix + \
2675                             " ON default(age) USING GSI  WITH {{'num_replica': {0}}};".format(self.num_index_replicas)
2676        try:
2677            self.n1ql_helper.run_cbq_query(query=create_index_query,
2678                                           server=self.n1ql_node)
2679        except Exception, ex:
2680            self.log.info(str(ex))
2681            if self.expected_err_msg not in str(ex):
2682                self.fail(
2683                    "index creation did not fail with expected error : {0}".format(
2684                        str(ex)))
2685            else:
2686                self.log.info("Index creation failed as expected")
2687        self.sleep(30)
2688        index_map = self.get_index