1import copy
2import json
3from membase.api.rest_client import RestConnection, RestHelper
4from membase.helper.rebalance_helper import RebalanceHelper
5from remote.remote_util import RemoteMachineShellConnection
6from remote.remote_util import RemoteUtilHelper
7from failoverbasetests import FailoverBaseTest
8
9class NegativeFailoverTests(FailoverBaseTest):
10
11    def setUp(self):
12        super(NegativeFailoverTests, self).setUp()
13
14    def tearDown(self):
15        super(NegativeFailoverTests, self).tearDown()
16
17    def graceful_failover_when_rebalance_running(self):
18        try:
19            self.rest = RestConnection(self.master)
20            nodes = self.rest.node_statuses()
21            chosen = RebalanceHelper.pick_nodes(self.master, howmany=1)
22            status = self.rest.rebalance(otpNodes=[node.id for node in nodes],
23                ejectedNodes=[node.id for node in nodes[1:]])
24            self.assertTrue(status," Rebalance did not run ")
25            success_failed_over = self.rest.fail_over(chosen[0].id, graceful=True)
26            self.assertFalse(success_failed_over," Failover did not fail as expected ")
27        except Exception,ex:
28            self.assertTrue(("Rebalance running" in str(ex)),"unexpected exception {0}".format(ex))
29
30    def graceful_failover_when_graceful_failover_running(self):
31        try:
32            self.rest = RestConnection(self.master)
33            nodes = self.get_nodes(self.master)
34            chosen = RebalanceHelper.pick_nodes(self.master, howmany=1)
35            success_failed_over = self.rest.fail_over(chosen[0].id, graceful=True)
36            self.assertTrue(success_failed_over," Failover failed ")
37            success_failed_over = self.rest.fail_over(chosen[0].id, graceful=True)
38            self.assertFalse(success_failed_over," Failover did not fail as expected ")
39        except Exception,ex:
40            self.assertTrue(("Rebalance running" in str(ex)),"unexpected exception {0}".format(ex))
41
42    def hard_failover_when_graceful_failover_running(self):
43        try:
44            self.rest = RestConnection(self.master)
45            nodes = self.get_nodes(self.master)
46            chosen = RebalanceHelper.pick_nodes(self.master, howmany=1)
47            success_failed_over = self.rest.fail_over(chosen[0].id, graceful=True)
48            self.assertTrue(success_failed_over," Failover failed ")
49            success_failed_over = self.rest.fail_over(chosen[0].id, graceful=False)
50            self.assertFalse(success_failed_over," Failover did not fail as expected ")
51        except Exception,ex:
52            self.assertTrue(("Rebalance running" in str(ex)),"unexpected exception {0}".format(ex))
53
54    def hard_failover_nonexistant_node(self):
55        try:
56            self.rest = RestConnection(self.master)
57            nodes = self.get_nodes(self.master)
58            success_failed_over = self.rest.fail_over("non-existant", graceful=False)
59            self.assertFalse(success_failed_over," Failover did not fail as expected ")
60        except Exception,ex:
61            self.assertTrue(("Unknown server given" in str(ex)),"unexpected exception {0}".format(ex))
62
63    def graceful_failover_nonexistant_node(self):
64        try:
65            self.rest = RestConnection(self.master)
66            nodes = self.get_nodes(self.master)
67            success_failed_over = self.rest.fail_over("non-existant", graceful=True)
68            self.assertFalse(success_failed_over," Failover did not fail as expected ")
69        except Exception,ex:
70            self.assertTrue(("Unknown server given" in str(ex)),"unexpected exception {0}".format(ex))
71
72    def failover_failed_node(self):
73        try:
74            self.rest = RestConnection(self.master)
75            nodes = self.get_nodes(self.master)
76            chosen = RebalanceHelper.pick_nodes(self.master, howmany=1)
77            success_failed_over = self.rest.fail_over(chosen[0].id, graceful=False)
78            self.assertTrue(success_failed_over," Failover did not happen as expected ")
79            fail_failed_over = self.rest.fail_over(chosen[0].id, graceful=False)
80            self.assertFalse(fail_failed_over," Failover did not fail as expected ")
81        except Exception,ex:
82            self.assertTrue(("Unknown server given" in str(ex)),"unexpected exception {0}".format(ex))
83
84    def addback_non_existant_node(self):
85        try:
86            self.rest = RestConnection(self.master)
87            nodes = self.get_nodes(self.master)
88            chosen = RebalanceHelper.pick_nodes(self.master, howmany=1)
89            # Mark Node for failover
90            success_failed_over = self.rest.fail_over(chosen[0].id, graceful=False)
91            # Mark Node for full recovery
92            if success_failed_over:
93                self.rest.set_recovery_type(otpNode="non-existant", recoveryType="delta")
94        except Exception, ex:
95            self.assertTrue(("invalid node name or node" in str(ex)),"unexpected exception {0}".format(ex))
96
97    def addback_an_unfailed_node(self):
98        try:
99            self.rest = RestConnection(self.master)
100            nodes = self.get_nodes(self.master)
101            chosen = RebalanceHelper.pick_nodes(self.master, howmany=1)
102            self.rest.set_recovery_type(otpNode=chosen[0].id, recoveryType="delta")
103        except Exception, ex:
104            self.assertTrue(("invalid node name or node" in str(ex)),"unexpected exception {0}".format(ex))
105
106    def addback_with_incorrect_recovery_type(self):
107        try:
108            self.rest = RestConnection(self.master)
109            nodes = self.get_nodes(self.master)
110            chosen = RebalanceHelper.pick_nodes(self.master, howmany=1)
111            # Mark Node for failover
112            success_failed_over = self.rest.fail_over(chosen[0].id, graceful=False)
113            # Mark Node for full recovery
114            if success_failed_over:
115                self.rest.set_recovery_type(otpNode=chosen[0].id, recoveryType="xxx")
116        except Exception, ex:
117            self.assertTrue(("recoveryType" in str(ex)),"unexpected exception {0}".format(ex))
118
119    def failure_recovery_delta_node_with_failover_node(self):
120        try:
121            self.rest = RestConnection(self.master)
122            chosen = RebalanceHelper.pick_nodes(self.master, howmany=2)
123            # Mark Node(s) for failover
124            success_failed_over = self.rest.fail_over(chosen[0].id, graceful=False)
125            success_failed_over = self.rest.fail_over(chosen[1].id, graceful=False)
126            # Mark Node for full recovery
127            if success_failed_over:
128                self.rest.set_recovery_type(otpNode=chosen[0].id, recoveryType="delta")
129            servers_out = self.add_remove_servers(self.servers,[],[],[chosen[1]])
130            rebalance = self.cluster.async_rebalance(self.servers[:self.nodes_init], [],servers_out)
131            rebalance.result()
132        except Exception, ex:
133            self.assertTrue(("deltaRecoveryNotPossible" in str(ex)),"unexpected exception {0}".format(ex))
134
135    def failure_recovery_delta_node_before_rebalance_in(self):
136        try:
137            self.rest = RestConnection(self.master)
138            chosen = RebalanceHelper.pick_nodes(self.master, howmany=1)
139            # Mark Node for failover
140            success_failed_over = self.rest.fail_over(chosen[0].id, graceful=False)
141            # Mark Node for full recovery
142            if success_failed_over:
143                self.rest.set_recovery_type(otpNode=chosen[0].id, recoveryType="delta")
144            rebalance = self.cluster.async_rebalance(self.servers[:self.nodes_init], [self.servers[self.nodes_init]], [])
145            rebalance.result()
146        except Exception, ex:
147            self.assertTrue(("deltaRecoveryNotPossible" in str(ex)),"unexpected exception {0}".format(ex))
148
149    def failure_recovery_delta_node_after_add_node(self):
150        try:
151            self.rest = RestConnection(self.master)
152            chosen = RebalanceHelper.pick_nodes(self.master, howmany=1)
153            self.rest.add_node(self.master.rest_username, self.master.rest_password,self.servers[self.nodes_init].ip,self.servers[self.nodes_init].port)
154            # Mark Node for failover
155            success_failed_over = self.rest.fail_over(chosen[0].id, graceful=False)
156            # Mark Node for full recovery
157            if success_failed_over:
158                self.rest.set_recovery_type(otpNode=chosen[0].id, recoveryType="delta")
159            self.nodes = self.rest.node_statuses()
160            self.rest.rebalance(otpNodes=[node.id for node in self.nodes],ejectedNodes=[])
161            self.assertFalse(self.rest.monitorRebalance(stop_if_loop=True), msg="Rebalance did not fail as expected")
162        except Exception, ex:
163            self.assertTrue(("deltaRecoveryNotPossible" in str(ex)),"unexpected exception {0}".format(ex))
164
165    def failure_recovery_delta_node_after_eject_node(self):
166        try:
167            self.rest = RestConnection(self.master)
168            eject_out_node = self.find_node_info(self.master,self.servers[self.nodes_init-1])
169            chosen = RebalanceHelper.pick_nodes(self.master, howmany=1)
170            self.rest.eject_node(self.master.rest_username, self.master.rest_password,self.servers[self.nodes_init-1])
171            # Mark Node for failover
172            success_failed_over = self.rest.fail_over(chosen[0].id, graceful=False)
173            # Mark Node for full recovery
174            if success_failed_over:
175                self.rest.set_recovery_type(otpNode=chosen[0].id, recoveryType="delta")
176            self.nodes = self.rest.node_statuses()
177            self.rest.rebalance(otpNodes=[node.id for node in self.nodes],ejectedNodes=[chosen[0].id,eject_out_node.id])
178            self.assertFalse(self.rest.monitorRebalance(stop_if_loop=True), msg="Rebalance did not fail as expected")
179        except Exception, ex:
180            self.assertTrue(("deltaRecoveryNotPossible" in str(ex)),"unexpected exception {0}".format(ex))
181
182    def failure_recovery_delta_node_before_rebalance_in_out(self):
183        try:
184            self.rest = RestConnection(self.master)
185            chosen = RebalanceHelper.pick_nodes(self.master, howmany=1)
186            # Mark Node for failover
187            success_failed_over = self.rest.fail_over(chosen[0].id, graceful=False)
188            # Mark Node for full recovery
189            if success_failed_over:
190                self.rest.set_recovery_type(otpNode=chosen[0].id, recoveryType="delta")
191            ejectedNodes  = []
192            self.rest.add_node(self.master.rest_username, self.master.rest_password,
193                self.servers[self.nodes_init].ip,self.servers[self.nodes_init].port)
194            self.rest.add_node(self.master.rest_username, self.master.rest_password,
195                self.servers[self.nodes_init+1].ip,self.servers[self.nodes_init+1].port)
196            self.nodes = self.rest.node_statuses()
197            for server in self.nodes:
198                if server.ip == self.servers[self.nodes_init].ip:
199                   ejectedNodes.append(server.id)
200            self.rest.rebalance(otpNodes=[node.id for node in self.nodes],ejectedNodes=ejectedNodes)
201            self.assertFalse(self.rest.monitorRebalance(stop_if_loop=True), msg="Rebalance did not fail as expected")
202        except Exception, ex:
203            self.assertTrue(("deltaRecoveryNotPossible" in str(ex)),"unexpected exception {0}".format(ex))
204
205    def graceful_failover_unhealthy_node_not_allowed(self):
206        try:
207            self.rest = RestConnection(self.master)
208            nodes = self.get_nodes(self.master)
209            self.stop_server(self.servers[1])
210            # Mark Node for failover
211            chosen = RebalanceHelper.pick_nodes(self.master, howmany=1)
212            success_failed_over = self.rest.fail_over(chosen[0].id, graceful=False)
213            self.assertFalse(success_failed_over," Graceful failover allowed for unhealthy node")
214        finally:
215            self.start_server(self.servers[1])
216
217    def stop_server(self, node):
218        """ Method to stop a server which is subject to failover """
219        for server in self.servers:
220            if server.ip == node.ip:
221                shell = RemoteMachineShellConnection(server)
222                if shell.is_couchbase_installed():
223                    shell.stop_couchbase()
224                    self.log.info("Couchbase stopped")
225                else:
226                    shell.stop_membase()
227                    self.log.info("Membase stopped")
228                shell.disconnect()
229
230    def start_server(self, node):
231        """ Method to stop a server which is subject to failover """
232        for server in self.servers:
233            if server.ip == node.ip:
234                shell = RemoteMachineShellConnection(server)
235                if shell.is_couchbase_installed():
236                    shell.start_couchbase()
237                    self.log.info("Couchbase started")
238                else:
239                    shell.start_membase()
240                    self.log.info("Membase started")
241                shell.disconnect()
242                break
243
244