1import time
2import unittest
3import urllib
4import random
5import testconstants
6from TestInput import TestInputSingleton
7
8from cwc.cwc_base import CWCBaseTest
9from membase.api.rest_client import RestConnection
10from remote.remote_util import RemoteMachineShellConnection
11
12
13
14class CWCTests(CWCBaseTest):
15    def setUp(self):
16        super(CWCTests, self).setUp()
17        self.command = self.input.param("command", "")
18        self.command_options = self.input.param("command_options", '')
19        self.item_size = self.input.param("item_size", 128)
20        self.shutdown_node = self.input.param("shutdown_node", 1)
21        self.do_verify = self.input.param("do-verify", True)
22        self.timeout = 6000
23
24
25    def tearDown(self):
26        super(CWCTests, self).tearDown()
27
28    def test_start_collect_log(self):
29        rest = RestConnection(self.master)
30        shell = RemoteMachineShellConnection(self.master)
31        if "*" not in str(self.collect_nodes) and self.nodes_init > 1:
32            self.collect_nodes = self._generate_random_collecting_node(rest)
33        status, content = rest.start_cluster_logs_collection(nodes=self.collect_nodes, \
34                                upload=self.upload, uploadHost=self.uploadHost, \
35                                customer=self.customer, ticket=self.ticket)
36        if status:
37            collected, uploaded, cancel_collect  = \
38                       self._monitor_collecting_log(rest, timeout=1200)
39            if collected:
40                self._verify_log_file(rest)
41            if self.upload and uploaded:
42                self._verify_log_uploaded(rest)
43            if self.cancel_collect:
44                if cancel_collect:
45                    self.log.info("Logs collection were cancelled")
46                else:
47                    self.fail("Failed to cancel log collection")
48            shell.disconnect()
49        else:
50            self.fail("ERROR:  {0}".format(content))
51
52    def _monitor_collecting_log(self, rest, timeout):
53        start_time = time.time()
54        end_time = start_time + timeout
55        collected = False
56        uploaded = False
57        cancel_collect = False
58        progress = 0
59        progress, stt, perNode = rest.get_cluster_logs_collection_status()
60        while (progress != 100 or stt == "running") and time.time() <= end_time :
61            progress, stt, perNode = rest.get_cluster_logs_collection_status()
62            if stt is not None and self.cancel_collect:
63                count = 0
64                if "running" in stt and count == 0:
65                    self.log.info("Start to cancel collect logs ")
66                    status, content = rest.cancel_cluster_logs_collection()
67                    count += 1
68                if "cancelled" in stt:
69                    cancel_collect = True
70                    break
71                elif count == 2:
72                    self.fail("Failed to cancel log collection")
73            self.log.info("Cluster-wide collectinfo progress: {0}".format(progress))
74            if perNode is not None:
75                for node in perNode:
76                    self.log.info("Node: {0} **** Collect status: {1}" \
77                                  .format(node, perNode[node]["status"]))
78                    if "collected" in perNode[node]["status"]:
79                        collected = True
80                    elif "uploaded" in perNode[node]["status"]:
81                        uploaded = True
82            self.sleep(10)
83        if time.time() > end_time:
84            if self.cancel_collect:
85                self.log.error("Could not cancel log collection after {0} seconds ".format(timeout))
86            elif self.upload:
87                self.log.error("Log could not upload after {0} seconds ".format(timeout))
88            else:
89                self.log.error("Log could not collect after {0} seconds ".format(timeout))
90            return collected, uploaded, cancel_collect
91        else:
92            duration = time.time() - start_time
93            self.log.info("log collection took {0} seconds ".format(duration))
94            return collected, uploaded, cancel_collect
95
96
97    def _verify_log_file(self, rest):
98        progress, status, perNode = rest.get_cluster_logs_collection_status()
99        node_failed_to_collect = []
100        for node in perNode:
101            for server in self.servers[:self.nodes_init]:
102                if server.ip in node or (self.nodes_init == 1 \
103                                         and "127.0.0.1" in node):
104                    shell = RemoteMachineShellConnection(server)
105            file_name = perNode[node]["path"].replace(self.log_path, "")
106            collected = False
107            retry = 0
108            while not collected and retry < 5:
109                existed = shell.file_exists(self.log_path, file_name)
110                if existed:
111                    self.log.info("file {0} exists on node {1}"
112                                  .format(perNode[node]["path"], node.replace("ns_1@", "")))
113                    collected = True
114                else:
115                    self.log.info("retry {0} ".format(retry))
116                    retry += 1
117                    self.sleep(5)
118                if retry == 5:
119                    self.log.error("failed to collect log after {0} try at node {1}"
120                                   .format(retry, node.replace("ns_1@", "")))
121                    node_failed_to_collect.append(node)
122        if not node_failed_to_collect:
123            return True
124        else:
125            self.fail("Cluster-wide collectinfo failed to collect log at {0}" \
126                           .format(node_failed_to_collect))
127
128    def _verify_log_uploaded(self, rest):
129        node_failed_to_uploaded = []
130        progress, status, perNode = rest.get_cluster_logs_collection_status()
131        for node in perNode:
132            self.log.info("Verify log of node {0} uploaded to host: {1}" \
133                          .format(node, self.uploadHost))
134            uploaded = urllib.urlopen(perNode[node]["url"]).getcode()
135            if uploaded == 200 and self.uploadHost in perNode[node]["url"]:
136                self.log.info("Log of node {0} was uploaded to {1}" \
137                              .format(node, perNode[node]["url"]))
138            else:
139                node_failed_to_uploaded.append(node)
140        if not node_failed_to_uploaded:
141            return True
142        else:
143            self.fail("Cluster-wide collectinfo failed to upload log at node(s) {0}" \
144                           .format(node_failed_to_uploaded))
145
146    def test_cli_start_collect_log(self):
147        command = "couchbase-cli collect-logs-start"
148        rest = RestConnection(self.master)
149        shell = RemoteMachineShellConnection(self.master)
150        num_node_collect = self.cli_collect_nodes
151        if "--all-nodes" not in str(self.cli_collect_nodes) and self.nodes_init > 1:
152            self.cli_collect_nodes = self._cli_generate_random_collecting_node(rest)
153            num_node_collect = '--nodes="{0}"'.format(self.cli_collect_nodes)
154        if not self.cli_upload:
155            o, e = shell.execute_command("{0}{1} -c {2}:8091 -u Administrator -p password {3} " \
156                             .format(self.bin_path, command, self.master.ip, num_node_collect))
157        else:
158            o, e = shell.execute_command("{0}{1} -c {2}:8091 -u Administrator -p password {3} --upload \
159                           --upload-host='{4}' --customer='{5}' --ticket='{6}' " .format(self.bin_path, \
160                           command, self.master.ip, num_node_collect, self.uploadHost, self.customer, \
161                           self.ticket))
162        shell.log_command_output(o, e)
163        """ output when --nodes is used
164                ['NODES: ns_1@12,ns_1@11,ns_1@10', 'SUCCESS: Log collection started']
165            output when --all-nodes is used
166                'SUCCESS: Log collection started' """
167        status_check = o[0]
168        if "--all-nodes" not in str(self.cli_collect_nodes):
169            status_check = o[1]
170        if "SUCCESS" in status_check:
171            self.log.info("start monitoring cluster-wide collectinfo using CLI ...")
172            collected, uploaded, cancel_collect = \
173                   self._cli_monitor_collecting_log(shell, timeout=1200)
174            if collected:
175                self._cli_verify_log_file(shell)
176            if self.cli_upload and uploaded:
177                self._cli_verify_log_uploaded(shell)
178            if self.cli_cancel_collect:
179                if cancel_collect:
180                    self.log.info("Logs collection were cancelled by CLI")
181                else:
182                    self.fail("Failed to cancel log collection by CLI")
183            shell.disconnect()
184        elif o and "ERROR" in o[0]:
185            self.fail("ERROR:  {0}".format(o[0]))
186
187    def _generate_random_collecting_node(self, rest):
188        random_nodes = []
189        nodes = rest.get_nodes()
190        for k in random.sample(range(int(self.nodes_init)), int(self.collect_nodes)):
191            random_nodes.append(nodes[k].id)
192        random_nodes =",".join(random_nodes)
193        self.log.info("nodes randomly selected to do CWC {0}".format(random_nodes))
194        return random_nodes
195