1import os
2import sys
3import urllib
4import uuid
5import time
6import logging
7from datetime import datetime
8import logger
9from builds.build_query import BuildQuery
10import testconstants
11from testconstants import WIN_REGISTER_ID
12from testconstants import MEMBASE_VERSIONS
13from testconstants import COUCHBASE_VERSIONS
14from testconstants import MISSING_UBUNTU_LIB
15from testconstants import MV_LATESTBUILD_REPO
16from testconstants import WIN_CB_VERSION_3
17from testconstants import COUCHBASE_VERSION_2
18from testconstants import COUCHBASE_VERSION_3
19from testconstants import WIN_COUCHBASE_BIN_PATH
20
21
22from membase.api.rest_client import RestConnection, RestHelper
23
24
25log = logger.Logger.get_logger()
26logging.getLogger("paramiko").setLevel(logging.WARNING)
27
28try:
29    import paramiko
30except ImportError:
31    log.warn("{0} {1} {2}".format("Warning: proceeding without importing",
32                                  "paramiko due to import error.",
33                                  "ssh connections to remote machines will fail!\n"))
34
35class RemoteMachineInfo(object):
36    def __init__(self):
37        self.type = ''
38        self.ip = ''
39        self.distribution_type = ''
40        self.architecture_type = ''
41        self.distribution_version = ''
42        self.deliverable_type = ''
43        self.ram = ''
44        self.cpu = ''
45        self.disk = ''
46        self.hostname = ''
47
48
49class RemoteMachineProcess(object):
50    def __init__(self):
51        self.pid = ''
52        self.name = ''
53
54
55class RemoteMachineHelper(object):
56    remote_shell = None
57
58    def __init__(self, remote_shell):
59        self.remote_shell = remote_shell
60
61    def monitor_process(self, process_name,
62                        duration_in_seconds=120):
63        # monitor this process and return if it crashes
64        end_time = time.time() + float(duration_in_seconds)
65        last_reported_pid = None
66        while time.time() < end_time:
67            # get the process list
68            process = self.is_process_running(process_name)
69            if process:
70                if not last_reported_pid:
71                    last_reported_pid = process.pid
72                elif not last_reported_pid == process.pid:
73                    message = 'process {0} was restarted. old pid : {1} new pid : {2}'
74                    log.info(message.format(process_name, last_reported_pid, process.pid))
75                    return False
76                    # check if its equal
77            else:
78                # we should have an option to wait for the process
79                # to start during the timeout
80                # process might have crashed
81                log.info("{0}:process {1} is not running or it might have crashed!".format(self.remote_shell.ip, process_name))
82                return False
83            time.sleep(1)
84        #            log.info('process {0} is running'.format(process_name))
85        return True
86
87    def is_process_running(self, process_name):
88        if getattr(self.remote_shell, "info", None) is None:
89            self.remote_shell.info = self.remote_shell.extract_remote_info()
90
91        if self.remote_shell.info.type.lower() == 'windows':
92             print "process name in is process running ", process_name
93             output, error = self.remote_shell.execute_command('tasklist| grep {0}'.format(process_name), debug=False)
94             if error or output == [""] or output == []:
95                 return None
96             words = output[0].split(" ")
97             words = filter(lambda x: x != "", words)
98             process = RemoteMachineProcess()
99             process.pid = words[1]
100             process.name = words[0]
101             log.info("process is running on {0}: {1}".format(self.remote_shell.ip, words))
102             return process
103        else:
104            processes = self.remote_shell.get_running_processes()
105            for process in processes:
106                if process.name == process_name:
107                    return process
108            return None
109
110
111class RemoteMachineShellConnection:
112    _ssh_client = None
113
114    def __init__(self, username='root',
115                 pkey_location='',
116                 ip=''):
117        self.username = username
118        self.use_sudo = True
119        if self.username == 'root':
120           self.use_sudo = False
121        # let's create a connection
122        self._ssh_client = paramiko.SSHClient()
123        self.ip = ip
124        self._ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
125        log.info('connecting to {0} with username : {1} pem key : {2}'.format(ip, username, pkey_location))
126        try:
127            self._ssh_client.connect(hostname=ip, username=username, key_filename=pkey_location)
128        except paramiko.AuthenticationException:
129            log.info("Authentication failed for {0}".format(self.ip))
130            exit(1)
131        except paramiko.BadHostKeyException:
132            log.info("Invalid Host key for {0}".format(self.ip))
133            exit(1)
134        except Exception:
135            log.info("Can't establish SSH session with {0}".format(self.ip))
136            exit(1)
137
138    def __init__(self, serverInfo):
139        # let's create a connection
140        self.username = serverInfo.ssh_username
141        self.use_sudo = True
142        if self.username == 'root':
143           self.use_sudo = False
144        self._ssh_client = paramiko.SSHClient()
145        self.ip = serverInfo.ip
146        self.port = serverInfo.port
147        self._ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
148        msg = 'connecting to {0} with username : {1} password : {2} ssh_key: {3}'
149        log.info(msg.format(serverInfo.ip, serverInfo.ssh_username, serverInfo.ssh_password, serverInfo.ssh_key))
150        # added attempts for connection because of PID check failed. RNG must be re-initialized after fork() error
151        # That's a paramiko bug
152        max_attempts_connect = 2
153        attempt = 0
154        while True:
155            try:
156                if serverInfo.ssh_key == '':
157                    self._ssh_client.connect(hostname=serverInfo.ip,
158                                             username=serverInfo.ssh_username,
159                                             password=serverInfo.ssh_password)
160                else:
161                    self._ssh_client.connect(hostname=serverInfo.ip,
162                                             username=serverInfo.ssh_username,
163                                             key_filename=serverInfo.ssh_key)
164                break
165            except paramiko.AuthenticationException:
166                log.error("Authentication failed")
167                exit(1)
168            except paramiko.BadHostKeyException:
169                log.error("Invalid Host key")
170                exit(1)
171            except Exception as e:
172                if str(e).find('PID check failed. RNG must be re-initialized') != -1 and\
173                        attempt != max_attempts_connect:
174                    log.error("Can't establish SSH session to node {1} :\
175                              {0}. Will try again in 1 sec".format(e, self.ip))
176                    attempt += 1
177                    time.sleep(1)
178                else:
179                    log.error("Can't establish SSH session to node {1} :\
180                                                   {0}".format(e, self.ip))
181                    exit(1)
182        log.info("Connected to {0}".format(serverInfo.ip))
183
184    def sleep(self, timeout=1, message=""):
185        log.info("{0}:sleep for {1} secs. {2} ...".format(self.ip, timeout, message))
186        time.sleep(timeout)
187
188    def get_running_processes(self):
189        # if its linux ,then parse each line
190        # 26989 ?        00:00:51 pdflush
191        # ps -Ao pid,comm
192        processes = []
193        output, error = self.execute_command('ps -Ao pid,comm', debug=False)
194        if output:
195            for line in output:
196                # split to words
197                words = line.strip().split(' ')
198                if len(words) >= 2:
199                    process = RemoteMachineProcess()
200                    process.pid = words[0]
201                    process.name = words[1]
202                    processes.append(process)
203        return processes
204
205    def get_mem_usage_by_process(self, process_name):
206        """Now only linux"""
207        output, error = self.execute_command('ps -e -o %mem,cmd|grep {0}'.format(process_name),
208                                             debug=False)
209        if output:
210            for line in output:
211                if not 'grep' in line.strip().split(' '):
212                    return float(line.strip().split(' ')[0])
213
214    def stop_membase(self):
215        self.extract_remote_info()
216        if self.info.type.lower() == 'windows':
217            log.info("STOP SERVER")
218            o, r = self.execute_command("net stop membaseserver")
219            self.log_command_output(o, r)
220            o, r = self.execute_command("net stop couchbaseserver")
221            self.log_command_output(o, r)
222            self.sleep(10, "Wait to stop service completely")
223        if self.info.type.lower() == "linux":
224            o, r = self.execute_command("/etc/init.d/membase-server stop")
225            self.log_command_output(o, r)
226
227    def start_membase(self):
228        self.extract_remote_info()
229        if self.info.type.lower() == 'windows':
230            o, r = self.execute_command("net start membaseserver")
231            self.log_command_output(o, r)
232        if self.info.type.lower() == "linux":
233            o, r = self.execute_command("/etc/init.d/membase-server start")
234            self.log_command_output(o, r)
235
236    def start_server(self, os="unix"):
237        if os == "windows":
238            o, r = self.execute_command("net start couchbaseserver")
239            self.log_command_output(o, r)
240        elif os == "unix" or os == "linux":
241            if self.is_couchbase_installed():
242                o, r = self.execute_command("/etc/init.d/couchbase-server start")
243            else:
244                o, r = self.execute_command("/etc/init.d/membase-server start")
245            self.log_command_output(o, r)
246        elif os == "mac":
247            o, r = self.execute_command("open /Applications/Couchbase\ Server.app")
248            self.log_command_output(o, r)
249        else:
250            self.log.error("don't know operating system or product version")
251
252    def stop_server(self, os="unix"):
253        if os == "windows":
254            o, r = self.execute_command("net stop couchbaseserver")
255            self.log_command_output(o, r)
256        elif os == "unix" or os == "linux":
257            if self.is_couchbase_installed():
258                o, r = self.execute_command("/etc/init.d/couchbase-server stop", use_channel=True)
259            else:
260                o, r = self.execute_command("/etc/init.d/membase-server stop")
261            self.log_command_output(o, r)
262        elif os == "mac":
263            o, r = self.execute_command("ps aux | grep Couchbase | awk '{print $2}' | xargs kill -9")
264            self.log_command_output(o, r)
265            o, r = self.execute_command("killall -9 epmd")
266            self.log_command_output(o, r)
267        else:
268            self.log.error("don't know operating system or product version")
269
270    def stop_schedule_tasks(self):
271        log.info("STOP ALL SCHEDULE TASKS: installme, removeme and upgrademe")
272        output, error = self.execute_command("cmd /c schtasks /end /tn installme")
273        self.log_command_output(output, error)
274        output, error = self.execute_command("cmd /c schtasks /end /tn removeme")
275        self.log_command_output(output, error)
276        output, error = self.execute_command("cmd /c schtasks /end /tn upgrademe")
277        self.log_command_output(output, error)
278
279    def kill_erlang(self, os="unix"):
280        if os == "windows":
281            o, r = self.execute_command("taskkill /F /T /IM epmd.exe*")
282            self.log_command_output(o, r)
283            o, r = self.execute_command("taskkill /F /T /IM erl.exe*")
284            self.log_command_output(o, r)
285        else:
286            o, r = self.execute_command("killall -9 beam.smp")
287            self.log_command_output(o, r)
288        return o, r
289
290    def kill_memcached(self):
291        self.extract_remote_info()
292        if self.info.type.lower() == 'windows':
293            o, r = self.execute_command("taskkill /F /T /IM memcached*")
294            self.log_command_output(o, r)
295        else:
296            o, r = self.execute_command("killall -9 memcached")
297            self.log_command_output(o, r)
298        return o, r
299
300    def change_log_level(self, new_log_level):
301        log.info("CHANGE LOG LEVEL TO %s".format(new_log_level))
302        # ADD NON_ROOT user config_details
303        output, error = self.execute_command("sed -i '/loglevel_default, /c \\{loglevel_default, %s\}'. %s"
304                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
305        self.log_command_output(output, error)
306        output, error = self.execute_command("sed -i '/loglevel_ns_server, /c \\{loglevel_ns_server, %s\}'. %s"
307                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
308        self.log_command_output(output, error)
309        output, error = self.execute_command("sed -i '/loglevel_stats, /c \\{loglevel_stats, %s\}'. %s"
310                                             % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
311        self.log_command_output(output, error)
312        output, error = self.execute_command("sed -i '/loglevel_rebalance, /c \\{loglevel_rebalance, %s\}'. %s"
313                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
314        self.log_command_output(output, error)
315        output, error = self.execute_command("sed -i '/loglevel_cluster, /c \\{loglevel_cluster, %s\}'. %s"
316                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
317        self.log_command_output(output, error)
318        output, error = self.execute_command("sed -i '/loglevel_views, /c \\{loglevel_views, %s\}'. %s"
319                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
320        self.log_command_output(output, error)
321        output, error = self.execute_command("sed -i '/loglevel_error_logger, /c \\{loglevel_error_logger, %s\}'. %s"
322                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
323        self.log_command_output(output, error)
324        output, error = self.execute_command("sed -i '/loglevel_mapreduce_errors, /c \\{loglevel_mapreduce_errors, %s\}'. %s"
325                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
326        self.log_command_output(output, error)
327        output, error = self.execute_command("sed -i '/loglevel_user, /c \\{loglevel_user, %s\}'. %s"
328                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
329        self.log_command_output(output, error)
330        output, error = self.execute_command("sed -i '/loglevel_xdcr, /c \\{loglevel_xdcr, %s\}'. %s"
331                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
332        self.log_command_output(output, error)
333        output, error = self.execute_command("sed -i '/loglevel_menelaus, /c \\{loglevel_menelaus, %s\}'. %s"
334                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
335        self.log_command_output(output, error)
336
337    def configure_log_location(self, new_log_location):
338        mv_logs = testconstants.LINUX_LOG_PATH + '/' + new_log_location
339        print " MV LOGS %s" % mv_logs
340        error_log_tag = "error_logger_mf_dir"
341        # ADD NON_ROOT user config_details
342        log.info("CHANGE LOG LOCATION TO %s".format(mv_logs))
343        output, error = self.execute_command("rm -rf %s" % mv_logs)
344        self.log_command_output(output, error)
345        output, error = self.execute_command("mkdir %s" % mv_logs)
346        self.log_command_output(output, error)
347        output, error = self.execute_command("chown -R couchbase %s" % mv_logs)
348        self.log_command_output(output, error)
349        output, error = self.execute_command("sed -i '/%s, /c \\{%s, \"%s\"\}.' %s"
350                                        % (error_log_tag, error_log_tag, mv_logs, testconstants.LINUX_STATIC_CONFIG))
351        self.log_command_output(output, error)
352
353    def change_stat_periodicity(self, ticks):
354        # ADD NON_ROOT user config_details
355        log.info("CHANGE STAT PERIODICITY TO every %s seconds" % ticks)
356        output, error = self.execute_command("sed -i '$ a\{grab_stats_every_n_ticks, %s}.'  %s"
357        % (ticks, testconstants.LINUX_STATIC_CONFIG))
358        self.log_command_output(output, error)
359
360    def change_port_static(self, new_port):
361        # ADD NON_ROOT user config_details
362        log.info("=========CHANGE PORTS for REST: %s, MCCOUCH: %s,MEMCACHED: %s, MOXI: %s, CAPI: %s==============="
363                % (new_port, new_port + 1, new_port + 2, new_port + 3, new_port + 4))
364        output, error = self.execute_command("sed -i '/{rest_port/d' %s" % testconstants.LINUX_STATIC_CONFIG)
365        self.log_command_output(output, error)
366        output, error = self.execute_command("sed -i '$ a\{rest_port, %s}.' %s"
367                                             % (new_port, testconstants.LINUX_STATIC_CONFIG))
368        self.log_command_output(output, error)
369        output, error = self.execute_command("sed -i '/{mccouch_port/d' %s" % testconstants.LINUX_STATIC_CONFIG)
370        self.log_command_output(output, error)
371        output, error = self.execute_command("sed -i '$ a\{mccouch_port, %s}.' %s"
372                                             % (new_port + 1, testconstants.LINUX_STATIC_CONFIG))
373        self.log_command_output(output, error)
374        output, error = self.execute_command("sed -i '/{memcached_port/d' %s" % testconstants.LINUX_STATIC_CONFIG)
375        self.log_command_output(output, error)
376        output, error = self.execute_command("sed -i '$ a\{memcached_port, %s}.' %s"
377                                             % (new_port + 2, testconstants.LINUX_STATIC_CONFIG))
378        self.log_command_output(output, error)
379        output, error = self.execute_command("sed -i '/{moxi_port/d' %s" % testconstants.LINUX_STATIC_CONFIG)
380        self.log_command_output(output, error)
381        output, error = self.execute_command("sed -i '$ a\{moxi_port, %s}.' %s"
382                                             % (new_port + 3, testconstants.LINUX_STATIC_CONFIG))
383        self.log_command_output(output, error)
384        output, error = self.execute_command("sed -i '/port = /c\port = %s' %s"
385                                             % (new_port + 4, testconstants.LINUX_CAPI_INI))
386        self.log_command_output(output, error)
387        output, error = self.execute_command("rm %s" % testconstants.LINUX_CONFIG_FILE)
388        self.log_command_output(output, error)
389        output, error = self.execute_command("cat %s" % testconstants.LINUX_STATIC_CONFIG)
390        self.log_command_output(output, error)
391
392    def is_couchbase_installed(self):
393        self.extract_remote_info()
394        if self.info.type.lower() == 'windows':
395            if self.file_exists(testconstants.WIN_CB_PATH, testconstants.VERSION_FILE):
396                # print running process on windows
397                RemoteMachineHelper(self).is_process_running('memcached')
398                RemoteMachineHelper(self).is_process_running('erl')
399                return True
400        elif self.info.distribution_type.lower() == 'mac':
401            output, error = self.execute_command('ls %s%s' % (testconstants.MAC_CB_PATH, testconstants.VERSION_FILE))
402            self.log_command_output(output, error)
403            for line in output:
404                if line.find('No such file or directory') == -1:
405                    return True
406        elif self.info.type.lower() == "linux":
407            if self.file_exists(testconstants.LINUX_CB_PATH, testconstants.VERSION_FILE):
408                return True
409        return False
410
411    def is_moxi_installed(self):
412        self.extract_remote_info()
413        if self.info.type.lower() == 'windows':
414            self.log.error('Not implemented')
415        elif self.info.distribution_type.lower() == 'mac':
416            self.log.error('Not implemented')
417        elif self.info.type.lower() == "linux":
418            if self.file_exists(testconstants.LINUX_MOXI_PATH, 'moxi'):
419                return True
420        return False
421
422    # /opt/moxi/bin/moxi -Z port_listen=11211 -u root -t 4 -O /var/log/moxi/moxi.log
423    def start_moxi(self, ip, bucket, port, user=None, threads=4, log_file="/var/log/moxi.log"):
424        if self.is_couchbase_installed():
425            prod = "couchbase"
426        else:
427            prod = "membase"
428        cli_path = "/opt/" + prod + "/bin/moxi"
429        args = ""
430        args += "http://{0}:8091/pools/default/bucketsStreaming/{1} ".format(ip, bucket)
431        args += "-Z port_listen={0} -u {1} -t {2} -O {3} -d".format(port,
432                                                                    user or prod,
433                                                                    threads, log_file)
434        self.extract_remote_info()
435        if self.info.type.lower() == "linux":
436            o, r = self.execute_command("{0} {1}".format(cli_path, args))
437            self.log_command_output(o, r)
438        else:
439            raise Exception("running standalone moxi is not supported for windows")
440
441    def stop_moxi(self):
442        self.extract_remote_info()
443        if self.info.type.lower() == "linux":
444            o, r = self.execute_command("killall -9 moxi")
445            self.log_command_output(o, r)
446        else:
447            raise Exception("stopping standalone moxi is not supported on windows")
448    def is_url_live(self, url):
449        live_url = False
450        log.info("Check if url {0} is ok".format(url))
451        status = urllib.urlopen(url).getcode()
452        if status == 200:
453            log.info("This url {0} is live".format(url))
454            live_url = True
455        else:
456            log.error("This url {0} is failed to connect".format(url))
457        return live_url
458
459    def download_build(self, build):
460        return self.download_binary(build.url, build.deliverable_type, build.name, latest_url=build.url_latest_build)
461
462    def disable_firewall(self):
463        self.extract_remote_info()
464        if self.info.type.lower() == "windows":
465            output, error = self.execute_command('netsh advfirewall set publicprofile state off')
466            self.log_command_output(output, error)
467            output, error = self.execute_command('netsh advfirewall set privateprofile state off')
468            self.log_command_output(output, error)
469            # for details see RemoteUtilHelper.enable_firewall for windows
470            output, error = self.execute_command('netsh advfirewall firewall delete rule name="block erl.exe in"')
471            self.log_command_output(output, error)
472            output, error = self.execute_command('netsh advfirewall firewall delete rule name="block erl.exe out"')
473            self.log_command_output(output, error)
474        else:
475            output, error = self.execute_command('/sbin/iptables -F')
476            self.log_command_output(output, error)
477            output, error = self.execute_command('/sbin/iptables -t nat -F')
478            self.log_command_output(output, error)
479
480    def download_binary(self, url, deliverable_type, filename, latest_url=None, skip_md5_check=True):
481        self.extract_remote_info()
482        self.disable_firewall()
483        if self.info.type.lower() == 'windows':
484            self.execute_command('taskkill /F /T /IM msiexec32.exe')
485            self.execute_command('taskkill /F /T /IM msiexec.exe')
486            self.execute_command('taskkill /F /T IM setup.exe')
487            self.execute_command('taskkill /F /T /IM ISBEW64.*')
488            self.execute_command('taskkill /F /T /IM firefox.*')
489            self.execute_command('taskkill /F /T /IM iexplore.*')
490            self.execute_command('taskkill /F /T /IM WerFault.*')
491            self.execute_command('taskkill /F /T /IM memcached.exe')
492            self.execute_command('taskkill /F /T /IM bash.exe')
493            self.disable_firewall()
494            remove_words = ["-rel", ".exe"]
495            for word in remove_words:
496                filename = filename.replace(word, "")
497            tmp = filename.split("-")
498            print "tmp at split filename", tmp
499            tmp.reverse()
500            print "tmp at reverse", tmp
501            version = tmp[1] + "-" + tmp[0]
502            print version
503            #filename = filename.replace(".exe", "")
504            exist = self.file_exists('/cygdrive/c/tmp/', '{0}.exe'.format(version))
505            if not exist:
506                output, error = self.execute_command(
507                     "cd /cygdrive/c/tmp;cmd /c 'c:\\automation\\wget.exe --no-check-certificate -q \
508                                                     {0} -O {1}.exe';ls -lh;".format(url, version))
509                self.log_command_output(output, error)
510                return self.file_exists('/cygdrive/c/tmp/', '{0}.exe'.format(version))
511            else:
512                log.info('File {0}.exe exist in tmp directory'.format(version))
513                return True
514
515
516
517#            output, error = self.execute_command("rm -rf /cygdrive/c/automation/setup.exe")
518#            self.log_command_output(output, error)
519#            output, error = self.execute_command(
520#                 "cd /cygdrive/c/tmp;cmd /c 'c:\\automation\\wget.exe --no-check-certificate -q {0} -O setup.exe';ls -lh;".format(url))
521#            self.log_command_output(output, error)
522            #return self.file_exists('/cygdrive/c/tmp/', 'setup.exe')
523        elif self.info.distribution_type.lower() == 'mac':
524            output, error = self.execute_command('cd ~/Downloads ; rm -rf couchbase-server* ; rm -rf Couchbase\ Server.app ; curl -O {0}'.format(url))
525            self.log_command_output(output, error)
526            output, error = self.execute_command('ls -lh  ~/Downloads/%s' % filename)
527            self.log_command_output(output, error)
528            for line in output:
529                if line.find('No such file or directory') == -1:
530                    return True
531            return False
532        else:
533        # try to push this build into
534        # depending on the os
535        # build.product has the full name
536        # first remove the previous file if it exist ?
537        # fix this :
538            output, error = self.execute_command_raw('cd /tmp ; D=$(mktemp -d cb_XXXX) ; mv {0} $D ; mv core.* $D ; rm -f * ; mv $D/* . ; rmdir $D'.format(filename))
539            self.log_command_output(output, error)
540            if skip_md5_check:
541                output, error = self.execute_command_raw('cd /tmp;wget -q -O {0} {1};cd /tmp;ls -lh'.format(filename, url))
542                self.log_command_output(output, error)
543            else:
544                log.info('get md5 sum for local and remote')
545                output, error = self.execute_command_raw('cd /tmp ; rm -f *.md5 *.md5l ; wget -q -O {1}.md5 {0}.md5 ; md5sum {1} > {1}.md5l'.format(url, filename))
546                self.log_command_output(output, error)
547                if str(error).find('No such file or directory') != -1 and latest_url != '':
548                    url = latest_url
549                    output, error = self.execute_command_raw('cd /tmp ; rm -f *.md5 *.md5l ; wget -q -O {1}.md5 {0}.md5 ; md5sum {1} > {1}.md5l'.format(url, filename))
550                log.info('comparing md5 sum and downloading if needed')
551                output, error = self.execute_command_raw('cd /tmp;diff {0}.md5 {0}.md5l || wget -q -O {0} {1};rm -f *.md5 *.md5l'.format(filename, url))
552                self.log_command_output(output, error)
553            # check if the file exists there now ?
554            return self.file_exists('/tmp', filename)
555            # for linux environment we can just
556            # figure out what version , check if /tmp/ has the
557            # binary and then return True if binary is installed
558
559
560    def get_file(self, remotepath, filename, todir):
561        if self.file_exists(remotepath, filename):
562            sftp = self._ssh_client.open_sftp()
563            try:
564                filenames = sftp.listdir(remotepath)
565                for name in filenames:
566                    if name == filename:
567                        sftp.get('{0}/{1}'.format(remotepath, filename), todir)
568                        sftp.close()
569                        return True
570                sftp.close()
571                return False
572            except IOError:
573                return False
574
575    def read_remote_file(self, remote_path, filename):
576        if self.file_exists(remote_path, filename):
577            sftp = self._ssh_client.open_sftp()
578            remote_file = sftp.open('{0}/{1}'.format(remote_path, filename))
579            try:
580                out = remote_file.readlines()
581            finally:
582                remote_file.close()
583            return out
584        return None
585
586    def write_remote_file(self, remote_path, filename, lines):
587        cmd = 'echo "%s" > %s/%s' % (''.join(lines), remote_path, filename)
588        self.execute_command(cmd)
589
590    def remove_directory(self, remote_path):
591        sftp = self._ssh_client.open_sftp()
592        try:
593            log.info("removing {0} directory...".format(remote_path))
594            sftp.rmdir(remote_path)
595        except IOError:
596            return False
597        finally:
598            sftp.close()
599        return True
600
601    def list_files(self, remote_path):
602        sftp = self._ssh_client.open_sftp()
603        files = []
604        try:
605            file_names = sftp.listdir(remote_path)
606            for name in file_names:
607                files.append({'path': remote_path, 'file': name})
608            sftp.close()
609        except IOError:
610            return []
611        return files
612
613    # check if this file exists in the remote
614    # machine or not
615    def file_starts_with(self, remotepath, pattern):
616        sftp = self._ssh_client.open_sftp()
617        files_matched = []
618        try:
619            file_names = sftp.listdir(remotepath)
620            for name in file_names:
621                if name.startswith(pattern):
622                    files_matched.append("{0}/{1}".format(remotepath, name))
623        except IOError:
624            # ignore this error
625            pass
626        sftp.close()
627        if len(files_matched) > 0:
628            log.info("found these files : {0}".format(files_matched))
629        return files_matched
630
631    def file_exists(self, remotepath, filename):
632        sftp = self._ssh_client.open_sftp()
633        try:
634            filenames = sftp.listdir_attr(remotepath)
635            for name in filenames:
636                if name.filename == filename and int(name.st_size) > 0:
637                    sftp.close()
638                    return True
639                elif name.filename == filename and int(name.st_size) == 0:
640                    log.info("File {0} will be deleted".format(filename))
641                    sftp.remove(remotepath + filename)
642            sftp.close()
643            return False
644        except IOError:
645            return False
646
647    def download_binary_in_win(self, url, version):
648        self.execute_command('taskkill /F /T /IM msiexec32.exe')
649        self.execute_command('taskkill /F /T /IM msiexec.exe')
650        self.execute_command('taskkill /F /T IM setup.exe')
651        self.execute_command('taskkill /F /T /IM ISBEW64.*')
652        self.execute_command('taskkill /F /T /IM iexplore.*')
653        self.execute_command('taskkill /F /T /IM WerFault.*')
654        self.execute_command('taskkill /F /T /IM Firefox.*')
655        self.execute_command('taskkill /F /T /IM bash.exe')
656        self.disable_firewall()
657        version = version.replace("-rel", "")
658        exist = self.file_exists('/cygdrive/c/tmp/', '{0}.exe'.format(version))
659        if not exist:
660            output, error = self.execute_command(
661                 "cd /cygdrive/c/tmp;cmd /c 'c:\\automation\\wget.exe --no-check-certificate -q \
662                                                     {0} -O {1}.exe';ls -lh;".format(url, version))
663            self.log_command_output(output, error)
664        else:
665            log.info('File {0}.exe exist in tmp directory'.format(version))
666        return self.file_exists('/cygdrive/c/tmp/', '{0}.exe'.format(version))
667
668    def copy_file_local_to_remote(self, src_path, des_path):
669        sftp = self._ssh_client.open_sftp()
670        try:
671            sftp.put(src_path, des_path)
672        except IOError:
673            log.error('Can not copy file')
674        finally:
675            sftp.close()
676
677    def copy_file_remote_to_local(self, rem_path, des_path):
678        sftp = self._ssh_client.open_sftp()
679        try:
680            sftp.get(rem_path, des_path)
681        except IOError:
682            log.error('Can not copy file')
683        finally:
684            sftp.close()
685
686
687    # copy multi files from local to remote server
688    def copy_files_local_to_remote(self, src_path, des_path):
689        files = os.listdir(src_path)
690        log.info("copy file from {0} to {1}".format(src_path, des_path))
691        # self.execute_batch_command("cp -r  {0}/* {1}".format(src_path, des_path))
692        for file in files:
693            if file.find("wget") != 1:
694                a = ""
695            full_src_path = os.path.join(src_path, file)
696            full_des_path = os.path.join(des_path, file)
697            self.copy_file_local_to_remote(full_src_path, full_des_path)
698
699
700    # create a remote file from input string
701    def create_file(self, remote_path, file_data):
702        output, error = self.execute_command("echo '{0}' > {1}".format(file_data, remote_path))
703
704    def find_file(self, remote_path, file):
705        sftp = self._ssh_client.open_sftp()
706        try:
707            files = sftp.listdir(remote_path)
708            for name in files:
709                if name == file:
710                    found_it = os.path.join(remote_path, name)
711                    log.info("File {0} was found".format(found_it))
712                    return found_it
713            else:
714                log.error('File(s) name in {0}'.format(remote_path))
715                for name in files:
716                    log.info(name)
717                log.error('Can not find {0}'.format(file))
718        except IOError:
719            pass
720        sftp.close()
721
722    def find_build_version(self, path_to_version, version_file, product):
723        sftp = self._ssh_client.open_sftp()
724        ex_type = "exe"
725        version_file = testconstants.VERSION_FILE
726        if self.file_exists(testconstants.WIN_CB_PATH, testconstants.VERSION_FILE):
727            path_to_version = testconstants.WIN_CB_PATH
728        else:
729            path_to_version = testconstants.WIN_MB_PATH
730        try:
731            log.info(path_to_version)
732            f = sftp.open(os.path.join(path_to_version, version_file), 'r+')
733            tmp_str = f.read().strip()
734            full_version = tmp_str.replace("-rel", "")
735            if full_version == "1.6.5.4-win64":
736                full_version = "1.6.5.4"
737            build_name = short_version = full_version
738            return build_name, short_version, full_version
739        except IOError:
740            log.error('Can not read version file')
741        sftp.close()
742
743    def find_windows_info(self):
744        found = self.find_file("/cygdrive/c/tmp", "windows_info.txt")
745        if isinstance(found, str):
746            sftp = self._ssh_client.open_sftp()
747            try:
748                f = sftp.open(found)
749                log.info("get windows information")
750                info = {}
751                for line in f:
752                    (key, value) = line.split('=')
753                    key = key.strip(' \t\n\r')
754                    value = value.strip(' \t\n\r')
755                    info[key] = value
756                return info
757            except IOError:
758                log.error("can not find windows info file")
759            sftp.close()
760        else:
761            return self.create_windows_info()
762
763    def create_windows_info(self):
764            systeminfo = self.get_windows_system_info()
765            info = {}
766            if "OS Name" in systeminfo:
767                info["os"] = systeminfo["OS Name"].find("indows") and "windows" or "NONE"
768                info["os_name"] = systeminfo["OS Name"].find("2008 R2") and "2k8" or "NONE"
769            if "System Type" in systeminfo:
770                info["os_arch"] = systeminfo["System Type"].find("64") and "x86_64" or "NONE"
771            info.update(systeminfo)
772            self.execute_batch_command("rm -rf  /cygdrive/c/tmp/windows_info.txt")
773            self.execute_batch_command("touch  /cygdrive/c/tmp/windows_info.txt")
774            sftp = self._ssh_client.open_sftp()
775            try:
776                f = sftp.open('/cygdrive/c/tmp/windows_info.txt', 'w')
777                content = ''
778                for key in sorted(info.keys()):
779                    content += '{0} = {1}\n'.format(key, info[key])
780                f.write(content)
781                log.info("/cygdrive/c/tmp/windows_info.txt was created with content: {0}".format(content))
782            except IOError:
783                log.error('Can not write windows_info.txt file')
784            finally:
785                sftp.close()
786            return info
787
788    # Need to add new windows register ID in testconstant file when
789    # new couchbase server version comes out.
790    def create_windows_capture_file(self, task, product, version):
791        src_path = "resources/windows/automation"
792        des_path = "/cygdrive/c/automation"
793
794        # remove dot in version (like 2.0.0 ==> 200)
795        reg_version = version[0:5:2]
796        reg_id = WIN_REGISTER_ID[reg_version]
797
798        if task == "install":
799            template_file = "cb-install.wct"
800            file = "install.iss"
801        elif task == "uninstall":
802            template_file = "cb-uninstall.wct"
803            file = "uninstall.iss"
804
805        # create in/uninstall file from windows capture template (wct) file
806        full_src_path_template = os.path.join(src_path, template_file)
807        full_src_path = os.path.join(src_path, file)
808        full_des_path = os.path.join(des_path, file)
809
810        f1 = open(full_src_path_template, 'r')
811        f2 = open(full_src_path, 'w')
812        # replace ####### with reg ID to install/uninstall
813        if "2.2.0-837" in version:
814            reg_id = "2B630EB8-BBC7-6FE4-C9B8-D8843EB1EFFA"
815        log.info("register ID: {0}".format(reg_id))
816        for line in f1:
817            line = line.replace("#######", reg_id)
818            if product == "mb" and task == "install":
819                line = line.replace("Couchbase", "Membase")
820            f2.write(line)
821        f1.close()
822        f2.close()
823
824        self.copy_file_local_to_remote(full_src_path, full_des_path)
825        # remove capture file from source after copy to destination
826        # os.remove(full_src_path)
827
828    def get_windows_system_info(self):
829        try:
830            info = {}
831            o = self.execute_batch_command('systeminfo')
832            for line in o:
833                line_list = line.split(':')
834                if len(line_list) > 2:
835                    if line_list[0] == 'Virtual Memory':
836                        key = "".join(line_list[0:2])
837                        value = " ".join(line_list[2:])
838                    else:
839                        key = line_list[0]
840                        value = " ".join(line_list[1:])
841                elif len(line_list) == 2:
842                    (key, value) = line_list
843                else:
844                    continue
845                key = key.strip(' \t\n\r')
846                if key.find("[") != -1:
847                    info[key_prev] += '|' + key + value.strip(' |')
848                else:
849                    value = value.strip(' |')
850                    info[key] = value
851                    key_prev = key
852            return info
853        except Exception as ex:
854            log.error("error {0} appeared during getting  windows info".format(ex))
855
856    # this function used to modify bat file to run task schedule in windows
857    def modify_bat_file(self, remote_path, file_name, name, version, task):
858        found = self.find_file(remote_path, file_name)
859        sftp = self._ssh_client.open_sftp()
860
861        product_version = ""
862        if version[:5] in MEMBASE_VERSIONS:
863            product_version = version[:5]
864            name = "mb"
865        elif version[:5] in COUCHBASE_VERSIONS:
866            product_version = version[:5]
867            name = "cb"
868        else:
869            log.error('Windows automation does not support {0} version yet'.format(version))
870            sys.exit()
871
872        try:
873            f = sftp.open(found, 'w')
874            name = name.strip()
875            version = version.strip()
876            if task == "upgrade":
877                content = 'c:\\tmp\{3}.exe /s -f1c:\\automation\{0}_{1}_{2}.iss'.format(name,
878                                                                         product_version, task, version)
879            else:
880                content = 'c:\\tmp\{0}.exe /s -f1c:\\automation\{1}.iss'.format(version, task)
881            log.info("create {0} task with content:{1}".format(task, content))
882            f.write(content)
883            log.info('Successful write to {0}'.format(found))
884        except IOError:
885            log.error('Can not write build name file to bat file {0}'.format(found))
886        sftp.close()
887
888    def set_vbuckets_win(self, vbuckets):
889        bin_path = WIN_COUCHBASE_BIN_PATH
890        bin_path = bin_path.replace("\\", "")
891        src_file = bin_path + "service_register.bat"
892        des_file = "/tmp/service_register.bat_{0}".format(self.ip)
893        local_file = "/tmp/service_register.bat.tmp_{0}".format(self.ip)
894
895        self.copy_file_remote_to_local(src_file, des_file)
896        f1 = open(des_file, "r")
897        f2 = open(local_file, "w")
898        """ when install new cb server on windows, there is not
899            env COUCHBASE_NUM_VBUCKETS yet.  We need to insert this
900            env to service_register.bat right after  ERL_FULLSWEEP_AFTER 512
901            like -env ERL_FULLSWEEP_AFTER 512 -env COUCHBASE_NUM_VBUCKETS vbuckets
902            where vbucket is params passed to function when run install scripts """
903        for line in f1:
904            if "-env COUCHBASE_NUM_VBUCKETS " in line:
905                tmp1 = line.split("COUCHBASE_NUM_VBUCKETS")
906                tmp2 = tmp1[1].strip().split(" ")
907                log.info("set vbuckets of node {0} to {1}" \
908                                 .format(self.ip, vbuckets))
909                tmp2[0] = vbuckets
910                tmp1[1] = " ".join(tmp2)
911                line = "COUCHBASE_NUM_VBUCKETS ".join(tmp1)
912            elif "-env ERL_FULLSWEEP_AFTER 512" in line:
913                log.info("set vbuckets of node {0} to {1}" \
914                                 .format(self.ip, vbuckets))
915                line = line.replace("-env ERL_FULLSWEEP_AFTER 512", \
916                  "-env ERL_FULLSWEEP_AFTER 512 -env COUCHBASE_NUM_VBUCKETS {0}" \
917                                 .format(vbuckets))
918            f2.write(line)
919        f1.close()
920        f2.close()
921        self.copy_file_local_to_remote(local_file, src_file)
922
923        """ re-register new setup to cb server """
924        self.execute_command(WIN_COUCHBASE_BIN_PATH + "service_stop.bat")
925        self.execute_command(WIN_COUCHBASE_BIN_PATH + "service_unregister.bat")
926        self.execute_command(WIN_COUCHBASE_BIN_PATH + "service_register.bat")
927        self.execute_command(WIN_COUCHBASE_BIN_PATH + "service_start.bat")
928        self.sleep(10, "wait for cb server start completely after reset vbuckets!")
929
930        """ remove temporary files on slave """
931        os.remove(local_file)
932        os.remove(des_file)
933
934
935    def create_directory(self, remote_path):
936        sftp = self._ssh_client.open_sftp()
937        try:
938            sftp.stat(remote_path)
939        except IOError, e:
940            if e[0] == 2:
941                log.info("Directory at {0} DOES NOT exist. We will create on here".format(remote_path))
942                sftp.mkdir(remote_path)
943                sftp.close()
944                return False
945            raise
946        else:
947            log.error("Directory at {0} DOES exist. Fx returns True".format(remote_path))
948            return True
949
950    # this function will remove the automation directory in windows
951    def create_multiple_dir(self, dir_paths):
952        sftp = self._ssh_client.open_sftp()
953        try:
954            for dir_path in dir_paths:
955                if dir_path != '/cygdrive/c/tmp':
956                    output = self.remove_directory('/cygdrive/c/automation')
957                    if output:
958                        log.info("{0} directory is removed.".format(dir_path))
959                    else:
960                        log.error("Can not delete {0} directory or directory {0} does not exist.".format(dir_path))
961                self.create_directory(dir_path)
962            sftp.close()
963        except IOError:
964            pass
965
966    def membase_upgrade(self, build, save_upgrade_config=False, forcefully=False):
967        # upgrade couchbase server
968        self.extract_remote_info()
969        log.info('deliverable_type : {0}'.format(self.info.deliverable_type))
970        log.info('/tmp/{0} or /tmp/{1}'.format(build.name, build.product))
971        command = ''
972        if self.info.type.lower() == 'windows':
973                print "build name in membase upgrade    ", build.product_version
974                self.membase_upgrade_win(self.info.architecture_type, self.info.windows_name, build.product_version)
975                log.info('********* continue upgrade process **********')
976
977        elif self.info.deliverable_type == 'rpm':
978            # run rpm -i to install
979            if save_upgrade_config:
980                self.membase_uninstall(save_upgrade_config=save_upgrade_config)
981                install_command = 'rpm -i /tmp/{0}'.format(build.name)
982                command = 'INSTALL_UPGRADE_CONFIG_DIR=/opt/membase/var/lib/membase/config {0}'.format(install_command)
983            else:
984                command = 'rpm -U /tmp/{0}'.format(build.name)
985                if forcefully:
986                    command = 'rpm -U --force /tmp/{0}'.format(build.name)
987        elif self.info.deliverable_type == 'deb':
988            if save_upgrade_config:
989                self.membase_uninstall(save_upgrade_config=save_upgrade_config)
990                install_command = 'dpkg -i /tmp/{0}'.format(build.name)
991                command = 'INSTALL_UPGRADE_CONFIG_DIR=/opt/membase/var/lib/membase/config {0}'.format(install_command)
992            else:
993                command = 'dpkg -i /tmp/{0}'.format(build.name)
994                if forcefully:
995                    command = 'dpkg -i --force /tmp/{0}'.format(build.name)
996        output, error = self.execute_command(command, use_channel=True)
997        self.log_command_output(output, error)
998        return output, error
999
1000    def couchbase_upgrade(self, build, save_upgrade_config=False, forcefully=False):
1001        # upgrade couchbase server
1002        self.extract_remote_info()
1003        log.info('deliverable_type : {0}'.format(self.info.deliverable_type))
1004        log.info('/tmp/{0} or /tmp/{1}'.format(build.name, build.product))
1005        command = ''
1006        if self.info.type.lower() == 'windows':
1007                print "build name in couchbase upgrade    ", build.product_version
1008                self.couchbase_upgrade_win(self.info.architecture_type, self.info.windows_name, build.product_version)
1009                log.info('********* continue upgrade process **********')
1010
1011        elif self.info.deliverable_type == 'rpm':
1012            # run rpm -i to install
1013            if save_upgrade_config:
1014                self.couchbase_uninstall(save_upgrade_config=save_upgrade_config)
1015                install_command = 'rpm -i /tmp/{0}'.format(build.name)
1016                command = 'INSTALL_UPGRADE_CONFIG_DIR=/opt/couchbase/var/lib/membase/config {0}'.format(install_command)
1017            else:
1018                command = 'rpm -U /tmp/{0}'.format(build.name)
1019                if forcefully:
1020                    command = 'rpm -U --force /tmp/{0}'.format(build.name)
1021        elif self.info.deliverable_type == 'deb':
1022            if save_upgrade_config:
1023                self.couchbase_uninstall(save_upgrade_config=save_upgrade_config)
1024                install_command = 'dpkg -i /tmp/{0}'.format(build.name)
1025                command = 'INSTALL_UPGRADE_CONFIG_DIR=/opt/couchbase/var/lib/membase/config {0}'.format(install_command)
1026            else:
1027                command = 'dpkg -i /tmp/{0}'.format(build.name)
1028                if forcefully:
1029                    command = 'dpkg -i --force /tmp/{0}'.format(build.name)
1030        output, error = self.execute_command(command, use_channel=True)
1031        self.log_command_output(output, error)
1032        return output, error
1033
1034    def membase_upgrade_win(self, architecture, windows_name, version):
1035        task = "upgrade"
1036        bat_file = "upgrade.bat"
1037        version_file = "VERSION.txt"
1038        print "version in membase upgrade windows ", version
1039        deleted = False
1040        self.modify_bat_file('/cygdrive/c/automation', bat_file, 'cb', version, task)
1041        self.stop_schedule_tasks()
1042        output, error = self.execute_command("cat '/cygdrive/c/Program Files/Couchbase/Server/VERSION.txt'")
1043        log.info("version to upgrade: {0}".format(output))
1044        log.info('before running task schedule upgrademe')
1045        if '1.8.0' in str(output):
1046            # run installer in second time as workaround for upgrade 1.8.0 only:
1047            # Installer needs to update registry value in order to upgrade from the previous version.
1048            # Please run installer again to continue."""
1049            output, error = self.execute_command("cmd /c schtasks /run /tn upgrademe")
1050            self.log_command_output(output, error)
1051            self.sleep(200, "because upgrade version is {0}".format(output))
1052            output, error = self.execute_command("cmd /c schtasks /Query /FO LIST /TN upgrademe /V")
1053            self.log_command_output(output, error)
1054            self.stop_schedule_tasks()
1055        # run task schedule to upgrade Membase server
1056        output, error = self.execute_command("cmd /c schtasks /run /tn upgrademe")
1057        self.log_command_output(output, error)
1058        deleted = self.wait_till_file_deleted(testconstants.WIN_CB_PATH, version_file, timeout_in_seconds=600)
1059        if not deleted:
1060            log.error("Uninstall was failed at node {0}".format(self.ip))
1061            sys.exit()
1062        self.wait_till_file_added(testconstants.WIN_CB_PATH, version_file, timeout_in_seconds=600)
1063        log.info("installed version:")
1064        output, error = self.execute_command("cat '/cygdrive/c/Program Files/Couchbase/Server/VERSION.txt'")
1065        """   """
1066        self.wait_till_process_ended(full_version[:10])
1067        self.sleep(10, "wait for server to start up completely")
1068        ct = time.time()
1069        while time.time() - ct < 10800:
1070            output, error = self.execute_command("cmd /c schtasks /Query /FO LIST /TN upgrademe /V| findstr Status ")
1071            if "Ready" in str(output):
1072                log.info("upgrademe task complteted")
1073                break
1074            elif "Could not start":
1075                log.exception("Ugrade failed!!!")
1076                break
1077            else:
1078                log.info("upgrademe task still running:{0}".format(output))
1079                self.sleep(30)
1080        output, error = self.execute_command("cmd /c schtasks /Query /FO LIST /TN upgrademe /V")
1081        self.log_command_output(output, error)
1082
1083    def couchbase_upgrade_win(self, architecture, windows_name, version):
1084        task = "upgrade"
1085        bat_file = "upgrade.bat"
1086        version_file = "VERSION.txt"
1087        print "version in couchbase upgrade windows ", version
1088        deleted = False
1089        self.modify_bat_file('/cygdrive/c/automation', bat_file, 'cb', version, task)
1090        self.stop_schedule_tasks()
1091        self.remove_win_backup_dir()
1092        self.remove_win_collect_tmp()
1093        output, error = self.execute_command("cat '/cygdrive/c/Program Files/Couchbase/Server/VERSION.txt'")
1094        log.info("version to upgrade: {0}".format(output))
1095        log.info('before running task schedule upgrademe')
1096        if '1.8.0' in str(output):
1097            # run installer in second time as workaround for upgrade 1.8.0 only:
1098            # Installer needs to update registry value in order to upgrade from the previous version.
1099            # Please run installer again to continue."""
1100            output, error = self.execute_command("cmd /c schtasks /run /tn upgrademe")
1101            self.log_command_output(output, error)
1102            self.sleep(200, "because upgrade version is {0}".format(output))
1103            output, error = self.execute_command("cmd /c schtasks /Query /FO LIST /TN upgrademe /V")
1104            self.log_command_output(output, error)
1105            self.stop_schedule_tasks()
1106        # run task schedule to upgrade Membase server
1107        output, error = self.execute_command("cmd /c schtasks /run /tn upgrademe")
1108        self.log_command_output(output, error)
1109        deleted = self.wait_till_file_deleted(testconstants.WIN_CB_PATH, version_file, timeout_in_seconds=600)
1110        if not deleted:
1111            log.error("Uninstall was failed at node {0}".format(self.ip))
1112            sys.exit()
1113        self.wait_till_file_added(testconstants.WIN_CB_PATH, version_file, timeout_in_seconds=600)
1114        log.info("installed version: {0}".format(version))
1115        output, error = self.execute_command("cat '/cygdrive/c/Program Files/Couchbase/Server/VERSION.txt'")
1116        self.wait_till_process_ended(version[:10])
1117        self.sleep(10, "wait for server to start up completely")
1118        ct = time.time()
1119        while time.time() - ct < 10800:
1120            output, error = self.execute_command("cmd /c schtasks /Query /FO LIST /TN upgrademe /V| findstr Status ")
1121            if "Ready" in str(output):
1122                log.info("upgrademe task complteted")
1123                break
1124            elif "Could not start":
1125                log.exception("Ugrade failed!!!")
1126                break
1127            else:
1128                log.info("upgrademe task still running:{0}".format(output))
1129                self.sleep(30)
1130        output, error = self.execute_command("cmd /c schtasks /Query /FO LIST /TN upgrademe /V")
1131        self.log_command_output(output, error)
1132
1133    def install_server(self, build, startserver=True, path='/tmp', vbuckets=None,
1134                       swappiness=10, force=False, openssl='', upr=None, xdcr_upr=None):
1135        server_type = None
1136        success = True
1137        track_words = ("warning", "error", "fail")
1138        if build.name.lower().find("membase") != -1:
1139            server_type = 'membase'
1140            abbr_product = "mb"
1141        elif build.name.lower().find("couchbase") != -1:
1142            server_type = 'couchbase'
1143            abbr_product = "cb"
1144        else:
1145            raise Exception("its not a membase or couchbase?")
1146        self.extract_remote_info()
1147        log.info('deliverable_type : {0}'.format(self.info.deliverable_type))
1148        if self.info.type.lower() == 'windows':
1149            win_processes = ["msiexec32.exe", "msiexec32.exe", "setup.exe", "ISBEW64.*",
1150                             "firefox.*", "WerFault.*", "iexplore.*"]
1151            self.terminate_processes(self.info, win_processes)
1152            # to prevent getting full disk let's delete some large files
1153            self.remove_win_backup_dir()
1154            self.remove_win_collect_tmp()
1155            output, error = self.execute_command("cmd /c schtasks /run /tn installme")
1156            success &= self.log_command_output(output, error, track_words)
1157            file_check = 'VERSION.txt'
1158            self.wait_till_file_added("/cygdrive/c/Program Files/{0}/Server/".format(server_type.title()), file_check,
1159                                          timeout_in_seconds=600)
1160            output, error = self.execute_command("cmd /c schtasks /Query /FO LIST /TN installme /V")
1161            self.log_command_output(output, error)
1162            self.wait_till_process_ended(build.product_version[:10])
1163            self.sleep(10, "wait for server to start up completely")
1164            if vbuckets:
1165                self.set_vbuckets_win(vbuckets)
1166
1167            # output, error = self.execute_command("cmd rm /cygdrive/c/tmp/{0}*.exe".format(build_name))
1168            # self.log_command_output(output, error)
1169        elif self.info.deliverable_type in ["rpm", "deb"]:
1170            if startserver and vbuckets == None:
1171                environment = ""
1172            else:
1173                environment = "INSTALL_DONT_START_SERVER=1 "
1174            log.info('/tmp/{0} or /tmp/{1}'.format(build.name, build.product))
1175
1176            # set default swappiness to 10 unless specify in params in all unix environment
1177            output, error = self.execute_command('/sbin/sysctl vm.swappiness={0}'.format(swappiness))
1178            success &= self.log_command_output(output, error, track_words)
1179
1180            if self.info.deliverable_type == 'rpm':
1181                self.check_openssl_version(self.info.deliverable_type, openssl)
1182                self.check_pkgconfig(self.info.deliverable_type, openssl)
1183                if force:
1184                    output, error = self.execute_command('{0}rpm -Uvh --force /tmp/{1}'.format(environment, build.name))
1185                else:
1186                    output, error = self.execute_command('{0}rpm -i /tmp/{1}'.format(environment, build.name))
1187            elif self.info.deliverable_type == 'deb':
1188                self.check_openssl_version(self.info.deliverable_type, openssl)
1189                self.install_missing_lib()
1190                if force:
1191                    output, error = self.execute_command('{0}dpkg --force-all -i /tmp/{1}'.format(environment, build.name))
1192                else:
1193                    output, error = self.execute_command('{0}dpkg -i /tmp/{1}'.format(environment, build.name))
1194            success &= self.log_command_output(output, error, track_words)
1195            self.create_directory(path)
1196            output, error = self.execute_command('/opt/{0}/bin/{1}enable_core_dumps.sh  {2}'.
1197                                    format(server_type, abbr_product, path))
1198            success &= self.log_command_output(output, error, track_words)
1199
1200            if vbuckets:
1201                output, error = self.execute_command("sed -i 's/ulimit -c unlimited/ulimit -c unlimited\\n    export {0}_NUM_VBUCKETS={1}/' /opt/{2}/etc/{2}_init.d".
1202                                                    format(server_type.upper(), vbuckets, server_type))
1203                success &= self.log_command_output(output, error, track_words)
1204            if upr is not None:
1205                protocol = "tap"
1206                if upr:
1207                    protocol = "upr"
1208                output, error = \
1209                self.execute_command("sed -i 's/END INIT INFO/END INIT INFO\\nexport COUCHBASE_REPL_TYPE={1}/'\
1210                    /opt/{0}/etc/{0}_init.d".format(server_type, protocol))
1211                success &= self.log_command_output(output, error, track_words)
1212            if xdcr_upr == False:
1213                output, error = \
1214                    self.execute_command("sed -i 's/ulimit -c unlimited/ulimit -c unlimited\\n    export XDCR_USE_OLD_PATH={1}/'\
1215                    /opt/{0}/etc/{0}_init.d".format(server_type, "true"))
1216                success &= self.log_command_output(output, error, track_words)
1217
1218            # skip output: [WARNING] couchbase-server is already started
1219            # dirname error skipping for CentOS-6.6 (MB-12536)
1220            track_words = ("error", "fail", "dirname")
1221            if startserver:
1222                output, error = self.execute_command('/etc/init.d/{0}-server start'.format(server_type))
1223
1224                if (build.product_version.startswith("1.") or build.product_version.startswith("2.0.0"))\
1225                        and build.deliverable_type == "deb":
1226                    # skip error '* Failed to start couchbase-server' for 1.* & 2.0.0 builds(MB-7288)
1227                    # fix in 2.0.1 branch Change-Id: I850ad9424e295bbbb79ede701495b018b5dfbd51
1228                    log.warn("Error '* Failed to start couchbase-server' for 1.* builds will be skipped")
1229                    self.log_command_output(output, error, track_words)
1230                else:
1231                    success &= self.log_command_output(output, error, track_words)
1232        elif self.info.deliverable_type in ["zip"]:
1233            o, r = self.execute_command("ps aux | grep Archive | awk '{print $2}' | xargs kill -9")
1234            self.log_command_output(o, r)
1235            self.sleep(30)
1236            output, error = self.execute_command("cd ~/Downloads ; open couchbase-server*.zip")
1237            self.log_command_output(output, error)
1238            self.sleep(30)
1239            output, error = self.execute_command("mv ~/Downloads/couchbase-server*/Couchbase\ Server.app /Applications/")
1240            self.log_command_output(output, error)
1241            output, error = self.execute_command("xattr -d -r com.apple.quarantine /Applications/Couchbase\ Server.app")
1242            self.log_command_output(output, error)
1243            output, error = self.execute_command("open /Applications/Couchbase\ Server.app")
1244            self.log_command_output(output, error)
1245
1246        output, error = self.execute_command("rm -f *-diag.zip")
1247        self.log_command_output(output, error, track_words)
1248        return success
1249
1250    def install_server_win(self, build, version, startserver=True, vbuckets=None):
1251        remote_path = None
1252        success = True
1253        track_words = ("warning", "error", "fail")
1254        if build.name.lower().find("membase") != -1:
1255            remote_path = testconstants.WIN_MB_PATH
1256            abbr_product = "mb"
1257        elif build.name.lower().find("couchbase") != -1:
1258            remote_path = testconstants.WIN_CB_PATH
1259            abbr_product = "cb"
1260
1261        if remote_path is None:
1262            raise Exception("its not a membase or couchbase?")
1263        self.extract_remote_info()
1264        log.info('deliverable_type : {0}'.format(self.info.deliverable_type))
1265        if self.info.type.lower() == 'windows':
1266            task = "install"
1267            bat_file = "install.bat"
1268            dir_paths = ['/cygdrive/c/automation', '/cygdrive/c/tmp']
1269            # build = self.build_url(params)
1270            self.create_multiple_dir(dir_paths)
1271            self.copy_files_local_to_remote('resources/windows/automation', '/cygdrive/c/automation')
1272            self.create_windows_capture_file(task, abbr_product, version)
1273            self.modify_bat_file('/cygdrive/c/automation', bat_file, abbr_product, version, task)
1274            self.stop_schedule_tasks()
1275            self.remove_win_backup_dir()
1276            self.remove_win_collect_tmp()
1277            log.info('sleep for 5 seconds before running task schedule install me')
1278            time.sleep(5)
1279            """ run task schedule to install cb server """
1280            output, error = self.execute_command("cmd /c schtasks /run /tn installme")
1281            success &= self.log_command_output(output, error, track_words)
1282            file_check = 'VERSION.txt'
1283            self.wait_till_file_added(remote_path, file_check, timeout_in_seconds=600)
1284            self.wait_till_process_ended(build.product_version[:10])
1285            self.sleep(10, "wait for server to start up completely")
1286            output, error = self.execute_command("rm -f *-diag.zip")
1287            self.log_command_output(output, error, track_words)
1288            if vbuckets:
1289                self.set_vbuckets_win(vbuckets)
1290            return success
1291
1292
1293    def install_moxi(self, build):
1294        success = True
1295        track_words = ("warning", "error", "fail")
1296        self.extract_remote_info()
1297        log.info('deliverable_type : {0}'.format(self.info.deliverable_type))
1298        if self.info.type.lower() == 'windows':
1299            self.log.error('Not implemented')
1300        elif self.info.deliverable_type in ["rpm"]:
1301            output, error = self.execute_command('rpm -i /tmp/{0}'.format(build.name))
1302            if error and ' '.join(error).find("ERROR") != -1:
1303                success = False
1304        elif self.info.deliverable_type == 'deb':
1305            output, error = self.execute_command('dpkg -i /tmp/{0}'.format(build.name))
1306            if error and ' '.join(error).find("ERROR") != -1:
1307                success = False
1308        success &= self.log_command_output(output, '', track_words)
1309        return success
1310
1311    def wait_till_file_deleted(self, remotepath, filename, timeout_in_seconds=180):
1312        end_time = time.time() + float(timeout_in_seconds)
1313        deleted = False
1314        while time.time() < end_time and not deleted:
1315            # get the process list
1316            exists = self.file_exists(remotepath, filename)
1317            if exists:
1318                log.error('at {2} file still exists : {0}{1}'.format(remotepath, filename, self.ip))
1319                time.sleep(5)
1320            else:
1321                log.info('at {2} FILE DOES NOT EXIST ANYMORE : {0}{1}'.format(remotepath, filename, self.ip))
1322                deleted = True
1323        return deleted
1324
1325    def wait_till_file_added(self, remotepath, filename, timeout_in_seconds=180):
1326        end_time = time.time() + float(timeout_in_seconds)
1327        added = False
1328        while time.time() < end_time and not added:
1329            # get the process list
1330            exists = self.file_exists(remotepath, filename)
1331            if not exists:
1332                log.error('at {2} file does not exist : {0}{1}'.format(remotepath, filename, self.ip))
1333                time.sleep(5)
1334            else:
1335                log.info('at {2} FILE EXISTS : {0}{1}'.format(remotepath, filename, self.ip))
1336                added = True
1337        return added
1338
1339    def wait_till_compaction_end(self, rest, bucket, timeout_in_seconds=60):
1340        end_time = time.time() + float(timeout_in_seconds)
1341        compaction_started = False
1342        while time.time() < end_time:
1343            status, progress = rest.check_compaction_status(bucket)
1344            if status:
1345                log.info("compaction progress is %s" % progress)
1346                time.sleep(1)
1347                compaction_started = True
1348            elif compaction_started:
1349                return True
1350            else:
1351                log.info("auto compaction is not started yet.")
1352                time.sleep(1)
1353        log.error("auto compaction is not started in {0} sec.".format(str(timeout_in_seconds)))
1354        return False
1355
1356    def wait_till_process_ended(self, process_name, timeout_in_seconds=240):
1357        end_time = time.time() + float(timeout_in_seconds)
1358        process_ended = False
1359        process_running = False
1360        while time.time() < end_time and not process_ended:
1361            output, error = self.execute_command("tasklist | grep {0}" \
1362                                                 .format(process_name))
1363            self.log_command_output(output, error)
1364            if output and process_name in output[0]:
1365                self.sleep(5, "wait for process ended!")
1366                process_running = True
1367            else:
1368                if process_running:
1369                    log.info("{1}: Alright, PROCESS {0} ENDED!" \
1370                             .format(process_name, self.ip))
1371                    process_ended = True
1372                else:
1373                    log.error("{1}: process {0} may not run" \
1374                              .format(process_name, self.ip))
1375        return process_ended
1376
1377    def terminate_processes(self, info, list):
1378        for process in list:
1379            type = info.distribution_type.lower()
1380            if type == "windows":
1381                self.execute_command("taskkill /F /T /IM {0}".format(process))
1382            elif type in ["ubuntu", "centos", "red hat"]:
1383                self.terminate_process(info, process)
1384
1385    def remove_folders(self, list):
1386        for folder in list:
1387            output, error = self.execute_command("rm -rf {0}".format(folder))
1388            self.log_command_output(output, error)
1389
1390
1391    def couchbase_uninstall(self):
1392        linux_folders = ["/var/opt/membase", "/opt/membase", "/etc/opt/membase",
1393                         "/var/membase/data/*", "/opt/membase/var/lib/membase/*",
1394                         "/opt/couchbase", "/data/"]
1395        terminate_process_list = ["beam.smp", "memcached", "moxi", "vbucketmigrator",
1396                                  "couchdb", "epmd", "memsup", "cpu_sup"]
1397        version_file = "VERSION.txt"
1398        self.extract_remote_info()
1399        log.info(self.info.distribution_type)
1400        type = self.info.distribution_type.lower()
1401        if type == 'windows':
1402            product = "cb"
1403            query = BuildQuery()
1404            os_type = "exe"
1405            task = "uninstall"
1406            bat_file = "uninstall.bat"
1407            product_name = "couchbase-server-enterprise"
1408            version_path = "/cygdrive/c/Program Files/Couchbase/Server/"
1409            deleted = False
1410
1411            exist = self.file_exists(version_path, version_file)
1412            log.info("Is VERSION file existed on {0}? {1}".format(self.ip, exist))
1413            if exist:
1414                log.info("VERSION file exists.  Start to uninstall {0} on {1} server".format(product, self.ip))
1415                build_name, short_version, full_version = self.find_build_version(version_path, version_file, product)
1416                log.info('Build name: {0}'.format(build_name))
1417                build_name = build_name.rstrip() + ".exe"
1418                log.info('Check if {0} is in tmp directory on {1} server'.format(build_name, self.ip))
1419                exist = self.file_exists("/cygdrive/c/tmp/", build_name)
1420                if not exist:  # if not exist in tmp dir, start to download that version build
1421                    builds, changes = query.get_all_builds(version=full_version, \
1422                                      deliverable_type=self.info.deliverable_type, \
1423                                      architecture_type=self.info.architecture_type, \
1424                                      edition_type=product_name, repo=MV_LATESTBUILD_REPO, \
1425                                      distribution_version=self.info.distribution_version.lower())
1426                    build = query.find_build(builds, product_name, os_type, self.info.architecture_type, \
1427                                      full_version, distribution_version=self.info.distribution_version.lower())
1428                    downloaded = self.download_binary_in_win(build.url, short_version)
1429                    if downloaded:
1430                        log.info('Successful download {0}.exe on {1} server'.format(short_version, self.ip))
1431                    else:
1432                        log.error('Download {0}.exe failed'.format(short_version))
1433                dir_paths = ['/cygdrive/c/automation', '/cygdrive/c/tmp']
1434                self.create_multiple_dir(dir_paths)
1435                self.copy_files_local_to_remote('resources/windows/automation', '/cygdrive/c/automation')
1436                self.stop_couchbase()
1437                # modify bat file to run uninstall schedule task
1438                self.create_windows_capture_file(task, product, full_version)
1439                self.modify_bat_file('/cygdrive/c/automation', bat_file, product, short_version, task)
1440                self.stop_schedule_tasks()
1441                log.info('sleep for 5 seconds before running task schedule uninstall on {0}'.format(self.ip))
1442
1443                """ the code below need to remove when bug MB-11328 is fixed in 3.0.1 """
1444                output, error = self.kill_erlang(os="windows")
1445                self.log_command_output(output, error)
1446                """ end remove code """
1447
1448                time.sleep(5)
1449                # run schedule task uninstall couchbase server
1450                output, error = self.execute_command("cmd /c schtasks /run /tn removeme")
1451                self.log_command_output(output, error)
1452                deleted = self.wait_till_file_deleted(version_path, version_file, timeout_in_seconds=600)
1453                if not deleted:
1454                    log.error("Uninstall was failed at node {0}".format(self.ip))
1455                    sys.exit()
1456                self.wait_till_process_ended(full_version[:10])
1457                self.sleep(10, "next step is to install")
1458                """ delete binary after uninstall """
1459                output, error = self.execute_command("rm /cygdrive/c/tmp/{0}".format(build_name))
1460                self.log_command_output(output, error)
1461
1462                """ the code below need to remove when bug MB-11328 is fixed in 3.0.1 """
1463                output, error = self.kill_erlang(os="windows")
1464                self.log_command_output(output, error)
1465                """ end remove code """
1466
1467
1468                """ the code below need to remove when bug MB-11985 is fixed in 3.0.1 """
1469                if full_version[:5] in COUCHBASE_VERSION_2 or \
1470                   full_version[:5] in COUCHBASE_VERSION_3:
1471                    log.info("due to bug MB-11985, we need to delete below registry")
1472                    output, error = self.execute_command("reg delete \
1473                               'HKLM\Software\Wow6432Node\Ericsson\Erlang\ErlSrv' /f ")
1474                    self.log_command_output(output, error)
1475                """ end remove code """
1476            else:
1477                log.info("No couchbase server on {0} server. Free to install".format(self.ip))
1478        elif type in ["ubuntu", "centos", "red hat"]:
1479            # uninstallation command is different
1480            if type == "ubuntu":
1481                uninstall_cmd = "dpkg -r {0};dpkg --purge {1};".format("couchbase-server", "couchbase-server")
1482                output, error = self.execute_command(uninstall_cmd)
1483                self.log_command_output(output, error)
1484            elif type in ["centos", "red hat"]:
1485                """ Sometimes, vm left with unfinish uninstall/install process.
1486                    We need to kill them before doing uninstall """
1487                output, error = self.execute_command("killall -9 rpm")
1488                self.log_command_output(output, error)
1489                uninstall_cmd = 'rpm -e {0}'.format("couchbase-server")
1490                log.info('running rpm -e to remove couchbase-server')
1491                output, error = self.execute_command(uninstall_cmd)
1492                self.log_command_output(output, error)
1493            self.terminate_processes(self.info, terminate_process_list)
1494            self.remove_folders(linux_folders)
1495        elif self.info.distribution_type.lower() == 'mac':
1496            self.stop_server(os='mac')
1497            self.terminate_processes(self.info, terminate_process_list)
1498            output, error = self.execute_command("rm -rf /Applications/Couchbase\ Server.app")
1499            self.log_command_output(output, error)
1500            output, error = self.execute_command("rm -rf ~/Library/Application\ Support/Couchbase")
1501            self.log_command_output(output, error)
1502
1503    def couchbase_win_uninstall(self, product, version, os_name, query):
1504        builds, changes = query.get_all_builds(version=version)
1505        version_file = 'VERSION.txt'
1506        bat_file = "uninstall.bat"
1507        task = "uninstall"
1508        deleted = False
1509
1510        self.extract_remote_info()
1511        ex_type = self.info.deliverable_type
1512        if self.info.architecture_type == "x86_64":
1513            os_type = "64"
1514        elif self.info.architecture_type == "x86":
1515            os_type = "32"
1516        if product == "cse":
1517            name = "couchbase-server-enterprise"
1518            version_path = "/cygdrive/c/Program Files/Couchbase/Server/"
1519
1520        exist = self.file_exists(version_path, version_file)
1521        if exist:
1522            # call uninstall function to install couchbase server
1523            # Need to detect csse or cse when uninstall.
1524            log.info("Start uninstall cb server on this server")
1525            build_name, rm_version = self.find_build_version(version_path, version_file)
1526            log.info('build needed to do auto uninstall {0}'.format(build_name))
1527            # find installed build in tmp directory to match with currently installed version
1528            build_name = build_name.rstrip() + ".exe"
1529            log.info('Check if {0} is in tmp directory'.format(build_name))
1530            exist = self.file_exists("/cygdrive/c/tmp/", build_name)
1531            if not exist:  # if not exist in tmp dir, start to download that version build
1532                build = query.find_build(builds, name, ex_type, self.info.architecture_type, rm_version)
1533                downloaded = self.download_binary_in_win(build.url, rm_version)
1534                if downloaded:
1535                    log.info('Successful download {0}.exe'.format(rm_version))
1536                else:
1537                    log.error('Download {0}.exe failed'.format(rm_version))
1538            # copy required files to automation directory
1539            dir_paths = ['/cygdrive/c/automation', '/cygdrive/c/tmp']
1540            self.create_multiple_dir(dir_paths)
1541            self.copy_files_local_to_remote('resources/windows/automation', '/cygdrive/c/automation')
1542            # modify bat file to run uninstall schedule task
1543            self.modify_bat_file('/cygdrive/c/automation', bat_file, product, rm_version, task)
1544
1545            """ the code below need to remove when bug MB-11328 is fixed in 3.0.1 """
1546            output, error = self.kill_erlang(os="windows")
1547            self.log_command_output(output, error)
1548            """ end remove code """
1549
1550            self.sleep(5, "before running task schedule uninstall")
1551            # run schedule task uninstall couchbase server
1552            output, error = self.execute_command("cmd /c schtasks /run /tn removeme")
1553            self.log_command_output(output, error)
1554            deleted = self.wait_till_file_deleted(version_path, version_file, timeout_in_seconds=600)
1555            if not deleted:
1556                log.error("Uninstall was failed at node {0}".format(self.ip))
1557                sys.exit()
1558            self.wait_till_process_ended(build_name[:10])
1559            self.sleep(10, "next step is to install")
1560            output, error = self.execute_command("cmd /c schtasks /Query /FO LIST /TN removeme /V")
1561            self.log_command_output(output, error)
1562            output, error = self.execute_command("rm /cygdrive/c/tmp/{0}".format(build_name))
1563            self.log_command_output(output, error)
1564
1565            """ the code below need to remove when bug MB-11328 is fixed in 3.0.1 """
1566            output, error = self.kill_erlang(os="windows")
1567            self.log_command_output(output, error)
1568            """ end remove code """
1569        else:
1570            log.info('No couchbase server on this server')
1571
1572    def membase_uninstall(self, save_upgrade_config=False):
1573        linux_folders = ["/var/opt/membase", "/opt/membase", "/etc/opt/membase",
1574                         "/var/membase/data/*", "/opt/membase/var/lib/membase/*"]
1575        terminate_process_list = ["beam", "memcached", "moxi", "vbucketmigrator",
1576                                  "couchdb", "epmd"]
1577        version_file = "VERSION.txt"
1578        self.extract_remote_info()
1579        log.info(self.info.distribution_type)
1580        type = self.info.distribution_type.lower()
1581        if type == 'windows':
1582            product = "mb"
1583            query = BuildQuery()
1584            builds, changes = query.get_all_builds()
1585            os_type = "exe"
1586            task = "uninstall"
1587            bat_file = "uninstall.bat"
1588            deleted = False
1589            product_name = "membase-server-enterprise"
1590            version_path = "/cygdrive/c/Program Files/Membase/Server/"
1591
1592            exist = self.file_exists(version_path, version_file)
1593            log.info("Is VERSION file existed? {0}".format(exist))
1594            if exist:
1595                log.info("VERSION file exists.  Start to uninstall")
1596                build_name, short_version, full_version = self.find_build_version(version_path, version_file, product)
1597                if "1.8.0" in full_version or "1.8.1" in full_version:
1598                    product_name = "couchbase-server-enterprise"
1599                    product = "cb"
1600                log.info('Build name: {0}'.format(build_name))
1601                build_name = build_name.rstrip() + ".exe"
1602                log.info('Check if {0} is in tmp directory'.format(build_name))
1603                exist = self.file_exists("/cygdrive/c/tmp/", build_name)
1604                if not exist:  # if not exist in tmp dir, start to download that version build
1605                    build = query.find_build(builds, product_name, os_type, self.info.architecture_type, full_version)
1606                    downloaded = self.download_binary_in_win(build.url, short_version)
1607                    if downloaded:
1608                        log.info('Successful download {0}_{1}.exe'.format(product, short_version))
1609                    else:
1610                        log.error('Download {0}_{1}.exe failed'.format(product, short_version))
1611                dir_paths = ['/cygdrive/c/automation', '/cygdrive/c/tmp']
1612                self.create_multiple_dir(dir_paths)
1613                self.copy_files_local_to_remote('resources/windows/automation', '/cygdrive/c/automation')
1614                # modify bat file to run uninstall schedule task
1615                self.create_windows_capture_file(task, product, full_version)
1616                self.modify_bat_file('/cygdrive/c/automation', bat_file, product, short_version, task)
1617                self.stop_schedule_tasks()
1618                self.sleep(5, "before running task schedule uninstall")
1619                # run schedule task uninstall Couchbase server
1620                output, error = self.execute_command("cmd /c schtasks /run /tn removeme")
1621                self.log_command_output(output, error)
1622                deleted = self.wait_till_file_deleted(version_path, version_file, timeout_in_seconds=600)
1623                if not deleted:
1624                    log.error("Uninstall was failed at node {0}".format(self.ip))
1625                    sys.exit()
1626                self.wait_till_process_ended(full_version[:10])
1627                self.sleep(10, "next step is to install")
1628                output, error = self.execute_command("cmd /c schtasks /Query /FO LIST /TN removeme /V")
1629                self.log_command_output(output, error)
1630            else:
1631                log.info("No membase server on this server.  Free to install")
1632        elif type in ["ubuntu", "centos", "red hat"]:
1633            # uninstallation command is different
1634            if type == "ubuntu":
1635                uninstall_cmd = 'dpkg -r {0};dpkg --purge {1};'.format('membase-server', 'membase-server')
1636                output, error = self.execute_command(uninstall_cmd)
1637                self.log_command_output(output, error)
1638            elif type in ["centos", "red hat"]:
1639                """ Sometimes, vm left with unfinish uninstall/install process.
1640                    We need to kill them before doing uninstall """
1641                output, error = self.execute_command("killall -9 rpm")
1642                self.log_command_output(output, error)
1643                uninstall_cmd = 'rpm -e {0}'.format('membase-server')
1644                log.info('running rpm -e to remove membase-server')
1645                output, error = self.execute_command(uninstall_cmd)
1646                self.log_command_output(output, error)
1647            self.terminate_processes(self.info, terminate_process_list)
1648            if not save_upgrade_config:
1649                self.remove_folders(linux_folders)
1650
1651    def moxi_uninstall(self):
1652        terminate_process_list = ["moxi"]
1653        self.extract_remote_info()
1654        log.info(self.info.distribution_type)
1655        type = self.info.distribution_type.lower()
1656        if type == 'windows':
1657            self.log.error("Not implemented")
1658        elif type == "ubuntu":
1659            uninstall_cmd = "dpkg -r {0};dpkg --purge {1};".format("moxi-server", "moxi-server")
1660            output, error = self.execute_command(uninstall_cmd)
1661            self.log_command_output(output, error)
1662        elif type in ["centos", "red hat"]:
1663            uninstall_cmd = 'rpm -e {0}'.format("moxi-server")
1664            log.info('running rpm -e to remove couchbase-server')
1665            output, error = self.execute_command(uninstall_cmd)
1666            self.log_command_output(output, error)
1667        self.terminate_processes(self.info, terminate_process_list)
1668
1669    def log_command_output(self, output, error, track_words=()):
1670        # success means that there are no track_words in the output
1671        # and there are no errors at all, if track_words is not empty
1672        # if track_words=(), the result is not important, and we return True
1673        success = True
1674        for line in error:
1675            log.error(line)
1676            if track_words:
1677                if "Warning" in line and "hugepages" in line:
1678                    log.info("There is a warning about transparent_hugepage may be in used when install cb server.\
1679                              So we will disable transparent_hugepage in this vm")
1680                    output, error = self.execute_command("echo never > /sys/kernel/mm/transparent_hugepage/enabled")
1681                    self.log_command_output(output, error)
1682                    success = True
1683                elif "dirname" in line:
1684                    log.warning("Ignore dirname error message during couchbase startup/stop/restart for CentOS 6.6 (MB-12536)")
1685                    success = True
1686                else:
1687                    success = False
1688        for line in output:
1689            log.info(line)
1690            if any(s.lower() in line.lower() for s in track_words):
1691                if "Warning" in line and "hugepages" in line:
1692                    log.info("There is a warning about transparent_hugepage may be in used when install cb server.\
1693                              So we will disable transparent_hugepage in this vm")
1694                    output, error = self.execute_command("echo never > /sys/kernel/mm/transparent_hugepage/enabled")
1695                    self.log_command_output(output, error)
1696                    success = True
1697                else:
1698                    success = False
1699                    log.error('something wrong happened on {0}!!! output:{1}, error:{2}, track_words:{3}'
1700                              .format(self.ip, output, error, track_words))
1701        return success
1702
1703    def execute_commands_inside(self, main_command, subcommands=[], min_output_size=0,
1704                                end_msg='', timeout=250):
1705        log.info("running command on {0}: {1}".format(self.ip, main_command))
1706
1707        stdin, stdout, stderro = self._ssh_client.exec_command(main_command)
1708        time.sleep(1)
1709        for cmd in subcommands:
1710              log.info("running command {0} inside {1} ({2})".format(
1711                                                        main_command, cmd, self.ip))
1712              stdin.channel.send("{0}\n".format(cmd))
1713              end_time = time.time() + float(timeout)
1714              while True:
1715                  if time.time() > end_time:
1716                      raise Exception("no output in {3} sec running command \
1717                                       {0} inside {1} ({2})".format(main_command,
1718                                                                    cmd, self.ip,
1719                                                                    timeout))
1720                  output = stdout.channel.recv(1024)
1721                  if output.strip().endswith(end_msg) and len(output) >= min_output_size:
1722                          break
1723                  time.sleep(2)
1724              log.info("{0}:'{1}' -> '{2}' output\n: {3}".format(self.ip, main_command, cmd, output))
1725        stdin.close()
1726        stdout.close()
1727        stderro.close()
1728        return output
1729
1730    def execute_command(self, command, info=None, debug=True, use_channel=False):
1731        if getattr(self, "info", None) is None and info is not None :
1732            self.info = info
1733        else:
1734            self.extract_remote_info()
1735
1736        if self.info.type.lower() == 'windows':
1737            self.use_sudo = False
1738
1739        if self.use_sudo:
1740            command = "sudo " + command
1741
1742        return self.execute_command_raw(command, debug=debug, use_channel=use_channel)
1743
1744    def execute_command_raw(self, command, debug=True, use_channel=False):
1745        if debug:
1746            log.info("running command.raw on {0}: {1}".format(self.ip, command))
1747        output = []
1748        error = []
1749        temp = ''
1750        if self.use_sudo or use_channel:
1751            channel = self._ssh_client.get_transport().open_session()
1752            channel.get_pty()
1753            channel.settimeout(900)
1754            stdin = channel.makefile('wb')
1755            stdout = channel.makefile('rb')
1756            stderro = channel.makefile_stderr('rb')
1757            channel.exec_command(command)
1758            data = channel.recv(1024)
1759            while data:
1760                temp += data
1761                data = channel.recv(1024)
1762            channel.close()
1763        else:
1764            stdin, stdout, stderro = self._ssh_client.exec_command(command)
1765        stdin.close()
1766
1767        for line in stdout.read().splitlines():
1768            output.append(line)
1769        for line in stderro.read().splitlines():
1770            error.append(line)
1771        if temp:
1772            line = temp.splitlines()
1773            output.extend(line)
1774        if debug:
1775            log.info('command executed successfully')
1776        stdout.close()
1777        stderro.close()
1778        return output, error
1779
1780    def execute_non_sudo_command(self, command, info=None, debug=True, use_channel=False):
1781        info = info or self.extract_remote_info()
1782        self.info = info
1783
1784        return self.execute_command_raw(command, debug=debug, use_channel=use_channel)
1785
1786    def terminate_process(self, info=None, process_name=''):
1787        self.extract_remote_info()
1788        if self.info.type.lower() == 'windows':
1789            o, r = self.execute_command("taskkill /F /T /IM {0}*".format(process_name))
1790            self.log_command_output(o, r)
1791        else:
1792            o, r = self.execute_command("killall -9 {0}".format(process_name))
1793            self.log_command_output(o, r)
1794
1795    def disconnect(self):
1796        self._ssh_client.close()
1797
1798    def extract_remote_info(self):
1799        # use ssh to extract remote machine info
1800        # use sftp to if certain types exists or not
1801        if getattr(self, "info", None) is not None and isinstance(self.info, RemoteMachineInfo):
1802            return self.info
1803        mac_check_cmd = "sw_vers | grep ProductVersion | awk '{ print $2 }'"
1804        stdin, stdout, stderro = self._ssh_client.exec_command(mac_check_cmd)
1805        stdin.close()
1806        ver, err = stdout.read(), stderro.read()
1807        if not err:
1808            os_distro = "Mac"
1809            os_version = ver
1810            is_linux_distro = True
1811            is_mac = True
1812            self.use_sudo = False
1813        else:
1814            is_mac = False
1815            sftp = self._ssh_client.open_sftp()
1816            filenames = sftp.listdir('/etc/')
1817            os_distro = ""
1818            os_version = ""
1819            is_linux_distro = False
1820            for name in filenames:
1821                if name == 'issue':
1822                    # it's a linux_distro . let's downlaod this file
1823                    # format Ubuntu 10.04 LTS \n \l
1824                    filename = 'etc-issue-{0}'.format(uuid.uuid4())
1825                    sftp.get(localpath=filename, remotepath='/etc/issue')
1826                    file = open(filename)
1827                    etc_issue = ''
1828                    # let's only read the first line
1829                    for line in file.xreadlines():
1830                        etc_issue = line
1831                        break
1832                        # strip all extra characters
1833                    etc_issue = etc_issue.rstrip('\n').rstrip('\\l').rstrip('\\n')
1834                    if etc_issue.lower().find('ubuntu') != -1:
1835                        os_distro = 'Ubuntu'
1836                        os_version = etc_issue
1837                        is_linux_distro = True
1838                    elif etc_issue.lower().find('debian') != -1:
1839                        os_distro = 'Ubuntu'
1840                        os_version = etc_issue
1841                        is_linux_distro = True
1842                    elif etc_issue.lower().find('mint') != -1:
1843                        os_distro = 'Ubuntu'
1844                        os_version = etc_issue
1845                        is_linux_distro = True
1846                    elif etc_issue.lower().find('amazon linux ami') != -1:
1847                        os_distro = 'CentOS'
1848                        os_version = etc_issue
1849                        is_linux_distro = True
1850                    elif etc_issue.lower().find('centos') != -1:
1851                        os_distro = 'CentOS'
1852                        os_version = etc_issue
1853                        is_linux_distro = True
1854                    elif etc_issue.lower().find('red hat') != -1:
1855                        os_distro = 'Red Hat'
1856                        os_version = etc_issue
1857                        is_linux_distro = True
1858                    file.close()
1859                    # now remove this file
1860                    os.remove(filename)
1861                    break
1862        if not is_linux_distro:
1863            arch = ''
1864            os_version = 'unknown windows'
1865            win_info = self.find_windows_info()
1866            info = RemoteMachineInfo()
1867            info.type = win_info['os']
1868            info.windows_name = win_info['os_name']
1869            info.distribution_type = win_info['os']
1870            info.architecture_type = win_info['os_arch']
1871            info.ip = self.ip
1872            info.distribution_version = win_info['os']
1873            info.deliverable_type = 'exe'
1874            info.cpu = self.get_cpu_info(win_info)
1875            info.disk = self.get_disk_info(win_info)
1876            info.ram = self.get_ram_info(win_info)
1877            info.hostname = self.get_hostname(win_info)
1878            info.domain = self.get_domain(win_info)
1879            self.info = info
1880            return info
1881        else:
1882            # now run uname -m to get the architechtre type
1883            stdin, stdout, stderro = self._ssh_client.exec_command('uname -m')
1884            stdin.close()
1885            os_arch = ''
1886            text = stdout.read().splitlines()
1887            for line in text:
1888                os_arch += line
1889                # at this point we should know if its a linux or windows ditro
1890            ext = {'Ubuntu': 'deb', 'CentOS': 'rpm',
1891                   'Red Hat': 'rpm', "Mac": "zip",
1892                   "Debian": "deb"}.get(os_distro, '')
1893            arch = {'i686': 'x86', 'i386': 'x86'}.get(os_arch, os_arch)
1894            info = RemoteMachineInfo()
1895            info.type = "Linux"
1896            info.distribution_type = os_distro
1897            info.architecture_type = arch
1898            info.ip = self.ip
1899            info.distribution_version = os_version
1900            info.deliverable_type = ext
1901            info.cpu = self.get_cpu_info(mac=is_mac)
1902            info.disk = self.get_disk_info(mac=is_mac)
1903            info.ram = self.get_ram_info(mac=is_mac)
1904            info.hostname = self.get_hostname()
1905            info.domain = self.get_domain()
1906            self.info = info
1907            return info
1908
1909    def get_extended_windows_info(self):
1910        info = {}
1911        win_info = self.extend_windows_info()
1912        info['ram'] = self.get_ram_info(win_info)
1913        info['disk'] = self.get_disk_info(win_info)
1914        info['cpu'] = self.get_cpu_info(win_info)
1915        info['hostname'] = self.get_hostname()
1916        return info
1917
1918    def get_hostname(self, win_info=None):
1919        if win_info:
1920            if 'Host Name' not in win_info:
1921                win_info = self.create_windows_info()
1922            o = win_info['Host Name']
1923        o, r = self.execute_command_raw('hostname')
1924        if o:
1925            return o
1926
1927    def get_domain(self, win_info=None):
1928        if win_info:
1929            o = self.execute_batch_command('ipconfig')
1930            suffix_dns_row = [row for row in o if row.find(" Connection-specific DNS Suffix") != -1]
1931            ret = suffix_dns_row[0].split(':')[1].strip()
1932        else:
1933            ret = self.execute_command_raw('hostname -d')
1934        return ret
1935
1936    def get_full_hostname(self):
1937        info = self.extract_remote_info()
1938        if not info.domain:
1939            return None
1940        return '%s.%s' % (info.hostname[0], info.domain[0][0])
1941
1942    def get_cpu_info(self, win_info=None, mac=False):
1943        if win_info:
1944            if 'Processor(s)' not in win_info:
1945                win_info = self.create_windows_info()
1946            o = win_info['Processor(s)']
1947        elif mac:
1948            o, r = self.execute_command_raw('/sbin/sysctl -n machdep.cpu.brand_string')
1949        else:
1950            o, r = self.execute_command_raw('sudo cat /proc/cpuinfo')
1951        if o:
1952            return o
1953
1954    def get_ram_info(self, win_info=None, mac=False):
1955        if win_info:
1956            if 'Virtual Memory Max Size' not in win_info:
1957                win_info = self.create_windows_info()
1958            o = "Virtual Memory Max Size =" + win_info['Virtual Memory Max Size'] + '\n'
1959            o += "Virtual Memory Available =" + win_info['Virtual Memory Available'] + '\n'
1960            o += "Virtual Memory In Use =" + win_info['Virtual Memory In Use']
1961        elif mac:
1962            o, r = self.execute_command_raw('/sbin/sysctl -n hw.memsize')
1963        else:
1964            o, r = self.execute_command_raw('sudo cat /proc/meminfo')
1965        if o:
1966            return o
1967
1968    def get_disk_info(self, win_info=None, mac=False):
1969        if win_info:
1970            if 'Total Physical Memory' not in win_info:
1971                win_info = self.create_windows_info()
1972            o = "Total Physical Memory =" + win_info['Total Physical Memory'] + '\n'
1973            o += "Available Physical Memory =" + win_info['Available Physical Memory']
1974        elif mac:
1975            o, r = self.execute_command_raw('df -h')
1976        else:
1977            o, r = self.execute_command_raw('df -Th')
1978        if o:
1979            return o
1980
1981    def stop_couchbase(self):
1982        self.extract_remote_info()
1983        if self.info.type.lower() == 'windows':
1984            o, r = self.execute_command("net stop couchbaseserver")
1985            self.log_command_output(o, r)
1986            self.sleep(10, "Wait to stop service completely")
1987        if self.info.type.lower() == "linux":
1988            o, r = self.execute_command("/etc/init.d/couchbase-server stop", self.info, use_channel=True)
1989            self.log_command_output(o, r)
1990        if self.info.distribution_type.lower() == "mac":
1991            o, r = self.execute_command("ps aux | grep Couchbase | awk '{print $2}' | xargs kill -9")
1992            self.log_command_output(o, r)
1993            o, r = self.execute_command("killall -9 epmd")
1994            self.log_command_output(o, r)
1995
1996    def start_couchbase(self):
1997        self.extract_remote_info()
1998        if self.info.type.lower() == 'windows':
1999            o, r = self.execute_command("net start couchbaseserver")
2000            self.log_command_output(o, r)
2001        if self.info.type.lower() == "linux":
2002            o, r = self.execute_command("/etc/init.d/couchbase-server start")
2003            self.log_command_output(o, r)
2004        if self.info.distribution_type.lower() == "mac":
2005            o, r = self.execute_command("open /Applications/Couchbase\ Server.app")
2006            self.log_command_output(o, r)
2007
2008    def pause_memcached(self):
2009        o, r = self.execute_command("killall -SIGSTOP memcached")
2010        self.log_command_output(o, r)
2011
2012    def unpause_memcached(self):
2013        o, r = self.execute_command("killall -SIGCONT memcached")
2014        self.log_command_output(o, r)
2015
2016    def pause_beam(self):
2017        o, r = self.execute_command("killall -SIGSTOP beam")
2018        self.log_command_output(o, r)
2019
2020    def unpause_beam(self):
2021        o, r = self.execute_command("killall -SIGCONT beam")
2022        self.log_command_output(o, r)
2023
2024
2025    # TODO: Windows
2026    def flush_os_caches(self):
2027        self.extract_remote_info()
2028        if self.info.type.lower() == "linux":
2029            o, r = self.execute_command("sync")
2030            self.log_command_output(o, r)
2031            o, r = self.execute_command("/sbin/sysctl vm.drop_caches=3")
2032            self.log_command_output(o, r)
2033
2034    def get_data_file_size(self, path=None):
2035
2036        output, error = self.execute_command('du -b {0}'.format(path))
2037        if error:
2038            return 0
2039        else:
2040            for line in output:
2041                size = line.strip().split('\t')
2042                if size[0].isdigit():
2043                    print size[0]
2044                    return size[0]
2045                else:
2046                    return 0
2047
2048    def get_process_statistics(self, process_name=None, process_pid=None):
2049        '''
2050        Gets process statistics for windows nodes
2051        WMI is required to be intalled on the node
2052        stats_windows_helper should be located on the node
2053        '''
2054        self.extract_remote_info()
2055        remote_command = "cd ~; /cygdrive/c/Python27/python stats_windows_helper.py"
2056        if process_name:
2057            remote_command.append(" " + process_name)
2058        elif process_pid:
2059            remote_command.append(" " + process_pid)
2060
2061        if self.info.type.lower() == "windows":
2062            o, r = self.execute_command(remote_command, info)
2063            if r:
2064                log.error("Command didn't run successfully. Error: {0}".format(r))
2065            return o;
2066        else:
2067            log.error("Function is implemented only for Windows OS")
2068            return None
2069
2070    def get_process_statistics_parameter(self, parameter, process_name=None, process_pid=None):
2071       if not parameter:
2072           log.error("parameter cannot be None")
2073
2074       parameters_list = self.get_process_statistics(process_name, process_pid)
2075
2076       if not parameters_list:
2077           log.error("no statistics found")
2078           return None
2079       parameters_dic = dict(item.split(' = ') for item in parameters_list)
2080
2081       if parameter in parameters_dic:
2082           return parameters_dic[parameter]
2083       else:
2084           log.error("parameter '{0}' is not found".format(parameter))
2085           return None
2086
2087    def set_environment_variable(self, name, value):
2088        """Request an interactive shell session, export custom variable and
2089        restart Couchbase server.
2090
2091        Shell session is necessary because basic SSH client is stateless.
2092        """
2093
2094        shell = self._ssh_client.invoke_shell()
2095        self.extract_remote_info()
2096        if self.info.type.lower() == "windows":
2097            shell.send('net stop CouchbaseServer\n')
2098            shell.send('set {0}={1}\n'.format(name, value))
2099            shell.send('net start CouchbaseServer\n')
2100        elif self.info.type.lower() == "linux":
2101            shell.send('export {0}={1}\n'.format(name, value))
2102            shell.send('/etc/init.d/couchbase-server restart\n')
2103        shell.close()
2104
2105    def change_env_variables(self, dict):
2106        prefix = "\\n    "
2107        shell = self._ssh_client.invoke_shell()
2108        init_file = "couchbase_init.d"
2109        file_path = "/opt/couchbase/etc/"
2110        environmentVariables = ""
2111        self.extract_remote_info()
2112        if self.info.type.lower() == "windows":
2113            init_file = "service_start.bat"
2114            file_path = "/cygdrive/c/Program Files/Couchbase/Server/bin/"
2115            prefix = "\\n"
2116        backupfile = file_path + init_file + ".bak"
2117        sourceFile = file_path + init_file
2118        o, r = self.execute_command("cp " + sourceFile + " " + backupfile)
2119        self.log_command_output(o, r)
2120        command = "sed -i 's/{0}/{0}".format("ulimit -l unlimited")
2121        for key in dict.keys():
2122            o, r = self.execute_command("sed -i 's/{1}.*//' {0}".format(sourceFile, key))
2123            self.log_command_output(o, r)
2124
2125        if self.info.type.lower() == "windows":
2126            command = "sed -i 's/{0}/{0}".format("set NS_ERTS=%NS_ROOT%\erts-5.8.5.cb1\bin")
2127
2128        for key in dict.keys():
2129            if self.info.type.lower() == "windows":
2130                environmentVariables += prefix + 'set {0}={1}'.format(key, dict[key])
2131            else:
2132                environmentVariables += prefix + 'export {0}={1}'.format(key, dict[key])
2133
2134        command += environmentVariables + "/'" + " " + sourceFile
2135        o, r = self.execute_command(command)
2136        self.log_command_output(o, r)
2137
2138        if self.info.type.lower() == "linux":
2139            o, r = self.execute_command("/etc/init.d/couchbase-server restart")
2140            self.log_command_output(o, r)
2141        else:
2142            o, r = self.execute_command("net stop couchbaseserver")
2143            self.log_command_output(o, r)
2144            o, r = self.execute_command("net start couchbaseserver")
2145            self.log_command_output(o, r)
2146        shell.close()
2147
2148    def reset_env_variables(self):
2149        shell = self._ssh_client.invoke_shell()
2150        if getattr(self, "info", None) is None:
2151            self.info = self.extract_remote_info()
2152        init_file = "couchbase_init.d"
2153        file_path = "/opt/couchbase/etc/"
2154        if self.info.type.lower() == "windows":
2155            init_file = "service_start.bat"
2156            file_path = "/cygdrive/c/Program Files/Couchbase/Server/bin"
2157        backupfile = file_path + init_file + ".bak"
2158        sourceFile = file_path + init_file
2159        o, r = self.execute_command("mv " + backupfile + " " + sourceFile)
2160        self.log_command_output(o, r)
2161        if self.info.type.lower() == "linux":
2162            o, r = self.execute_command("/etc/init.d/couchbase-server restart")
2163            self.log_command_output(o, r)
2164        else:
2165            o, r = self.execute_command("net stop couchbaseserver")
2166            self.log_command_output(o, r)
2167            o, r = self.execute_command("net start couchbaseserver")
2168            self.log_command_output(o, r)
2169        shell.close()
2170
2171    def set_node_name(self, name):
2172        """Edit couchbase-server shell script in place and set custom node name.
2173        This is necessary for cloud installations where nodes have both
2174        private and public addresses.
2175
2176        It only works on Unix-like OS.
2177
2178        Reference: http://bit.ly/couchbase-bestpractice-cloud-ip
2179        """
2180
2181        # Stop server
2182        self.stop_couchbase()
2183
2184        # Edit _start function
2185        cmd = r"sed -i 's/\(.*\-run ns_bootstrap.*\)/\1\n\t-name ns_1@{0} \\/' \
2186                /opt/couchbase/bin/couchbase-server".format(name)
2187        self.execute_command(cmd)
2188
2189        # Cleanup
2190        for cmd in ('rm -fr /opt/couchbase/var/lib/couchbase/data/*',
2191                    'rm -fr /opt/couchbase/var/lib/couchbase/mnesia/*',
2192                    'rm -f /opt/couchbase/var/lib/couchbase/config/config.dat'):
2193            self.execute_command(cmd)
2194
2195        # Start server
2196        self.start_couchbase()
2197
2198    def execute_cluster_backup(self, login_info="Administrator:password", backup_location="/tmp/backup",
2199                               command_options='', cluster_ip="", cluster_port="8091", delete_backup=True):
2200        backup_command = "%scbbackup" % (testconstants.LINUX_COUCHBASE_BIN_PATH)
2201        backup_file_location = backup_location
2202        # TODO: define WIN_COUCHBASE_BIN_PATH and implement a new function under RestConnectionHelper to use nodes/self info to get os info
2203        self.extract_remote_info()
2204        if self.info.type.lower() == 'windows':
2205            backup_command = "%scbbackup.exe" % (testconstants.WIN_COUCHBASE_BIN_PATH_RAW)
2206            backup_file_location = "C:%s" % (backup_location)
2207            output, error = self.execute_command("taskkill /F /T /IM cbbackup.exe")
2208            self.log_command_output(output, error)
2209        if self.info.distribution_type.lower() == 'mac':
2210            backup_command = "%scbbackup" % (testconstants.MAC_COUCHBASE_BIN_PATH)
2211
2212        command_options_string = ""
2213        if command_options is not '':
2214            command_options_string = ' '.join(command_options)
2215        cluster_ip = cluster_ip or self.ip
2216        cluster_port = cluster_port or self.port
2217
2218        if '-m accu' not in command_options_string and '-m diff' not in command_options_string and delete_backup:
2219            self.delete_files(backup_file_location)
2220            self.create_directory(backup_file_location)
2221
2222
2223        command = "%s %s%s@%s:%s %s %s" % (backup_command, "http://", login_info,
2224                                           cluster_ip, cluster_port, backup_file_location, command_options_string)
2225        if self.info.type.lower() == 'windows':
2226            command = "cmd /c START \"\" \"%s\" \"%s%s@%s:%s\" \"%s\" %s" % (backup_command, "http://", login_info,
2227                                               cluster_ip, cluster_port, backup_file_location, command_options_string)
2228
2229        output, error = self.execute_command(command)
2230        self.log_command_output(output, error)
2231
2232    def restore_backupFile(self, login_info, backup_location, buckets):
2233        restore_command = "%scbrestore" % (testconstants.LINUX_COUCHBASE_BIN_PATH)
2234        backup_file_location = backup_location
2235        self.extract_remote_info()
2236        if self.info.type.lower() == 'windows':
2237            restore_command = "%scbrestore.exe" % (testconstants.WIN_COUCHBASE_BIN_PATH_RAW)
2238            backup_file_location = "C:%s" % (backup_location)
2239        if self.info.distribution_type.lower() == 'mac':
2240            restore_command = "%scbrestore" % (testconstants.MAC_COUCHBASE_BIN_PATH)
2241
2242        for bucket in buckets:
2243            command = "%s %s %s%s@%s:%s %s %s" % (restore_command, backup_file_location, "http://",
2244                                                  login_info, self.ip, self.port, "-b", bucket)
2245            if self.info.type.lower() == 'windows':
2246                command = "cmd /c \"%s\" \"%s\" \"%s%s@%s:%s\" %s %s" % (restore_command, backup_file_location, "http://",
2247                                                      login_info, self.ip, self.port, "-b", bucket)
2248            output, error = self.execute_command(command)
2249            self.log_command_output(output, error)
2250
2251    def delete_files(self, file_location):
2252        command = "%s%s" % ("rm -rf ", file_location)
2253        output, error = self.execute_command(command)
2254        self.log_command_output(output, error)
2255
2256    def get_data_map_using_cbtransfer(self, buckets, data_path=None, userId="Administrator", password="password", getReplica=False, mode="memory"):
2257        self.extract_remote_info()
2258        temp_path = "/tmp/"
2259        if self.info.type.lower() == 'windows':
2260            return "",{}
2261            temp_path = testconstants.WIN_TMP_PATH
2262        replicaOption = ""
2263        prefix = str(uuid.uuid1())
2264        fileName = prefix + ".csv"
2265        if getReplica:
2266             replicaOption = "  --source-vbucket-state=replica"
2267
2268        source = "http://" + self.ip + ":8091"
2269        if mode == "disk":
2270            source = "couchstore-files://" + data_path
2271        elif mode == "backup":
2272            source = data_path
2273            fileName = ""
2274        # Initialize Output
2275        bucketMap = {}
2276        headerInfo = ""
2277        # Iterate per bucket and generate maps
2278        for bucket in buckets:
2279            if data_path == None:
2280                options = " -b " + bucket.name + " -u " + userId + " -p password --single-node"
2281            else:
2282                options = " -b " + bucket.name + " -u " + userId + " -p password" + replicaOption
2283            suffix = "_" + bucket.name + "_N%2FA.csv"
2284            if mode == "memory" or mode == "backup":
2285               suffix = "_" + bucket.name + "_" + self.ip + "%3A8091.csv"
2286
2287            genFileName = prefix + suffix
2288            csv_path = temp_path + fileName
2289            path = temp_path + genFileName
2290            dest_path = "/tmp/" + fileName
2291            destination = "csv:" + csv_path
2292            self.execute_cbtransfer(source, destination, options)
2293            file_existed = self.file_exists(temp_path, genFileName)
2294            if file_existed:
2295                self.copy_file_remote_to_local(path, dest_path)
2296                self.delete_files(path)
2297                content = []
2298                headerInfo = ""
2299                with open(dest_path) as f:
2300                    headerInfo = f.readline()
2301                    content = f.readlines()
2302                bucketMap[bucket.name] = content
2303                os.remove(dest_path)
2304        return headerInfo, bucketMap
2305
2306    def execute_cbtransfer(self, source, destination, command_options=''):
2307        transfer_command = "%scbtransfer" % (testconstants.LINUX_COUCHBASE_BIN_PATH)
2308        self.extract_remote_info()
2309        if self.info.type.lower() == 'windows':
2310            transfer_command = "%scbtransfer.exe" % (testconstants.WIN_COUCHBASE_BIN_PATH_RAW)
2311        if self.info.distribution_type.lower() == 'mac':
2312            transfer_command = "%scbtransfer" % (testconstants.MAC_COUCHBASE_BIN_PATH)
2313
2314        command = "%s %s %s %s" % (transfer_command, source, destination, command_options)
2315        if self.info.type.lower() == 'windows':
2316            command = "cmd /c \"%s\" \"%s\" \"%s\" %s" % (transfer_command, source, destination, command_options)
2317        output, error = self.execute_command(command, use_channel=True)
2318        self.log_command_output(output, error)
2319        return output
2320
2321    def execute_cbdocloader(self, username, password, bucket, memory_quota, file):
2322        cbdocloader_command = "%scbdocloader" % (testconstants.LINUX_COUCHBASE_BIN_PATH)
2323        self.extract_remote_info()
2324        command = "%s -u %s -p %s -n %s:%s -b %s -s %s %ssamples/%s.zip" % (cbdocloader_command,
2325                                                                            username, password, self.ip,
2326                                                                            self.port, bucket, memory_quota,
2327                                                                            testconstants.LINUX_CB_PATH, file)
2328        if self.info.distribution_type.lower() == 'mac':
2329            cbdocloader_command = "%scbdocloader" % (testconstants.MAC_COUCHBASE_BIN_PATH)
2330            command = "%s -u %s -p %s -n %s:%s -b %s -s %s %ssamples/%s.zip" % (cbdocloader_command,
2331                                                                            username, password, self.ip,
2332                                                                            self.port, bucket, memory_quota,
2333                                                                            testconstants.MAC_CB_PATH, file)
2334
2335        if self.info.type.lower() == 'windows':
2336            cbdocloader_command = "%scbdocloader.exe" % (testconstants.WIN_COUCHBASE_BIN_PATH)
2337            WIN_COUCHBASE_SAMPLES_PATH = "C:/Program\ Files/Couchbase/Server/samples/"
2338            command = "%s -u %s -p %s -n %s:%s -b %s -s %s %s%s.zip" % (cbdocloader_command,
2339                                                                        username, password, self.ip,
2340                                                                        self.port, bucket, memory_quota,
2341                                                                        WIN_COUCHBASE_SAMPLES_PATH, file)
2342        output, error = self.execute_command(command)
2343        self.log_command_output(output, error)
2344        return output, error
2345
2346    def execute_cbcollect_info(self, file):
2347        cbcollect_command = "%scbcollect_info" % (testconstants.LINUX_COUCHBASE_BIN_PATH)
2348        self.extract_remote_info()
2349        if self.info.type.lower() == 'windows':
2350            cbcollect_command = "%scbcollect_info.exe" % (testconstants.WIN_COUCHBASE_BIN_PATH)
2351        if self.info.distribution_type.lower() == 'mac':
2352            cbcollect_command = "%scbcollect_info" % (testconstants.MAC_COUCHBASE_BIN_PATH)
2353
2354        command = "%s %s" % (cbcollect_command, file)
2355        output, error = self.execute_command(command, use_channel=True)
2356        self.log_command_output(output, error)
2357        return output, error
2358
2359    def execute_cbepctl(self, bucket, persistence, param_type, param, value):
2360        cbepctl_command = "%scbepctl" % (testconstants.LINUX_COUCHBASE_BIN_PATH)
2361        self.extract_remote_info()
2362        if self.info.type.lower() == 'windows':
2363            cbepctl_command = "%scbepctl.exe" % (testconstants.WIN_COUCHBASE_BIN_PATH)
2364        if self.info.distribution_type.lower() == 'mac':
2365            cbepctl_command = "%scbepctl" % (testconstants.MAC_COUCHBASE_BIN_PATH)
2366
2367        if bucket.saslPassword == None:
2368            bucket.saslPassword = ''
2369        if persistence != "":
2370            command = "%s %s:11210 -b %s -p \"%s\" %s" % (cbepctl_command, self.ip,
2371                                                          bucket.name, bucket.saslPassword,
2372                                                          persistence)
2373        else:
2374            command = "%s %s:11210 -b %s -p \"%s\" %s %s %s" % (cbepctl_command, self.ip,
2375                                                                bucket.name, bucket.saslPassword,
2376                                                                param_type, param, value)
2377        output, error = self.execute_command(command)
2378        self.log_command_output(output, error)
2379        return output, error
2380
2381    def execute_cbstats(self, bucket, command, keyname="", vbid=0):
2382        cbstat_command = "%scbstats" % (testconstants.LINUX_COUCHBASE_BIN_PATH)
2383        self.extract_remote_info()
2384        if self.info.type.lower() == 'windows':
2385            cbstat_command = "%scbstats.exe" % (testconstants.WIN_COUCHBASE_BIN_PATH)
2386        if self.info.distribution_type.lower() == 'mac':
2387            cbstat_command = "%scbstats" % (testconstants.MAC_COUCHBASE_BIN_PATH)
2388
2389        if command != "key" and command != "raw":
2390            if bucket.saslPassword == None:
2391                bucket.saslPassword = ''
2392            command = "%s %s:11210 %s -b %s -p \"%s\" " % (cbstat_command, self.ip, command,
2393                                                                bucket.name, bucket.saslPassword)
2394        else:
2395            command = "%s %s:11210 %s %s %s " % (cbstat_command, self.ip, command,
2396                                                                keyname, vbid)
2397        output, error = self.execute_command(command)
2398        self.log_command_output(output, error)
2399        return output, error
2400
2401    def execute_couchbase_cli(self, cli_command, cluster_host='localhost', options='', cluster_port=None, user='Administrator', password='password'):
2402        cb_client = "%scouchbase-cli" % (testconstants.LINUX_COUCHBASE_BIN_PATH)
2403        self.extract_remote_info()
2404        if self.info.type.lower() == 'windows':
2405            cb_client = "%scouchbase-cli.exe" % (testconstants.WIN_COUCHBASE_BIN_PATH)
2406        if self.info.distribution_type.lower() == 'mac':
2407            cb_client = "%scouchbase-cli" % (testconstants.MAC_COUCHBASE_BIN_PATH)
2408
2409        cluster_param = (" --cluster={0}".format(cluster_host), "")[cluster_host is None]
2410        if cluster_param is not None:
2411            cluster_param += (":{0}".format(cluster_port), "")[cluster_port is None]
2412
2413        user_param = (" -u {0}".format(user), "")[user is None]
2414        passwd_param = (" -p {0}".format(password), "")[password is None]
2415        # now we can run command in format where all parameters are optional
2416        # {PATH}/couchbase-cli [COMMAND] [CLUSTER:[PORT]] [USER] [PASWORD] [OPTIONS]
2417        command = cb_client + " " + cli_command + cluster_param + user_param + passwd_param + " " + options
2418
2419        output, error = self.execute_command(command, use_channel=True)
2420        self.log_command_output(output, error)
2421        return output, error
2422
2423    def execute_cbworkloadgen(self, username, password, num_items, ratio, bucket, item_size, command_options):
2424        cbworkloadgen_command = "%scbworkloadgen" % (testconstants.LINUX_COUCHBASE_BIN_PATH)
2425        self.extract_remote_info()
2426
2427        if self.info.distribution_type.lower() == 'mac':
2428            cbworkloadgen_command = "%scbworkloadgen" % (testconstants.MAC_COUCHBASE_BIN_PATH)
2429
2430        if self.info.type.lower() == 'windows':
2431            cbworkloadgen_command = "%scbworkloadgen.exe" % (testconstants.WIN_COUCHBASE_BIN_PATH)
2432
2433        command = "%s -n %s:%s -r %s -i %s -b %s -s %s %s -u %s -p %s" % (cbworkloadgen_command, self.ip, self.port,
2434                                                                          ratio, num_items, bucket, item_size,
2435                                                                          command_options, username, password)
2436
2437        output, error = self.execute_command(command)
2438        self.log_command_output(output, error)
2439        return output, error
2440
2441    def execute_cbhealthchecker(self, username, password, command_options=None, path_to_store=''):
2442        command_options_string = ""
2443        if command_options is not None:
2444            command_options_string = ' '.join(command_options)
2445
2446        cbhealthchecker_command = "%scbhealthchecker" % (testconstants.LINUX_COUCHBASE_BIN_PATH)
2447        if self.info.distribution_type.lower() == 'mac':
2448            cbhealthchecker_command = "%scbhealthchecker" % (testconstants.MAC_COUCHBASE_BIN_PATH)
2449
2450        if self.info.type.lower() == 'windows':
2451            cbhealthchecker_command = "%scbhealthchecker" % (testconstants.WIN_COUCHBASE_BIN_PATH)
2452
2453        if path_to_store:
2454            self.execute_command('rm -rf %s; mkdir %s;cd %s' % (path_to_store,
2455                                                path_to_store, path_to_store))
2456
2457        command = "%s -u %s -p %s -c %s:%s %s" % (cbhealthchecker_command,
2458                                                username, password, self.ip,
2459                                                self.port, command_options_string)
2460
2461        if path_to_store:
2462            command = "cd %s; %s -u %s -p %s -c %s:%s %s" % (path_to_store,
2463                                                cbhealthchecker_command,
2464                                                username, password, self.ip,
2465                                                self.port, command_options_string)
2466
2467        output, error = self.execute_command_raw(command)
2468        self.log_command_output(output, error)
2469        return output, error
2470
2471    def execute_vbuckettool(self, keys, prefix=None):
2472        command = "%stools/vbuckettool" % (testconstants.LINUX_COUCHBASE_BIN_PATH)
2473        self.extract_remote_info()
2474        if self.info.type.lower() == 'windows':
2475            command = "%stools/vbuckettool.exe" % (testconstants.WIN_COUCHBASE_BIN_PATH)
2476        if prefix:
2477            command = prefix + command
2478        command = "%s - %s" % (command, ' '.join(keys))
2479        output, error = self.execute_command(command, use_channel=True)
2480        self.log_command_output(output, error)
2481        return output, error
2482
2483    def execute_batch_command(self, command):
2484        remote_command = "echo \"{0}\" > /tmp/cmd.bat; /tmp/cmd.bat".format(command)
2485        o, r = self.execute_command_raw(remote_command)
2486        if r:
2487            log.error("Command didn't run successfully. Error: {0}".format(r))
2488        return o;
2489
2490    def remove_win_backup_dir(self):
2491        win_paths = [testconstants.WIN_CB_PATH, testconstants.WIN_MB_PATH]
2492        for each_path in win_paths:
2493            backup_files = []
2494            files = self.list_files(each_path)
2495            for f in files:
2496                if f["file"].startswith("backup-"):
2497                    backup_files.append(f["file"])
2498             # keep the last one
2499            if len(backup_files) > 2:
2500                log.info("start remove previous backup directory")
2501                for f in backup_files[:-1]:
2502                    self.execute_command("rm -rf '{0}{1}'".format(each_path, f))
2503
2504    def remove_win_collect_tmp(self):
2505        win_tmp_path = testconstants.WIN_TMP_PATH
2506        log.info("start remove tmp files from directory %s" % win_tmp_path)
2507        self.execute_command("rm -rf '%stmp*'" % win_tmp_path)
2508
2509    # ps_name_or_id means process name or ID will be suspended
2510    def windows_process_utils(self, ps_name_or_id, cmd_file_name, option=""):
2511