xref: /5.5.2/testrunner/lib/remote/remote_util.py (revision a990513c)
1import os
2import re
3import sys
4import copy
5import urllib
6import uuid
7import time
8import logging
9import stat
10import json
11import TestInput
12from subprocess import Popen, PIPE
13
14import logger
15from builds.build_query import BuildQuery
16import testconstants
17from testconstants import VERSION_FILE
18from testconstants import WIN_REGISTER_ID
19from testconstants import MEMBASE_VERSIONS
20from testconstants import COUCHBASE_VERSIONS
21from testconstants import MISSING_UBUNTU_LIB
22from testconstants import MV_LATESTBUILD_REPO
23from testconstants import SHERLOCK_BUILD_REPO
24from testconstants import COUCHBASE_VERSIONS
25from testconstants import WIN_CB_VERSION_3
26from testconstants import COUCHBASE_VERSION_2
27from testconstants import COUCHBASE_VERSION_3
28from testconstants import COUCHBASE_FROM_VERSION_3,\
29                          COUCHBASE_FROM_SPOCK, SYSTEMD_SERVER
30from testconstants import COUCHBASE_RELEASE_VERSIONS_3, CB_RELEASE_BUILDS
31from testconstants import SHERLOCK_VERSION, WIN_PROCESSES_KILLED
32from testconstants import COUCHBASE_FROM_VERSION_4, COUCHBASE_FROM_WATSON,\
33                          COUCHBASE_FROM_SPOCK
34from testconstants import RPM_DIS_NAME
35from testconstants import LINUX_DISTRIBUTION_NAME, LINUX_CB_PATH, \
36                          LINUX_COUCHBASE_BIN_PATH
37from testconstants import WIN_COUCHBASE_BIN_PATH,\
38                          WIN_CB_PATH
39from testconstants import WIN_COUCHBASE_BIN_PATH_RAW
40from testconstants import WIN_TMP_PATH, WIN_TMP_PATH_RAW
41from testconstants import WIN_UNZIP, WIN_PSSUSPEND
42
43from testconstants import MAC_CB_PATH, MAC_COUCHBASE_BIN_PATH
44
45from testconstants import CB_VERSION_NAME
46from testconstants import CB_REPO
47from testconstants import CB_RELEASE_APT_GET_REPO
48from testconstants import CB_RELEASE_YUM_REPO
49
50from testconstants import LINUX_NONROOT_CB_BIN_PATH,\
51                          NR_INSTALL_LOCATION_FILE
52
53from membase.api.rest_client import RestConnection, RestHelper
54
55log = logger.Logger.get_logger()
56logging.getLogger("paramiko").setLevel(logging.WARNING)
57
58try:
59    import paramiko
60except ImportError:
61    log.warn("{0} {1} {2}".format("Warning: proceeding without importing",
62                                  "paramiko due to import error.",
63                                  "ssh connections to remote machines will fail!\n"))
64
65class RemoteMachineInfo(object):
66    def __init__(self):
67        self.type = ''
68        self.ip = ''
69        self.distribution_type = ''
70        self.architecture_type = ''
71        self.distribution_version = ''
72        self.deliverable_type = ''
73        self.ram = ''
74        self.cpu = ''
75        self.disk = ''
76        self.hostname = ''
77
78
79class RemoteMachineProcess(object):
80    def __init__(self):
81        self.pid = ''
82        self.name = ''
83        self.vsz = 0
84        self.rss = 0
85        self.args = ''
86
87
88class RemoteMachineHelper(object):
89    remote_shell = None
90
91    def __init__(self, remote_shell):
92        self.remote_shell = remote_shell
93
94    def monitor_process(self, process_name,
95                        duration_in_seconds=120):
96        # monitor this process and return if it crashes
97        end_time = time.time() + float(duration_in_seconds)
98        last_reported_pid = None
99        while time.time() < end_time:
100            # get the process list
101            process = self.is_process_running(process_name)
102            if process:
103                if not last_reported_pid:
104                    last_reported_pid = process.pid
105                elif not last_reported_pid == process.pid:
106                    message = 'process {0} was restarted. old pid : {1} new pid : {2}'
107                    log.info(message.format(process_name, last_reported_pid, process.pid))
108                    return False
109                    # check if its equal
110            else:
111                # we should have an option to wait for the process
112                # to start during the timeout
113                # process might have crashed
114                log.info("{0}:process {1} is not running or it might have crashed!"
115                                       .format(self.remote_shell.ip, process_name))
116                return False
117            time.sleep(1)
118        #            log.info('process {0} is running'.format(process_name))
119        return True
120
121    def monitor_process_memory(self, process_name,
122                               duration_in_seconds=180,
123                               end=False):
124        # monitor this process and return list of memories in 7 seconds interval
125        end_time = time.time() + float(duration_in_seconds)
126        count = 0
127        vsz = []
128        rss = []
129        while time.time() < end_time and not end:
130            # get the process list
131            process = self.is_process_running(process_name)
132            if process:
133                vsz.append(process.vsz)
134                rss.append(process.rss)
135            else:
136                log.info("{0}:process {1} is not running.  Wait for 2 seconds"
137                                   .format(self.remote_shell.ip, process_name))
138                count += 1
139                time.sleep(2)
140                if count == 5:
141                    log.error("{0}:process {1} is not running at all."
142                                   .format(self.remote_shell.ip, process_name))
143                    exit(1)
144            log.info("sleep for 7 seconds before poll new processes")
145            time.sleep(7)
146        return vsz, rss
147
148    def is_process_running(self, process_name):
149        if getattr(self.remote_shell, "info", None) is None:
150            self.remote_shell.info = self.remote_shell.extract_remote_info()
151
152        if self.remote_shell.info.type.lower() == 'windows':
153             output, error = self.remote_shell.execute_command('tasklist| grep {0}'
154                                                .format(process_name), debug=False)
155             if error or output == [""] or output == []:
156                 return None
157             words = output[0].split(" ")
158             words = filter(lambda x: x != "", words)
159             process = RemoteMachineProcess()
160             process.pid = words[1]
161             process.name = words[0]
162             log.info("process is running on {0}: {1}".format(self.remote_shell.ip, words))
163             return process
164        else:
165            processes = self.remote_shell.get_running_processes()
166            for process in processes:
167                if process.name == process_name:
168                    return process
169                elif process_name in process.args:
170                    return process
171            return None
172
173
174class RemoteMachineShellConnection:
175    _ssh_client = None
176
177    def __init__(self, username='root',
178                 pkey_location='',
179                 ip=''):
180        self.username = username
181        self.use_sudo = True
182        self.nonroot = False
183        """ in nonroot, we could extract Couchbase Server at
184            any directory that non root user could create
185        """
186        self.nr_home_path = "/home/%s/" % self.username
187        if self.username == 'root':
188            self.use_sudo = False
189        elif self.username != "Administrator":
190            self.use_sudo = False
191            self.nonroot = True
192        # let's create a connection
193        self._ssh_client = paramiko.SSHClient()
194        self.ip = ip
195        self.remote = (self.ip != "localhost" and self.ip != "127.0.0.1")
196        self._ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
197        log.info('connecting to {0} with username : {1} pem key : {2}'.format(ip, username, pkey_location))
198        try:
199            if self.remote:
200                self._ssh_client.connect(hostname=ip, username=username, key_filename=pkey_location)
201        except paramiko.AuthenticationException:
202            log.info("Authentication failed for {0}".format(self.ip))
203            exit(1)
204        except paramiko.BadHostKeyException:
205            log.info("Invalid Host key for {0}".format(self.ip))
206            exit(1)
207        except Exception:
208            log.info("Can't establish SSH session with {0}".format(self.ip))
209            exit(1)
210
211    def __init__(self, serverInfo):
212        # let's create a connection
213        self.username = serverInfo.ssh_username
214        self.password = serverInfo.ssh_password
215        self.ssh_key = serverInfo.ssh_key
216        self.input = TestInput.TestInputParser.get_test_input(sys.argv)
217        self.use_sudo = True
218        self.nonroot = False
219        self.nr_home_path = "/home/%s/" % self.username
220        if self.username == 'root':
221            self.use_sudo = False
222        elif self.username != "Administrator":
223            self.use_sudo = False
224            self.nonroot = True
225        self._ssh_client = paramiko.SSHClient()
226        self.ip = serverInfo.ip
227        self.remote = (self.ip != "localhost" and self.ip != "127.0.0.1")
228        self.port = serverInfo.port
229        self._ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
230        msg = 'connecting to {0} with username:{1} password:{2} ssh_key:{3}'
231        log.info(msg.format(serverInfo.ip, serverInfo.ssh_username,
232                            serverInfo.ssh_password, serverInfo.ssh_key))
233        # added attempts for connection because of PID check failed.
234        # RNG must be re-initialized after fork() error
235        # That's a paramiko bug
236        max_attempts_connect = 2
237        attempt = 0
238        while True:
239            try:
240                if self.remote and serverInfo.ssh_key == '':
241                    self._ssh_client.connect(hostname=serverInfo.ip.replace('[', '').replace(']',''),
242                                             username=serverInfo.ssh_username,
243                                             password=serverInfo.ssh_password)
244                elif self.remote:
245                    self._ssh_client.connect(hostname=serverInfo.ip.replace('[', '').replace(']',''),
246                                             username=serverInfo.ssh_username,
247                                             key_filename=serverInfo.ssh_key)
248                break
249            except paramiko.AuthenticationException:
250                log.error("Authentication failed")
251                exit(1)
252            except paramiko.BadHostKeyException:
253                log.error("Invalid Host key")
254                exit(1)
255            except Exception as e:
256                if str(e).find('PID check failed. RNG must be re-initialized') != -1 and\
257                        attempt != max_attempts_connect:
258                    log.error("Can't establish SSH session to node {1} :\
259                              {0}. Will try again in 1 sec".format(e, self.ip))
260                    attempt += 1
261                    time.sleep(1)
262                else:
263                    log.error("Can't establish SSH session to node {1} :\
264                                                   {0}".format(e, self.ip))
265                    exit(1)
266        log.info("Connected to {0}".format(serverInfo.ip))
267        """ self.info.distribution_type.lower() == "ubuntu" """
268        self.cmd_ext = ""
269        self.bin_path = LINUX_COUCHBASE_BIN_PATH
270        self.msi = False
271        self.extract_remote_info()
272        os_type = self.info.distribution_type.lower()
273        if os_type == "windows":
274            self.cmd_ext = ".exe"
275            self.bin_path = WIN_COUCHBASE_BIN_PATH
276        elif os_type == "mac":
277            self.nonroot = False
278
279        if self.nonroot:
280            self.bin_path = self.nr_home_path + self.bin_path
281
282    """
283        In case of non root user, we need to switch to root to
284        run command
285    """
286    def connect_with_user(self, user="root"):
287        if self.info.distribution_type.lower() == "mac":
288            log.info("This is Mac Server.  Skip re-connect to it as %s" % user)
289            return
290        max_attempts_connect = 2
291        attempt = 0
292        while True:
293            try:
294                log.info("Connect to node: %s as user: %s" % (self.ip, user))
295                if self.remote and self.ssh_key == '':
296                    self._ssh_client.connect(hostname=self.ip.replace('[', '').replace(']',''),
297                                             username=user,
298                                             password=self.password)
299                break
300            except paramiko.AuthenticationException:
301                log.error("Authentication for root failed")
302                exit(1)
303            except paramiko.BadHostKeyException:
304                log.error("Invalid Host key")
305                exit(1)
306            except Exception as e:
307                if str(e).find('PID check failed. RNG must be re-initialized') != -1 and\
308                        attempt != max_attempts_connect:
309                    log.error("Can't establish SSH session to node {1} as root:\
310                              {0}. Will try again in 1 sec".format(e, self.ip))
311                    attempt += 1
312                    time.sleep(1)
313                else:
314                    log.error("Can't establish SSH session to node {1} :\
315                                                   {0}".format(e, self.ip))
316                    exit(1)
317        log.info("Connected to {0} as {1}".format(self.ip, user))
318
319    def sleep(self, timeout=1, message=""):
320        log.info("{0}:sleep for {1} secs. {2} ...".format(self.ip, timeout, message))
321        time.sleep(timeout)
322
323    def get_running_processes(self):
324        # if its linux ,then parse each line
325        # 26989 ?        00:00:51 pdflush
326        # ps -Ao pid,comm
327        processes = []
328        output, error = self.execute_command('ps -Ao pid,comm,vsz,rss,args', debug=False)
329        if output:
330            for line in output:
331                # split to words
332                words = line.strip().split(' ')
333                words = filter(None, words)
334                if len(words) >= 2:
335                    process = RemoteMachineProcess()
336                    process.pid = words[0]
337                    process.name = words[1]
338                    if words[2].isdigit():
339                        process.vsz = int(words[2])/1024
340                    else:
341                        process.vsz = words[2]
342                    if words[3].isdigit():
343                        process.rss = int(words[3])/1024
344                    else:
345                        process.rss = words[3]
346                    process.args = " ".join(words[4:])
347                    processes.append(process)
348        return processes
349
350    def get_mem_usage_by_process(self, process_name):
351        """Now only linux"""
352        output, error = self.execute_command('ps -e -o %mem,cmd|grep {0}'.format(process_name),
353                                             debug=False)
354        if output:
355            for line in output:
356                if not 'grep' in line.strip().split(' '):
357                    return float(line.strip().split(' ')[0])
358
359    def stop_network(self, stop_time):
360        """
361        Stop the network for given time period and then restart the network
362        on the machine.
363        :param stop_time: Time duration for which the network service needs
364        to be down in the machine
365        :return: Nothing
366        """
367        self.extract_remote_info()
368        os_type = self.info.type.lower()
369        if os_type == "unix" or os_type == "linux":
370            if self.info.distribution_type.lower() == "ubuntu":
371                command = "ifdown -a && sleep {} && ifup -a"
372            else:
373                command = "service network stop && sleep {} && service network " \
374                          "start"
375            output, error = self.execute_command(command.format(stop_time))
376            self.log_command_output(output, error)
377        elif os_type == "windows":
378            command = "net stop Netman && timeout {} && net start Netman"
379            output, error = self.execute_command(command.format(stop_time))
380            self.log_command_output(output, error)
381
382    def stop_network(self, stop_time):
383        """
384        Stop the network for given time period and then restart the network
385        on the machine.
386        :param stop_time: Time duration for which the network service needs
387        to be down in the machine
388        :return: Nothing
389        """
390        self.extract_remote_info()
391        os_type = self.info.type.lower()
392        if os_type == "unix" or os_type == "linux":
393            if self.info.distribution_type.lower() == "ubuntu":
394                command = "ifdown -a && sleep {} && ifup -a"
395            else:
396                command = "nohup service network stop && sleep {} && service network " \
397                          "start &"
398            output, error = self.execute_command(command.format(stop_time))
399            self.log_command_output(output, error)
400        elif os_type == "windows":
401            command = "net stop Netman && timeout {} && net start Netman"
402            output, error = self.execute_command(command.format(stop_time))
403            self.log_command_output(output, error)
404
405    def stop_membase(self):
406        self.extract_remote_info()
407        if self.info.type.lower() == 'windows':
408            log.info("STOP SERVER")
409            o, r = self.execute_command("net stop membaseserver")
410            self.log_command_output(o, r)
411            o, r = self.execute_command("net stop couchbaseserver")
412            self.log_command_output(o, r)
413            self.sleep(10, "Wait to stop service completely")
414        if self.info.type.lower() == "linux":
415            o, r = self.execute_command("/etc/init.d/membase-server stop")
416            self.log_command_output(o, r)
417
418    def start_membase(self):
419        self.extract_remote_info()
420        if self.info.type.lower() == 'windows':
421            o, r = self.execute_command("net start membaseserver")
422            self.log_command_output(o, r)
423        if self.info.type.lower() == "linux":
424            o, r = self.execute_command("/etc/init.d/membase-server start")
425            self.log_command_output(o, r)
426
427    def start_server(self, os="unix"):
428        self.extract_remote_info()
429        os = self.info.type.lower()
430        if not os or os == "centos":
431            os = "unix"
432        if os == "windows":
433            o, r = self.execute_command("net start couchbaseserver")
434            self.log_command_output(o, r)
435        elif os == "unix" or os == "linux":
436            if self.is_couchbase_installed():
437                if self.nonroot:
438                    log.info("Start Couchbase Server with non root method")
439                    o, r = self.execute_command('%s%scouchbase-server \-- -noinput -detached '\
440                                              % (self.nr_home_path, LINUX_COUCHBASE_BIN_PATH))
441                    self.log_command_output(o, r)
442                else:
443                    fv, sv, bn = self.get_cbversion("linux")
444                    if self.info.distribution_version.lower() in SYSTEMD_SERVER \
445                                                 and sv in COUCHBASE_FROM_WATSON:
446                        """from watson, systemd is used in centos 7, suse 12 """
447                        log.info("Running systemd command on this server")
448                        o, r = self.execute_command("systemctl start couchbase-server.service")
449                        self.log_command_output(o, r)
450                    else:
451                        o, r = self.execute_command("/etc/init.d/couchbase-server start")
452                        self.log_command_output(o, r)
453        elif os == "mac":
454            o, r = self.execute_command("open /Applications/Couchbase\ Server.app")
455            self.log_command_output(o, r)
456        else:
457            log.error("don't know operating system or product version")
458
459    def stop_server(self, os="unix"):
460        self.extract_remote_info()
461        os = self.info.distribution_type.lower()
462        if not os or os == "centos":
463            os = "unix"
464        if os == "windows":
465            o, r = self.execute_command("net stop couchbaseserver")
466            self.log_command_output(o, r)
467        elif os == "unix" or os == "linux":
468            if self.is_couchbase_installed():
469                if self.nonroot:
470                    o, r = self.execute_command("%s%scouchbase-server -k"
471                                                % (self.nr_home_path,
472                                                   LINUX_COUCHBASE_BIN_PATH))
473                    self.log_command_output(o, r)
474                else:
475                    fv, sv, bn = self.get_cbversion("linux")
476                    if self.info.distribution_version.lower() in SYSTEMD_SERVER \
477                                                 and sv in COUCHBASE_FROM_WATSON:
478                        """from watson, systemd is used in centos 7, suse 12 """
479                        log.info("Running systemd command on this server")
480                        o, r = self.execute_command("systemctl stop couchbase-server.service")
481                        self.log_command_output(o, r)
482                    else:
483                        o, r = self.execute_command("/etc/init.d/couchbase-server stop"\
484                                                                     , use_channel=True)
485            else:
486                log.info("No Couchbase Server installed yet!")
487        elif os == "mac":
488            cb_process = '/Applications/Couchbase\ Server.app/Contents/MacOS/Couchbase\ Server'
489            cmd = "ps aux | grep {0} | awk '{{print $2}}' | xargs kill -9 ".format(cb_process)
490            o, r = self.execute_command(cmd)
491            self.log_command_output(o, r)
492            o, r = self.execute_command("killall -9 epmd")
493            self.log_command_output(o, r)
494        else:
495            log.error("don't know operating system or product version")
496
497    def restart_couchbase(self):
498        """
499        Restart the couchbase server on the machine.
500        :return: Nothing
501        """
502        self.extract_remote_info()
503        if self.info.type.lower() == 'windows':
504            o, r = self.execute_command("net stop couchbaseserver")
505            self.log_command_output(o, r)
506            o, r = self.execute_command("net start couchbaseserver")
507            self.log_command_output(o, r)
508        if self.info.type.lower() == "linux":
509            fv, sv, bn = self.get_cbversion("linux")
510            if "centos 7" in self.info.distribution_version.lower() \
511                    and sv in COUCHBASE_FROM_WATSON:
512                """from watson, systemd is used in centos 7 """
513                log.info("this node is centos 7.x")
514                o, r = self.execute_command("service couchbase-server restart")
515                self.log_command_output(o, r)
516            else:
517                o, r = self.execute_command(
518                    "/etc/init.d/couchbase-server restart")
519                self.log_command_output(o, r)
520        if self.info.distribution_type.lower() == "mac":
521            o, r = self.execute_command(
522                "open /Applications/Couchbase\ Server.app")
523            self.log_command_output(o, r)
524
525    def stop_schedule_tasks(self):
526        log.info("STOP ALL SCHEDULE TASKS: installme, removeme and upgrademe")
527        output, error = self.execute_command("cmd /c schtasks /end /tn installme")
528        self.log_command_output(output, error)
529        output, error = self.execute_command("cmd /c schtasks /end /tn removeme")
530        self.log_command_output(output, error)
531        output, error = self.execute_command("cmd /c schtasks /end /tn upgrademe")
532        self.log_command_output(output, error)
533
534    def kill_erlang(self, os="unix"):
535        if os == "windows":
536            o, r = self.execute_command("taskkill /F /T /IM epmd.exe*")
537            self.log_command_output(o, r)
538            o, r = self.execute_command("taskkill /F /T /IM erl*")
539            self.log_command_output(o, r)
540            o, r = self.execute_command("tasklist | grep erl.exe")
541            kill_all = False
542            while len(o) >= 1 and not kill_all:
543                self.execute_command("taskkill /F /T /IM erl*")
544                o, r = self.execute_command("tasklist | grep erl.exe")
545                if len(o) == 0:
546                    kill_all = True
547                    log.info("all erlang processes were killed")
548        else:
549            o, r = self.execute_command("kill "
550                        " $(ps aux | grep 'beam.smp' | awk '{print $2}')")
551            self.log_command_output(o, r)
552        return o, r
553
554    def kill_cbft_process(self):
555        self.extract_remote_info()
556        if self.info.type.lower() == 'windows':
557            o, r = self.execute_command("taskkill /F /T /IM cbft.exe*")
558            self.log_command_output(o, r)
559        else:
560            o, r = self.execute_command("killall -9 cbft")
561            self.log_command_output(o, r)
562        return o, r
563
564    def kill_memcached(self):
565        self.extract_remote_info()
566        if self.info.type.lower() == 'windows':
567            o, r = self.execute_command("taskkill /F /T /IM memcached*")
568            self.log_command_output(o, r)
569        else:
570            # Changed from kill -9 $(ps aux | grep 'memcached' | awk '{print $2}' as grep was also returning eventing
571            # process which was using memcached-cert
572            o, r = self.execute_command("kill -9 $(ps aux | pgrep 'memcached')")
573            self.log_command_output(o, r)
574        return o, r
575
576    def stop_memcached(self):
577        self.extract_remote_info()
578        if self.info.type.lower() == 'windows':
579            o, r = self.execute_command("taskkill /F /T /IM memcached*")
580            self.log_command_output(o, r)
581        else:
582            o, r = self.execute_command("kill -SIGSTOP $(pgrep memcached)")
583            self.log_command_output(o, r)
584        return o, r
585
586    def start_memcached(self):
587        self.extract_remote_info()
588        if self.info.type.lower() == 'windows':
589            o, r = self.execute_command("taskkill /F /T /IM memcached*")
590            self.log_command_output(o, r)
591        else:
592            o, r = self.execute_command("kill -SIGCONT $(pgrep memcached)")
593            self.log_command_output(o, r)
594        return o, r
595
596    def kill_goxdcr(self):
597        self.extract_remote_info()
598        if self.info.type.lower() == 'windows':
599            o, r = self.execute_command("taskkill /F /T /IM goxdcr*")
600            self.log_command_output(o, r)
601        else:
602            o, r = self.execute_command("killall -9 goxdcr")
603            self.log_command_output(o, r)
604        return o, r
605
606    def kill_eventing_process(self, name):
607        self.extract_remote_info()
608        if self.info.type.lower() == 'windows':
609            o, r = self.execute_command(command="taskkill /F /T /IM {0}*".format(name))
610            self.log_command_output(o, r)
611        else:
612            o, r = self.execute_command(command="killall -9 {0}".format(name))
613            self.log_command_output(o, r)
614        return o, r
615
616    def reboot_node(self):
617        self.extract_remote_info()
618        if self.extract_remote_info().type.lower() == 'windows':
619            o, r = self.execute_command("shutdown -r -f -t 0")
620            self.log_command_output(o, r)
621        elif self.extract_remote_info().type.lower() == 'linux':
622            o, r = self.execute_command("reboot")
623            self.log_command_output(o, r)
624        return o, r
625
626    def change_log_level(self, new_log_level):
627        log.info("CHANGE LOG LEVEL TO %s".format(new_log_level))
628        # ADD NON_ROOT user config_details
629        output, error = self.execute_command("sed -i '/loglevel_default, /c \\{loglevel_default, %s\}'. %s"
630                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
631        self.log_command_output(output, error)
632        output, error = self.execute_command("sed -i '/loglevel_ns_server, /c \\{loglevel_ns_server, %s\}'. %s"
633                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
634        self.log_command_output(output, error)
635        output, error = self.execute_command("sed -i '/loglevel_stats, /c \\{loglevel_stats, %s\}'. %s"
636                                             % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
637        self.log_command_output(output, error)
638        output, error = self.execute_command("sed -i '/loglevel_rebalance, /c \\{loglevel_rebalance, %s\}'. %s"
639                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
640        self.log_command_output(output, error)
641        output, error = self.execute_command("sed -i '/loglevel_cluster, /c \\{loglevel_cluster, %s\}'. %s"
642                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
643        self.log_command_output(output, error)
644        output, error = self.execute_command("sed -i '/loglevel_views, /c \\{loglevel_views, %s\}'. %s"
645                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
646        self.log_command_output(output, error)
647        output, error = self.execute_command("sed -i '/loglevel_error_logger, /c \\{loglevel_error_logger, %s\}'. %s"
648                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
649        self.log_command_output(output, error)
650        output, error = self.execute_command("sed -i '/loglevel_mapreduce_errors, /c \\{loglevel_mapreduce_errors, %s\}'. %s"
651                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
652        self.log_command_output(output, error)
653        output, error = self.execute_command("sed -i '/loglevel_user, /c \\{loglevel_user, %s\}'. %s"
654                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
655        self.log_command_output(output, error)
656        output, error = self.execute_command("sed -i '/loglevel_xdcr, /c \\{loglevel_xdcr, %s\}'. %s"
657                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
658        self.log_command_output(output, error)
659        output, error = self.execute_command("sed -i '/loglevel_menelaus, /c \\{loglevel_menelaus, %s\}'. %s"
660                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
661        self.log_command_output(output, error)
662
663    def configure_log_location(self, new_log_location):
664        mv_logs = testconstants.LINUX_LOG_PATH + '/' + new_log_location
665        print " MV LOGS %s" % mv_logs
666        error_log_tag = "error_logger_mf_dir"
667        # ADD NON_ROOT user config_details
668        log.info("CHANGE LOG LOCATION TO %s".format(mv_logs))
669        output, error = self.execute_command("rm -rf %s" % mv_logs)
670        self.log_command_output(output, error)
671        output, error = self.execute_command("mkdir %s" % mv_logs)
672        self.log_command_output(output, error)
673        output, error = self.execute_command("chown -R couchbase %s" % mv_logs)
674        self.log_command_output(output, error)
675        output, error = self.execute_command("sed -i '/%s, /c \\{%s, \"%s\"\}.' %s"
676                                        % (error_log_tag, error_log_tag, mv_logs, testconstants.LINUX_STATIC_CONFIG))
677        self.log_command_output(output, error)
678
679    def change_stat_periodicity(self, ticks):
680        # ADD NON_ROOT user config_details
681        log.info("CHANGE STAT PERIODICITY TO every %s seconds" % ticks)
682        output, error = self.execute_command("sed -i '$ a\{grab_stats_every_n_ticks, %s}.'  %s"
683        % (ticks, testconstants.LINUX_STATIC_CONFIG))
684        self.log_command_output(output, error)
685
686    def change_port_static(self, new_port):
687        # ADD NON_ROOT user config_details
688        log.info("=========CHANGE PORTS for REST: %s, MCCOUCH: %s,MEMCACHED: %s, MOXI: %s, CAPI: %s==============="
689                % (new_port, new_port + 1, new_port + 2, new_port + 3, new_port + 4))
690        output, error = self.execute_command("sed -i '/{rest_port/d' %s" % testconstants.LINUX_STATIC_CONFIG)
691        self.log_command_output(output, error)
692        output, error = self.execute_command("sed -i '$ a\{rest_port, %s}.' %s"
693                                             % (new_port, testconstants.LINUX_STATIC_CONFIG))
694        self.log_command_output(output, error)
695        output, error = self.execute_command("sed -i '/{mccouch_port/d' %s" % testconstants.LINUX_STATIC_CONFIG)
696        self.log_command_output(output, error)
697        output, error = self.execute_command("sed -i '$ a\{mccouch_port, %s}.' %s"
698                                             % (new_port + 1, testconstants.LINUX_STATIC_CONFIG))
699        self.log_command_output(output, error)
700        output, error = self.execute_command("sed -i '/{memcached_port/d' %s" % testconstants.LINUX_STATIC_CONFIG)
701        self.log_command_output(output, error)
702        output, error = self.execute_command("sed -i '$ a\{memcached_port, %s}.' %s"
703                                             % (new_port + 2, testconstants.LINUX_STATIC_CONFIG))
704        self.log_command_output(output, error)
705        output, error = self.execute_command("sed -i '/{moxi_port/d' %s" % testconstants.LINUX_STATIC_CONFIG)
706        self.log_command_output(output, error)
707        output, error = self.execute_command("sed -i '$ a\{moxi_port, %s}.' %s"
708                                             % (new_port + 3, testconstants.LINUX_STATIC_CONFIG))
709        self.log_command_output(output, error)
710        output, error = self.execute_command("sed -i '/port = /c\port = %s' %s"
711                                             % (new_port + 4, testconstants.LINUX_CAPI_INI))
712        self.log_command_output(output, error)
713        output, error = self.execute_command("rm %s" % testconstants.LINUX_CONFIG_FILE)
714        self.log_command_output(output, error)
715        output, error = self.execute_command("cat %s" % testconstants.LINUX_STATIC_CONFIG)
716        self.log_command_output(output, error)
717
718    def is_couchbase_installed(self):
719        self.extract_remote_info()
720        if self.info.type.lower() == 'windows':
721            if self.file_exists(WIN_CB_PATH, VERSION_FILE):
722                log.info("{0} **** The version file {1} {2}  exists"\
723                          .format(self.ip, WIN_CB_PATH, VERSION_FILE ))
724                # print running process on windows
725                RemoteMachineHelper(self).is_process_running('memcached')
726                RemoteMachineHelper(self).is_process_running('erl')
727                return True
728        elif self.info.distribution_type.lower() == 'mac':
729            output, error = self.execute_command('ls %s%s' % (MAC_CB_PATH, VERSION_FILE))
730            self.log_command_output(output, error)
731            for line in output:
732                if line.find('No such file or directory') == -1:
733                    return True
734        elif self.info.type.lower() == "linux":
735            if self.nonroot:
736                if self.file_exists("/home/%s/" % self.username, NR_INSTALL_LOCATION_FILE):
737                    output, error = self.execute_command("cat %s" % NR_INSTALL_LOCATION_FILE)
738                    if output and output[0]:
739                        log.info("Couchbase Server was installed in non default path %s"
740                                                                            % output[0])
741                        self.nr_home_path = output[0]
742                file_path = self.nr_home_path + LINUX_CB_PATH
743                if self.file_exists(file_path, VERSION_FILE):
744                    log.info("non root couchbase installed at %s " % self.ip)
745                    return True
746            else:
747                if self.file_exists(LINUX_CB_PATH, VERSION_FILE):
748                    log.info("{0} **** The version file {1} {2}  exists".format(self.ip,
749                                                      LINUX_CB_PATH, VERSION_FILE ))
750                    return True
751        return False
752
753    def is_moxi_installed(self):
754        self.extract_remote_info()
755        if self.info.type.lower() == 'windows':
756            log.error('Not implemented')
757        elif self.info.distribution_type.lower() == 'mac':
758            log.error('Not implemented')
759        elif self.info.type.lower() == "linux":
760            if self.file_exists(testconstants.LINUX_MOXI_PATH, 'moxi'):
761                return True
762        return False
763
764    # /opt/moxi/bin/moxi -Z port_listen=11211 -u root -t 4 -O /var/log/moxi/moxi.log
765    def start_moxi(self, ip, bucket, port, user=None, threads=4, log_file="/var/log/moxi.log"):
766        if self.is_couchbase_installed():
767            prod = "couchbase"
768        else:
769            prod = "membase"
770        cli_path = "/opt/" + prod + "/bin/moxi"
771        args = ""
772        args += "http://{0}:8091/pools/default/bucketsStreaming/{1} ".format(ip, bucket)
773        args += "-Z port_listen={0} -u {1} -t {2} -O {3} -d".format(port,
774                                                                    user or prod,
775                                                                    threads, log_file)
776        self.extract_remote_info()
777        if self.info.type.lower() == "linux":
778            o, r = self.execute_command("{0} {1}".format(cli_path, args))
779            self.log_command_output(o, r)
780        else:
781            raise Exception("running standalone moxi is not supported for windows")
782
783    def stop_moxi(self):
784        self.extract_remote_info()
785        if self.info.type.lower() == "linux":
786            o, r = self.execute_command("killall -9 moxi")
787            self.log_command_output(o, r)
788        else:
789            raise Exception("stopping standalone moxi is not supported on windows")
790    def is_url_live(self, url):
791        live_url = False
792        log.info("Check if url {0} is ok".format(url))
793        status = urllib.urlopen(url).getcode()
794        if status == 200:
795            log.info("This url {0} is live".format(url))
796            live_url = True
797        else:
798            mesg = "\n===============\n"\
799                   "        This url {0} \n"\
800                   "        is failed to connect.\n"\
801                   "        Check version in params to make sure it correct pattern or build number.\n"\
802                   "===============\n".format(url)
803            self.stop_current_python_running(mesg)
804        return live_url
805
806    def is_ntp_installed(self):
807        ntp_installed = False
808        do_install = False
809        os_version = ""
810        log.info("Check if ntp is installed")
811        self.extract_remote_info()
812        if self.info.type.lower() == 'linux':
813            log.info("\nThis OS version %s" % self.info.distribution_version.lower())
814            if "centos 7" in self.info.distribution_version.lower():
815                os_version = "centos 7"
816                output, e = self.execute_command("systemctl status ntpd")
817                for line in output:
818                    if "Active: active (running)" in line:
819                        ntp_installed = True
820                if not ntp_installed:
821                    log.info("ntp not installed yet or not run.\n"\
822                             "Let remove any old one and install ntp")
823                    self.execute_command("yum erase -y ntp", debug=False)
824                    self.execute_command("yum install -y ntp", debug=False)
825                    self.execute_command("systemctl start ntpd", debug=False)
826                    self.execute_command("systemctl enable ntpd", debug=False)
827                    self.execute_command("firewall-cmd --add-service=ntp --permanent",
828                                                                          debug=False)
829                    self.execute_command("firewall-cmd --reload", debug=False)
830                    do_install = True
831                timezone, _ = self.execute_command("date")
832                if "PST" not in timezone[0]:
833                    self.execute_command("timedatectl set-timezone America/Los_Angeles",
834                                                                            debug=False)
835            elif "centos release 6" in self.info.distribution_version.lower():
836                os_version = "centos 6"
837                output, e = self.execute_command("/etc/init.d/ntpd status")
838                if not output:
839                    log.info("ntp was not installed on {0} server yet.  "\
840                             "Let install ntp on this server ".format(self.ip))
841                    self.execute_command("yum install -y ntp ntpdate", debug=False)
842                    self.execute_command("chkconfig ntpd on", debug=False)
843                    self.execute_command("ntpdate pool.ntp.org", debug=False)
844                    self.execute_command("/etc/init.d/ntpd start", debug=False)
845                    do_install = True
846                elif output and "ntpd is stopped" in output[0]:
847                    log.info("ntp is not running.  Let remove it and install again in {0}"\
848                                                            .format(self.ip))
849                    self.execute_command("yum erase -y ntp", debug=False)
850                    self.execute_command("yum install -y ntp ntpdate", debug=False)
851                    self.execute_command("chkconfig ntpd on", debug=False)
852                    self.execute_command("ntpdate pool.ntp.org", debug=False)
853                    self.execute_command("/etc/init.d/ntpd start", debug=False)
854                    do_install = True
855                elif output and "is running..." in output[0]:
856                    ntp_installed = True
857                timezone, _ = self.execute_command("date")
858                if "PST" not in timezone[0]:
859                    self.execute_command("cp /etc/localtime /root/old.timezone",
860                                                                    debug=False)
861                    self.execute_command("rm -rf /etc/localtime", debug=False)
862                    self.execute_command("ln -s /usr/share/zoneinfo/America/Los_Angeles "
863                                         "/etc/localtime", debug=False)
864            else:
865                log.info("will add install in other os later, no set do install")
866
867        if do_install:
868            if os_version == "centos 7":
869                output, e = self.execute_command("systemctl status ntpd")
870                for line in output:
871                    if "Active: active (running)" in line:
872                        log.info("ntp is installed and running on this server %s" % self.ip)
873                        ntp_installed = True
874                        break;
875            if os_version == "centos 6":
876                output, e = self.execute_command("/etc/init.d/ntpd status")
877                if output and " is running..." in output[0]:
878                    log.info("ntp is installed and running on this server %s" % self.ip)
879                    ntp_installed = True
880
881        output, _ = self.execute_command("date", debug=False)
882        log.info("\n{0} IP: {1}".format(output, self.ip))
883        if not ntp_installed and "centos" in os_version:
884            mesg = "\n===============\n"\
885                   "        This server {0} \n"\
886                   "        failed to install ntp service.\n"\
887                   "===============\n".format(self.ip)
888            self.stop_current_python_running(mesg)
889
890    def download_build(self, build):
891        return self.download_binary(build.url, build.deliverable_type, build.name,
892                                                latest_url=build.url_latest_build,
893                                                version=build.product_version)
894
895    def disable_firewall(self):
896        self.extract_remote_info()
897        if self.info.type.lower() == "windows":
898            output, error = self.execute_command('netsh advfirewall set publicprofile state off')
899            self.log_command_output(output, error)
900            output, error = self.execute_command('netsh advfirewall set privateprofile state off')
901            self.log_command_output(output, error)
902            # for details see RemoteUtilHelper.enable_firewall for windows
903            output, error = self.execute_command('netsh advfirewall firewall delete rule name="block erl.exe in"')
904            self.log_command_output(output, error)
905            output, error = self.execute_command('netsh advfirewall firewall delete rule name="block erl.exe out"')
906            self.log_command_output(output, error)
907        else:
908            command_1 = "/sbin/iptables -F"
909            command_2 = "/sbin/iptables -t nat -F"
910            if self.nonroot:
911                log.info("\n Non root or non sudo has no right to disable firewall")
912                return
913            output, error = self.execute_command(command_1)
914            self.log_command_output(output, error)
915            output, error = self.execute_command(command_2)
916            self.log_command_output(output, error)
917            self.connect_with_user(user=self.username)
918
919    def download_binary(self, url, deliverable_type, filename, latest_url=None,
920                                              version="", skip_md5_check=True):
921        self.extract_remote_info()
922        self.disable_firewall()
923        file_status = False
924        if self.info.type.lower() == 'windows':
925            self.terminate_processes(self.info, [s for s in WIN_PROCESSES_KILLED])
926            self.terminate_processes(self.info, \
927                                     [s + "-*" for s in COUCHBASE_FROM_VERSION_3])
928            self.disable_firewall()
929            remove_words = ["-rel", ".exe"]
930            for word in remove_words:
931                filename = filename.replace(word, "")
932            """couchbase-server-enterprise_3.5.0-968-windows_amd64
933               couchbase-server-enterprise_4.0.0-1655-windows_amd64
934               sherlock changed from 3.5. to 4.0 """
935            filename_version = ""
936            if len(filename) > 40:
937                if "enterprise" in filename:
938                    filename_version = filename[28:33]
939                elif "community" in filename:
940                    filename_version = filename[27:32]
941            if filename_version in COUCHBASE_FROM_VERSION_4:
942                log.info("This version is {0}".format(filename_version))
943                tmp = filename.split("_")
944                version = tmp[1].replace("-windows", "")
945            else:
946                tmp = filename.split("-")
947                tmp.reverse()
948                version = tmp[1] + "-" + tmp[0]
949
950            exist = self.file_exists('/cygdrive/c/tmp/', '{0}.exe'.format(version))
951            command = "cd /cygdrive/c/tmp;cmd /c 'c:\\automation\\wget.exe "\
952                                        "--no-check-certificate"\
953                                        " -q {0} -O {1}.exe';ls -lh;".format(url, version)
954            file_location = "/cygdrive/c/tmp/"
955            deliverable_type = "exe"
956            if not exist:
957                output, error = self.execute_command(command)
958                self.log_command_output(output, error)
959                if not self.file_exists(file_location, '{0}.exe'.format(version)):
960                    file_status = self.check_and_retry_download_binary(command,
961                                                             file_location, version)
962                return file_status
963            else:
964                log.info('File {0}.exe exist in tmp directory'.format(version))
965                return True
966
967        elif self.info.distribution_type.lower() == 'mac':
968            command = "cd ~/Downloads ; rm -rf couchbase-server* ;"\
969                      " rm -rf Couchbase\ Server.app ; curl -O {0}".format(url)
970            output, error = self.execute_command(command)
971            self.log_command_output(output, error)
972            output, error = self.execute_command('ls -lh  ~/Downloads/%s' % filename)
973            self.log_command_output(output, error)
974            for line in output:
975                if line.find('No such file or directory') == -1:
976                    return True
977            return False
978        else:
979        # try to push this build into
980        # depending on the os
981        # build.product has the full name
982        # first remove the previous file if it exist ?
983        # fix this :
984            command1 = "cd /tmp ; D=$(mktemp -d cb_XXXX) ; mv {0} $D ; mv core.* $D ;"\
985                                     " rm -f * ; mv $D/* . ; rmdir $D".format(filename)
986            command_root = "cd /tmp;wget -q -O {0} {1};cd /tmp;ls -lh".format(filename, url)
987            file_location = "/tmp"
988            output, error = self.execute_command_raw(command1)
989            self.log_command_output(output, error)
990            if skip_md5_check:
991                if self.nonroot:
992                    output, error = self.execute_command("ls -lh ")
993                    self.log_command_output(output, error)
994                    log.info("remove old couchbase server binary ")
995                    if self.file_exists("/home/%s/" % self.username, NR_INSTALL_LOCATION_FILE):
996                        output, error = self.execute_command("cat %s"
997                                             % NR_INSTALL_LOCATION_FILE)
998                        if output and output[0]:
999                            log.info("Couchbase Server was installed in non default path %s"
1000                                                                            % output[0])
1001                        self.nr_home_path = output[0]
1002                    self.execute_command_raw('cd %s;rm couchbase-server-*'
1003                                                      % self.nr_home_path)
1004                    if "nr_install_dir" in self.input.test_params and \
1005                                           self.input.test_params["nr_install_dir"]:
1006                        self.nr_home_path = self.nr_home_path + self.input.test_params["nr_install_dir"]
1007                        op, er = self.execute_command("echo %s > %s" % (self.nr_home_path,
1008                                                              NR_INSTALL_LOCATION_FILE))
1009                        self.log_command_output(op, er)
1010                        op, er = self.execute_command("rm -rf %s" % self.nr_home_path)
1011                        self.log_command_output(op, er)
1012                        op, er = self.execute_command("mkdir %s" % self.nr_home_path)
1013                        self.log_command_output(op, er)
1014                    output, error = self.execute_command("ls -lh ")
1015                    self.log_command_output(output, error)
1016                    output, error = self.execute_command_raw('cd {2}; pwd;'
1017                                                             ' wget -q -O {0} {1};ls -lh'
1018                                                             .format(filename, url,
1019                                                                     self.nr_home_path))
1020                    self.log_command_output(output, error)
1021                else:
1022                    output, error = self.execute_command_raw(command_root)
1023                    self.log_command_output(output, error)
1024            else:
1025                log.info('get md5 sum for local and remote')
1026                output, error = self.execute_command_raw('cd /tmp ; rm -f *.md5 *.md5l ; wget -q -O {1}.md5 {0}.md5 ; md5sum {1} > {1}.md5l'.format(url, filename))
1027                self.log_command_output(output, error)
1028                if str(error).find('No such file or directory') != -1 and latest_url != '':
1029                    url = latest_url
1030                    output, error = self.execute_command_raw('cd /tmp ; rm -f *.md5 *.md5l ; wget -q -O {1}.md5 {0}.md5 ; md5sum {1} > {1}.md5l'.format(url, filename))
1031                log.info('comparing md5 sum and downloading if needed')
1032                output, error = self.execute_command_raw('cd /tmp;diff {0}.md5 {0}.md5l || wget -q -O {0} {1};rm -f *.md5 *.md5l'.format(filename, url))
1033                self.log_command_output(output, error)
1034            # check if the file exists there now ?
1035            if self.nonroot:
1036                """ binary is saved at current user directory """
1037                return self.file_exists(self.nr_home_path, filename)
1038            else:
1039                file_status = self.file_exists(file_location, filename)
1040                if not file_status:
1041                    file_status = self.check_and_retry_download_binary(command_root,
1042                                                                      file_location,
1043                                                                      version)
1044                return file_status
1045            # for linux environment we can just
1046            # figure out what version , check if /tmp/ has the
1047            # binary and then return True if binary is installed
1048
1049
1050    def get_file(self, remotepath, filename, todir):
1051        if self.file_exists(remotepath, filename):
1052            if self.remote:
1053                sftp = self._ssh_client.open_sftp()
1054                try:
1055                    filenames = sftp.listdir(remotepath)
1056                    for name in filenames:
1057                        if name == filename:
1058                            sftp.get('{0}/{1}'.format(remotepath, filename), todir)
1059                            sftp.close()
1060                            return True
1061                    sftp.close()
1062                    return False
1063                except IOError:
1064                    return False
1065        else:
1066            os.system("cp {0} {1}".format('{0}/{1}'.format(remotepath, filename), todir))
1067
1068    def read_remote_file(self, remote_path, filename):
1069        if self.file_exists(remote_path, filename):
1070            if self.remote:
1071                sftp = self._ssh_client.open_sftp()
1072                remote_file = sftp.open('{0}/{1}'.format(remote_path, filename))
1073                try:
1074                    out = remote_file.readlines()
1075                finally:
1076                    remote_file.close()
1077                return out
1078            else:
1079                txt = open('{0}/{1}'.format(remote_path, filename))
1080                return txt.read()
1081        return None
1082
1083    def write_remote_file(self, remote_path, filename, lines):
1084        cmd = 'echo "%s" > %s/%s' % (''.join(lines), remote_path, filename)
1085        self.execute_command(cmd)
1086
1087    def write_remote_file_single_quote(self, remote_path, filename, lines):
1088        cmd = 'echo \'%s\' > %s/%s' % (''.join(lines), remote_path, filename)
1089        self.execute_command(cmd)
1090
1091    def create_whitelist(self, path, whitelist):
1092        if not os.path.exists(path):
1093            self.execute_command("mkdir %s" % path)
1094        filepath = os.path.join(path, "curl_whitelist.json")
1095        if os.path.exists(filepath):
1096            os.remove(filepath)
1097        file_data = json.dumps(whitelist)
1098        self.create_file(filepath, file_data)
1099
1100    def remove_directory(self, remote_path):
1101        if self.remote:
1102            sftp = self._ssh_client.open_sftp()
1103            try:
1104                log.info("removing {0} directory...".format(remote_path))
1105                sftp.rmdir(remote_path)
1106            except IOError:
1107                return False
1108            finally:
1109                sftp.close()
1110        else:
1111            try:
1112                p = Popen("rm -rf {0}".format(remote_path) , shell=True, stdout=PIPE, stderr=PIPE)
1113                stdout, stderro = p.communicate()
1114            except IOError:
1115                return False
1116        return True
1117
1118    def rmtree(self, sftp, remote_path, level=0):
1119        for f in sftp.listdir_attr(remote_path):
1120            rpath = remote_path + "/" + f.filename
1121            if stat.S_ISDIR(f.st_mode):
1122                self.rmtree(sftp, rpath, level=(level + 1))
1123            else:
1124                rpath = remote_path + "/" + f.filename
1125                print('removing %s' % (rpath))
1126                sftp.remove(rpath)
1127        print('removing %s' % (remote_path))
1128        sftp.rmdir(remote_path)
1129
1130    def remove_directory_recursive(self, remote_path):
1131        if self.remote:
1132            sftp = self._ssh_client.open_sftp()
1133            try:
1134                log.info("removing {0} directory...".format(remote_path))
1135                self.rmtree(sftp, remote_path)
1136            except IOError:
1137                return False
1138            finally:
1139                sftp.close()
1140        else:
1141            try:
1142                p = Popen("rm -rf {0}".format(remote_path) , shell=True, stdout=PIPE, stderr=PIPE)
1143                p.communicate()
1144            except IOError:
1145                return False
1146        return True
1147
1148    def list_files(self, remote_path):
1149        if self.remote:
1150            sftp = self._ssh_client.open_sftp()
1151            files = []
1152            try:
1153                file_names = sftp.listdir(remote_path)
1154                for name in file_names:
1155                    files.append({'path': remote_path, 'file': name})
1156                sftp.close()
1157            except IOError:
1158                return []
1159            return files
1160        else:
1161            p = Popen("ls {0}".format(remote_path) , shell=True, stdout=PIPE, stderr=PIPE)
1162            files, stderro = p.communicate()
1163            return files
1164
1165    def file_ends_with(self, remotepath, pattern):
1166        """
1167         Check if file ending with this pattern is present in remote machine
1168        """
1169        sftp = self._ssh_client.open_sftp()
1170        files_matched = []
1171        try:
1172            file_names = sftp.listdir(remotepath)
1173            for name in file_names:
1174                if name.endswith(pattern):
1175                    files_matched.append("{0}/{1}".format(remotepath, name))
1176        except IOError:
1177            # ignore this error
1178            pass
1179        sftp.close()
1180        if len(files_matched) > 0:
1181            log.info("found these files : {0}".format(files_matched))
1182        return files_matched
1183
1184    # check if this file exists in the remote
1185    # machine or not
1186    def file_starts_with(self, remotepath, pattern):
1187        sftp = self._ssh_client.open_sftp()
1188        files_matched = []
1189        try:
1190            file_names = sftp.listdir(remotepath)
1191            for name in file_names:
1192                if name.startswith(pattern):
1193                    files_matched.append("{0}/{1}".format(remotepath, name))
1194        except IOError:
1195            # ignore this error
1196            pass
1197        sftp.close()
1198        if len(files_matched) > 0:
1199            log.info("found these files : {0}".format(files_matched))
1200        return files_matched
1201
1202    def file_exists(self, remotepath, filename, pause_time=30):
1203        sftp = self._ssh_client.open_sftp()
1204        try:
1205            filenames = sftp.listdir_attr(remotepath)
1206            for name in filenames:
1207                if filename in name.filename and int(name.st_size) > 0:
1208                    sftp.close()
1209                    return True
1210                elif filename in name.filename and int(name.st_size) == 0:
1211                    if name.filename == NR_INSTALL_LOCATION_FILE:
1212                        continue
1213                    log.info("File {0} will be deleted".format(filename))
1214                    if not remotepath.endswith("/"):
1215                        remotepath += "/"
1216                    self.execute_command("rm -rf {0}*{1}*".format(remotepath,filename))
1217                    self.sleep(pause_time, "** Network or sever may be busy. **"\
1218                                           "\nWait {0} seconds before executing next instrucion"\
1219                                                                             .format(pause_time))
1220
1221            sftp.close()
1222            return False
1223        except IOError:
1224            return False
1225
1226    def delete_file(self, remotepath, filename):
1227        sftp = self._ssh_client.open_sftp()
1228        delete_file = False
1229        try:
1230            filenames = sftp.listdir_attr(remotepath)
1231            for name in filenames:
1232                if name.filename == filename:
1233                    log.info("File {0} will be deleted".format(filename))
1234                    sftp.remove(remotepath + filename)
1235                    delete_file = True
1236                    break
1237            if delete_file:
1238                """ verify file is deleted """
1239                filenames = sftp.listdir_attr(remotepath)
1240                for name in filenames:
1241                    if name.filename == filename:
1242                        log.error("fail to remove file %s " % filename)
1243                        delete_file = False
1244                        break
1245            sftp.close()
1246            return delete_file
1247        except IOError:
1248            return False
1249
1250    def download_binary_in_win(self, url, version, msi_install=False):
1251        self.terminate_processes(self.info, [s for s in WIN_PROCESSES_KILLED])
1252        self.terminate_processes(self.info, \
1253                                 [s + "-*" for s in COUCHBASE_FROM_VERSION_3])
1254        self.disable_firewall()
1255        version = version.replace("-rel", "")
1256        deliverable_type = "exe"
1257        if url and url.split(".")[-1] == "msi":
1258            deliverable_type = "msi"
1259        exist = self.file_exists('/cygdrive/c/tmp/', version)
1260        log.info('**** about to do the wget ****')
1261        command = "cd /cygdrive/c/tmp;cmd /c 'c:\\automation\\wget.exe "\
1262                        " --no-check-certificate"\
1263                        " -q {0} -O {1}.{2}';ls -lh;".format(url, version, deliverable_type)
1264        file_location = "/cygdrive/c/tmp/"
1265        if not exist:
1266            output, error = self.execute_command(command)
1267            self.log_command_output(output, error)
1268        else:
1269            log.info('File {0}.{1} exist in tmp directory'.format(version,
1270                                                                  deliverable_type))
1271        file_status = self.file_exists('/cygdrive/c/tmp/', version)
1272        if not file_status:
1273            file_status = self.check_and_retry_download_binary(command, file_location, version)
1274        return file_status
1275
1276    def check_and_retry_download_binary(self, command, file_location,
1277                                        version, time_of_try=3):
1278        count = 1
1279        file_status = self.file_exists(file_location, version, pause_time=60)
1280        while count <= time_of_try and not file_status:
1281            log.info(" *** try to download binary again {0} time(s)".format(count))
1282            output, error = self.execute_command(command)
1283            self.log_command_output(output, error)
1284            file_status = self.file_exists(file_location, version, pause_time=60)
1285            count += 1
1286            if not file_status and count == 3:
1287                log.error("build {0} did not download completely at server {1}"
1288                                                   .format(version, self.ip))
1289                mesg = "stop job due to failure download build {0} at {1} " \
1290                                                     .format(version, self.ip)
1291                self.stop_current_python_running(mesg)
1292        return file_status
1293
1294    def copy_file_local_to_remote(self, src_path, des_path):
1295        sftp = self._ssh_client.open_sftp()
1296        try:
1297            sftp.put(src_path, des_path)
1298        except IOError:
1299            log.error('Can not copy file')
1300        finally:
1301            sftp.close()
1302
1303    def copy_file_remote_to_local(self, rem_path, des_path):
1304        sftp = self._ssh_client.open_sftp()
1305        try:
1306            sftp.get(rem_path, des_path)
1307        except IOError as e:
1308            if e:
1309                print e
1310            log.error('Can not copy file')
1311        finally:
1312            sftp.close()
1313
1314
1315    # copy multi files from local to remote server
1316    def copy_files_local_to_remote(self, src_path, des_path):
1317        files = os.listdir(src_path)
1318        log.info("copy files from {0} to {1}".format(src_path, des_path))
1319        # self.execute_batch_command("cp -r  {0}/* {1}".format(src_path, des_path))
1320        for file in files:
1321            if file.find("wget") != 1:
1322                a = ""
1323            full_src_path = os.path.join(src_path, file)
1324            full_des_path = os.path.join(des_path, file)
1325            self.copy_file_local_to_remote(full_src_path, full_des_path)
1326
1327
1328    # create a remote file from input string
1329    def create_file(self, remote_path, file_data):
1330        output, error = self.execute_command("echo '{0}' > {1}".format(file_data, remote_path))
1331
1332    def find_file(self, remote_path, file):
1333        sftp = self._ssh_client.open_sftp()
1334        try:
1335            files = sftp.listdir(remote_path)
1336            for name in files:
1337                if name == file:
1338                    found_it = os.path.join(remote_path, name)
1339                    log.info("File {0} was found".format(found_it))
1340                    return found_it
1341            else:
1342                log.error('File(s) name in {0}'.format(remote_path))
1343                for name in files:
1344                    log.info(name)
1345                log.error('Can not find {0}'.format(file))
1346        except IOError:
1347            pass
1348        sftp.close()
1349
1350    def find_build_version(self, path_to_version, version_file, product):
1351        sftp = self._ssh_client.open_sftp()
1352        ex_type = "exe"
1353        if self.file_exists(WIN_CB_PATH, VERSION_FILE):
1354            path_to_version = WIN_CB_PATH
1355        else:
1356            path_to_version = testconstants.WIN_MB_PATH
1357        try:
1358            log.info(path_to_version)
1359            f = sftp.open(os.path.join(path_to_version, VERSION_FILE), 'r+')
1360            tmp_str = f.read().strip()
1361            full_version = tmp_str.replace("-rel", "")
1362            if full_version == "1.6.5.4-win64":
1363                full_version = "1.6.5.4"
1364            build_name = short_version = full_version
1365            return build_name, short_version, full_version
1366        except IOError:
1367            log.error('Can not read version file')
1368        sftp.close()
1369
1370    def find_windows_info(self):
1371        if self.remote:
1372            found = self.find_file("/cygdrive/c/tmp", "windows_info.txt")
1373            if isinstance(found, basestring):
1374                if self.remote:
1375
1376                    sftp = self._ssh_client.open_sftp()
1377                    try:
1378                        f = sftp.open(found)
1379                        log.info("get windows information")
1380                        info = {}
1381                        for line in f:
1382                            (key, value) = line.split('=')
1383                            key = key.strip(' \t\n\r')
1384                            value = value.strip(' \t\n\r')
1385                            info[key] = value
1386                        return info
1387                    except IOError:
1388                        log.error("can not find windows info file")
1389                    sftp.close()
1390            else:
1391                return self.create_windows_info()
1392        else:
1393            try:
1394                txt = open("{0}/{1}".format("/cygdrive/c/tmp", "windows_info.txt"))
1395                log.info("get windows information")
1396                info = {}
1397                for line in txt.read():
1398                    (key, value) = line.split('=')
1399                    key = key.strip(' \t\n\r')
1400                    value = value.strip(' \t\n\r')
1401                    info[key] = value
1402                return info
1403            except IOError:
1404                log.error("can not find windows info file")
1405
1406    def create_windows_info(self):
1407            systeminfo = self.get_windows_system_info()
1408            info = {}
1409            info["os_name"] = "2k8"
1410            if "OS Name" in systeminfo:
1411                info["os"] = systeminfo["OS Name"].find("indows") and "windows" or "NONE"
1412            if systeminfo["OS Name"].find("2008 R2") != -1:
1413                info["os_name"] = 2008
1414            elif systeminfo["OS Name"].find("2016") != -1:
1415                info["os_name"] = 2016
1416            if "System Type" in systeminfo:
1417                info["os_arch"] = systeminfo["System Type"].find("64") and "x86_64" or "NONE"
1418            info.update(systeminfo)
1419            self.execute_batch_command("rm -rf  /cygdrive/c/tmp/windows_info.txt")
1420            self.execute_batch_command("touch  /cygdrive/c/tmp/windows_info.txt")
1421            sftp = self._ssh_client.open_sftp()
1422            try:
1423                f = sftp.open('/cygdrive/c/tmp/windows_info.txt', 'w')
1424                content = ''
1425                for key in sorted(info.keys()):
1426                    content += '{0} = {1}\n'.format(key, info[key])
1427                f.write(content)
1428                log.info("/cygdrive/c/tmp/windows_info.txt was created with content: {0}".format(content))
1429            except IOError:
1430                log.error('Can not write windows_info.txt file')
1431            finally:
1432                sftp.close()
1433            return info
1434
1435    # Need to add new windows register ID in testconstant file when
1436    # new couchbase server version comes out.
1437    def create_windows_capture_file(self, task, product, version):
1438        src_path = "resources/windows/automation"
1439        des_path = "/cygdrive/c/automation"
1440
1441        # remove dot in version (like 2.0.0 ==> 200)
1442        reg_version = version[0:5:2]
1443        reg_id = WIN_REGISTER_ID[reg_version]
1444        uuid_name = uuid.uuid4()
1445
1446        if task == "install":
1447            template_file = "cb-install.wct"
1448            file = "{0}_{1}_install.iss".format(uuid_name, self.ip)
1449            #file = "{0}_install.iss".format(self.ip)
1450        elif task == "uninstall":
1451            template_file = "cb-uninstall.wct"
1452            file = "{0}_{1}_uninstall.iss".format(uuid_name, self.ip)
1453            #file = "{0}_uninstall.iss".format(self.ip)
1454
1455        # create in/uninstall file from windows capture template (wct) file
1456        full_src_path_template = os.path.join(src_path, template_file)
1457        full_src_path = os.path.join(src_path, file)
1458        full_des_path = os.path.join(des_path, file)
1459
1460        f1 = open(full_src_path_template, 'r')
1461        f2 = open(full_src_path, 'w')
1462        """ replace ####### with reg ID to install/uninstall """
1463        if "2.2.0-837" in version:
1464            reg_id = "2B630EB8-BBC7-6FE4-C9B8-D8843EB1EFFA"
1465        log.info("register ID: {0}".format(reg_id))
1466        for line in f1:
1467            line = line.replace("#######", reg_id)
1468            if product == "mb" and task == "install":
1469                line = line.replace("Couchbase", "Membase")
1470            f2.write(line)
1471        f1.close()
1472        f2.close()
1473
1474        self.copy_file_local_to_remote(full_src_path, full_des_path)
1475        """ remove capture file from source after copy to destination """
1476        self.sleep(4, "wait for remote copy completed")
1477        """ need to implement to remove only at the end
1478            of installation """
1479        #os.remove(full_src_path)
1480        return uuid_name
1481
1482    def get_windows_system_info(self):
1483        try:
1484            info = {}
1485            o, _ = self.execute_batch_command('systeminfo')
1486            for line in o:
1487                line_list = line.split(':')
1488                if len(line_list) > 2:
1489                    if line_list[0] == 'Virtual Memory':
1490                        key = "".join(line_list[0:2])
1491                        value = " ".join(line_list[2:])
1492                    else:
1493                        key = line_list[0]
1494                        value = " ".join(line_list[1:])
1495                elif len(line_list) == 2:
1496                    (key, value) = line_list
1497                else:
1498                    continue
1499                key = key.strip(' \t\n\r')
1500                if key.find("[") != -1:
1501                    info[key_prev] += '|' + key + value.strip(' |')
1502                else:
1503                    value = value.strip(' |')
1504                    info[key] = value
1505                    key_prev = key
1506            return info
1507        except Exception as ex:
1508            log.error("error {0} appeared during getting  windows info".format(ex))
1509
1510    # this function used to modify bat file to run task schedule in windows
1511    def modify_bat_file(self, remote_path, file_name, name, version, task):
1512        found = self.find_file(remote_path, file_name)
1513        sftp = self._ssh_client.open_sftp()
1514        capture_iss_file = ""
1515
1516        product_version = ""
1517        if version[:5] in MEMBASE_VERSIONS:
1518            product_version = version[:5]
1519            name = "mb"
1520        elif version[:5] in COUCHBASE_VERSIONS:
1521            product_version = version[:5]
1522            name = "cb"
1523        else:
1524            log.error('Windows automation does not support {0} version yet'\
1525                                                               .format(version))
1526            sys.exit()
1527
1528        if "upgrade" not in task:
1529            uuid_name = self.create_windows_capture_file(task, name, version)
1530        try:
1531            f = sftp.open(found, 'w')
1532            name = name.strip()
1533            version = version.strip()
1534            if task == "upgrade":
1535                content = 'c:\\tmp\{3}.exe /s -f1c:\\automation\{0}_{1}_{2}.iss' \
1536                                .format(name, product_version, task, version)
1537            else:
1538                content = 'c:\\tmp\{0}.exe /s -f1c:\\automation\{3}_{2}_{1}.iss' \
1539                             .format(version, task, self.ip, uuid_name)
1540            log.info("create {0} task with content:{1}".format(task, content))
1541            f.write(content)
1542            log.info('Successful write to {0}'.format(found))
1543            if "upgrade" not in task:
1544                capture_iss_file = '{0}_{1}_{2}.iss'.format(uuid_name, self.ip, task)
1545        except IOError:
1546            log.error('Can not write build name file to bat file {0}'.format(found))
1547        sftp.close()
1548        return capture_iss_file
1549
1550    def compact_vbuckets(self, vbuckets, nodes, upto_seq, cbadmin_user="cbadminbucket",
1551                                                          cbadmin_password="password"):
1552        """
1553            compact each vbucket with cbcompact tools
1554        """
1555        for node in nodes:
1556            log.info("Purge 'deleted' keys in all %s vbuckets in node %s.  It will take times "
1557                                                                         % (vbuckets, node.ip))
1558            for vbucket in range(0, vbuckets):
1559                cmd = "%scbcompact%s %s:11210 compact %d -u %s -p %s "\
1560                      "--dropdeletes --purge-only-upto-seq=%d" \
1561                                                  % (self.bin_path,
1562                                                     self.cmd_ext,
1563                                                     node.ip, vbucket,
1564                                                     cbadmin_user,
1565                                                     cbadmin_password,
1566                                                     upto_seq)
1567                self.execute_command(cmd, debug=False)
1568        log.info("done compact")
1569
1570    def set_vbuckets_win(self, build, vbuckets):
1571        bin_path = WIN_COUCHBASE_BIN_PATH
1572        bin_path = bin_path.replace("\\", "")
1573        build = build.replace('-', '.')
1574        src_file = bin_path + "install/cb_winsvc_start_{0}.bat".format(build)
1575        des_file = "/tmp/cb_winsvc_start_{0}_{1}.bat".format(build, self.ip)
1576        local_file = "/tmp/cb_winsvc_start_{0}_{1}.bat_tmp".format(build, self.ip)
1577
1578        self.copy_file_remote_to_local(src_file, des_file)
1579        f1 = open(des_file, "r")
1580        f2 = open(local_file, "w")
1581        """ when install new cb server on windows, there is not
1582            env COUCHBASE_NUM_VBUCKETS yet.  We need to insert this
1583            env to service_register.bat right after  ERL_FULLSWEEP_AFTER 512
1584            like -env ERL_FULLSWEEP_AFTER 512 -env COUCHBASE_NUM_VBUCKETS vbuckets
1585            where vbucket is params passed to function when run install scripts """
1586        for line in f1:
1587            if "-env COUCHBASE_NUM_VBUCKETS " in line:
1588                tmp1 = line.split("COUCHBASE_NUM_VBUCKETS")
1589                tmp2 = tmp1[1].strip().split(" ")
1590                log.info("set vbuckets of node {0} to {1}" \
1591                                 .format(self.ip, vbuckets))
1592                tmp2[0] = vbuckets
1593                tmp1[1] = " ".join(tmp2)
1594                line = "COUCHBASE_NUM_VBUCKETS ".join(tmp1)
1595            elif "-env ERL_FULLSWEEP_AFTER 512" in line:
1596                log.info("set vbuckets of node {0} to {1}" \
1597                                 .format(self.ip, vbuckets))
1598                line = line.replace("-env ERL_FULLSWEEP_AFTER 512", \
1599                  "-env ERL_FULLSWEEP_AFTER 512 -env COUCHBASE_NUM_VBUCKETS {0}" \
1600                                 .format(vbuckets))
1601            f2.write(line)
1602        f1.close()
1603        f2.close()
1604        self.copy_file_local_to_remote(local_file, src_file)
1605
1606        """ re-register new setup to cb server """
1607        o, r = self.execute_command(WIN_COUCHBASE_BIN_PATH + "install/cb_winsvc_stop_{0}.bat".format(build))
1608        self.log_command_output(o, r)
1609        o, r = self.execute_command(WIN_COUCHBASE_BIN_PATH + "install/cb_winsvc_start_{0}.bat".format(build))
1610        self.log_command_output(o, r)
1611        self.sleep(20, "wait for cb server start completely after reset vbuckets!")
1612
1613        """ remove temporary files on slave """
1614        os.remove(local_file)
1615        os.remove(des_file)
1616
1617    def set_fts_query_limit_win(self, build, name, value, ipv6=False):
1618        bin_path = WIN_COUCHBASE_BIN_PATH
1619        bin_path = bin_path.replace("\\", "")
1620        build = build.replace('-', '.')
1621        src_file = bin_path + "install/cb_winsvc_start_{0}.bat".format(build)
1622        des_file = "/tmp/cb_winsvc_start_{0}_{1}.bat".format(build, self.ip)
1623        local_file = "/tmp/cb_winsvc_start_{0}_{1}.bat_tmp".format(build, self.ip)
1624
1625        self.copy_file_remote_to_local(src_file, des_file)
1626        f1 = open(des_file, "r")
1627        f2 = open(local_file, "w")
1628        """ when install new cb server on windows, there is not
1629            env CBFT_ENV_OPTIONS yet.  We need to insert this
1630            env to service_register.bat right after  ERL_FULLSWEEP_AFTER 512
1631            like -env ERL_FULLSWEEP_AFTER 512 -env CBFT_ENV_OPTIONS value
1632            where value is params passed to function when run install scripts """
1633        for line in f1:
1634            if "-env CBFT_ENV_OPTIONS " in line:
1635                tmp1 = line.split("CBFT_ENV_OPTIONS")
1636                tmp2 = tmp1[1].strip().split(" ")
1637                log.info("set CBFT_ENV_OPTIONS of node {0} to {1}" \
1638                                 .format(self.ip, value))
1639                tmp2[0] = value
1640                tmp1[1] = " ".join(tmp2)
1641                line = "CBFT_ENV_OPTIONS ".join(tmp1)
1642            elif "-env ERL_FULLSWEEP_AFTER 512" in line:
1643                log.info("set CBFT_ENV_OPTIONS of node {0} to {1}" \
1644                                 .format(self.ip, value))
1645                line = line.replace("-env ERL_FULLSWEEP_AFTER 512", \
1646                  "-env ERL_FULLSWEEP_AFTER 512 -env {0} {1}" \
1647                                 .format(name, value))
1648            f2.write(line)
1649        f1.close()
1650        f2.close()
1651        self.copy_file_local_to_remote(local_file, src_file)
1652
1653        """ re-register new setup to cb server """
1654        o, r = self.execute_command(WIN_COUCHBASE_BIN_PATH + "install/cb_winsvc_stop_{0}.bat ".format(build))
1655        self.log_command_output(o, r)
1656        cmd = WIN_COUCHBASE_BIN_PATH + "install/cb_winsvc_start_{0}.bat".format(build)
1657        if ipv6:
1658            cmd += " true"
1659        o, r = self.execute_command(cmd)
1660        self.log_command_output(o, r)
1661        self.sleep(20, "wait for cb server start completely after setting CBFT_ENV_OPTIONS")
1662
1663        """ remove temporary files on slave """
1664        os.remove(local_file)
1665        os.remove(des_file)
1666
1667    def create_directory(self, remote_path):
1668        sftp = self._ssh_client.open_sftp()
1669        try:
1670            sftp.stat(remote_path)
1671        except IOError, e:
1672            if e[0] == 2:
1673                log.info("Directory at {0} DOES NOT exist. We will create on here".format(remote_path))
1674                sftp.mkdir(remote_path)
1675                sftp.close()
1676                return False
1677            raise
1678        else:
1679            log.error("Directory at {0} DOES exist. Fx returns True".format(remote_path))
1680            return True
1681
1682    # this function will remove the automation directory in windows
1683    def create_multiple_dir(self, dir_paths):
1684        sftp = self._ssh_client.open_sftp()
1685        try:
1686            for dir_path in dir_paths:
1687                if dir_path != '/cygdrive/c/tmp':
1688                    output = self.remove_directory('/cygdrive/c/automation')
1689                    if output:
1690                        log.info("{0} directory is removed.".format(dir_path))
1691                    else:
1692                        log.error("Can not delete {0} directory or directory {0} does not exist.".format(dir_path))
1693                self.create_directory(dir_path)
1694            sftp.close()
1695        except IOError:
1696            pass
1697
1698    def couchbase_upgrade(self, build, save_upgrade_config=False, forcefully=False,
1699                                                             fts_query_limit=None):
1700        server_type = None
1701        success = True
1702        if fts_query_limit is None:
1703            fts_query_limit = 10000000
1704        start_server_after_install = True
1705        track_words = ("warning", "error", "fail")
1706        if build.name.lower().find("membase") != -1:
1707            server_type = 'membase'
1708            abbr_product = "mb"
1709        elif build.name.lower().find("couchbase") != -1:
1710            server_type = 'couchbase'
1711            abbr_product = "cb"
1712        else:
1713            raise Exception("its not a membase or couchbase?")
1714        # upgrade couchbase server
1715        nonroot_path_start = "/"
1716        self.extract_remote_info()
1717        log.info('deliverable_type : {0}'.format(self.info.deliverable_type))
1718        log.info('/tmp/{0} or /tmp/{1}'.format(build.name, build.product))
1719        command = ''
1720        if self.info.type.lower() == 'windows':
1721                print "build name in couchbase upgrade    ", build.product_version
1722                self.couchbase_upgrade_win(self.info.architecture_type, \
1723                                     self.info.windows_name, build.product_version)
1724                log.info('********* continue upgrade process **********')
1725
1726        elif self.info.deliverable_type == 'rpm':
1727            # run rpm -i to install
1728            if save_upgrade_config:
1729                self.couchbase_uninstall()
1730                install_command = 'rpm -i /tmp/{0}'.format(build.name)
1731                command = 'INSTALL_UPGRADE_CONFIG_DIR=/opt/couchbase/var/lib/membase/config {0}'\
1732                                             .format(install_command)
1733            else:
1734                command = 'rpm -U /tmp/{0}'.format(build.name)
1735                if forcefully:
1736                    command = 'rpm -U --force /tmp/{0}'.format(build.name)
1737        elif self.info.deliverable_type == 'deb':
1738            if save_upgrade_config:
1739                self.couchbase_uninstall()
1740                install_command = 'dpkg -i /tmp/{0}'.format(build.name)
1741                command = 'INSTALL_UPGRADE_CONFIG_DIR=/opt/couchbase/var/lib/membase/config {0}'\
1742                                             .format(install_command)
1743            else:
1744                command = 'dpkg -i /tmp/{0}'.format(build.name)
1745                if forcefully:
1746                    command = 'dpkg -i --force /tmp/{0}'.format(build.name)
1747        output, error = self.execute_command(command, use_channel=True)
1748        self.log_command_output(output, error)
1749        linux = ["deb", "rpm"]
1750        if float(build.product_version[:3]) >= 5.1:
1751            if self.info.deliverable_type in linux:
1752                o, e = \
1753                    self.execute_command("sed -i 's/export PATH/export PATH\\n"
1754                            "export CBFT_ENV_OPTIONS=bleveMaxResultWindow={1},hideUI=false/'\
1755                            {2}opt/{0}/bin/{0}-server".format(server_type, int(fts_query_limit),
1756                                                              nonroot_path_start))
1757                success &= self.log_command_output(o, e, track_words)
1758                self.stop_couchbase()
1759                self.start_couchbase()
1760        return output, error
1761
1762    def couchbase_upgrade_win(self, architecture, windows_name, version):
1763        task = "upgrade"
1764        bat_file = "upgrade.bat"
1765        deleted = False
1766        self.modify_bat_file('/cygdrive/c/automation', bat_file, 'cb',\
1767                                                         version, task)
1768        self.stop_schedule_tasks()
1769        self.remove_win_backup_dir()
1770        self.remove_win_collect_tmp()
1771        output, error = self.execute_command("cat "
1772                        "'/cygdrive/c/Program Files/Couchbase/Server/VERSION.txt'")
1773        log.info("version to upgrade from: {0} to {1}".format(output, version))
1774        log.info('before running task schedule upgrademe')
1775        if '1.8.0' in str(output):
1776            """ run installer in second time as workaround for upgrade 1.8.0 only:
1777            #   Installer needs to update registry value in order to upgrade
1778            #   from the previous version.
1779            #   Please run installer again to continue."""
1780            output, error = \
1781                        self.execute_command("cmd /c schtasks /run /tn upgrademe")
1782            self.log_command_output(output, error)
1783            self.sleep(200, "because upgrade version is {0}".format(output))
1784            output, error = self.execute_command("cmd /c "
1785                                      "schtasks /Query /FO LIST /TN upgrademe /V")
1786            self.log_command_output(output, error)
1787            self.stop_schedule_tasks()
1788        # run task schedule to upgrade Membase server
1789        output, error = self.execute_command("cmd /c schtasks /run /tn upgrademe")
1790        self.log_command_output(output, error)
1791        deleted = self.wait_till_file_deleted(WIN_CB_PATH, \
1792                       VERSION_FILE, timeout_in_seconds=600)
1793        if not deleted:
1794            log.error("Uninstall was failed at node {0}".format(self.ip))
1795            sys.exit()
1796        self.wait_till_file_added(WIN_CB_PATH, VERSION_FILE, \
1797                                       timeout_in_seconds=600)
1798        log.info("installed version: {0}".format(version))
1799        output, error = self.execute_command("cat "
1800                       "'/cygdrive/c/Program Files/Couchbase/Server/VERSION.txt'")
1801        ended = self.wait_till_process_ended(version[:10])
1802        if not ended:
1803            sys.exit("*****  Node %s failed to upgrade  *****" % (self.ip))
1804        self.sleep(10, "wait for server to start up completely")
1805        ct = time.time()
1806        while time.time() - ct < 10800:
1807            output, error = self.execute_command("cmd /c "
1808                      "schtasks /Query /FO LIST /TN upgrademe /V| findstr Status ")
1809            if "Ready" in str(output):
1810                log.info("upgrademe task complteted")
1811                break
1812            elif "Could not start":
1813                log.exception("Ugrade failed!!!")
1814                break
1815            else:
1816                log.info("upgrademe task still running:{0}".format(output))
1817                self.sleep(30)
1818        output, error = self.execute_command("cmd /c "
1819                                       "schtasks /Query /FO LIST /TN upgrademe /V")
1820        self.log_command_output(output, error)
1821        """ need to remove binary after done upgrade.  Since watson, current binary
1822            could not reused to uninstall or install cb server """
1823        self.delete_file(WIN_TMP_PATH, version[:10] + ".exe")
1824
1825    """
1826        This method install Couchbase Server
1827    """
1828    def install_server(self, build, startserver=True, path='/tmp', vbuckets=None,
1829                       swappiness=10, force=False, openssl='', upr=None, xdcr_upr=None,
1830                       fts_query_limit=None, enable_ipv6=None):
1831
1832        log.info('*****install server ***')
1833        server_type = None
1834        success = True
1835        start_server_after_install = True
1836        track_words = ("warning", "error", "fail")
1837        if build.name.lower().find("membase") != -1:
1838            server_type = 'membase'
1839            abbr_product = "mb"
1840        elif build.name.lower().find("couchbase") != -1:
1841            server_type = 'couchbase'
1842            abbr_product = "cb"
1843        else:
1844            raise Exception("its not a membase or couchbase?")
1845        self.extract_remote_info()
1846
1847        log.info('deliverable_type : {0}'.format(self.info.deliverable_type))
1848        if self.info.type.lower() == 'windows':
1849            log.info('***** Doing the windows install')
1850            self.terminate_processes(self.info, [s for s in WIN_PROCESSES_KILLED])
1851            self.terminate_processes(self.info, \
1852                                     [s + "-*" for s in COUCHBASE_FROM_VERSION_3])
1853            # to prevent getting full disk let's delete some large files
1854            self.remove_win_backup_dir()
1855            self.remove_win_collect_tmp()
1856            output, error = self.execute_command("cmd /c schtasks /run /tn installme")
1857            success &= self.log_command_output(output, error, track_words)
1858            file_check = 'VERSION.txt'
1859            self.wait_till_file_added("/cygdrive/c/Program Files/{0}/Server/".format(server_type.title()), file_check,
1860                                          timeout_in_seconds=600)
1861            output, error = self.execute_command("cmd /c schtasks /Query /FO LIST /TN installme /V")
1862            self.log_command_output(output, error)
1863            ended = self.wait_till_process_ended(build.product_version[:10])
1864            if not ended:
1865                sys.exit("*****  Node %s failed to install  *****" % (self.ip))
1866            self.sleep(10, "wait for server to start up completely")
1867            if vbuckets and int(vbuckets) != 1024:
1868                self.set_vbuckets_win(build.version_number, vbuckets)
1869            if fts_query_limit:
1870                self.set_environment_variable(
1871                    name="CBFT_ENV_OPTIONS",
1872                    value="bleveMaxResultWindow={0}".format(int(fts_query_limit))
1873                )
1874
1875            output, error = self.execute_command("rm -f \
1876                       /cygdrive/c/automation/*_{0}_install.iss".format(self.ip))
1877            self.log_command_output(output, error)
1878            self.delete_file(WIN_TMP_PATH, build.product_version[:10] + ".exe")
1879            # output, error = self.execute_command("cmd rm /cygdrive/c/tmp/{0}*.exe".format(build_name))
1880            # self.log_command_output(output, error)
1881        elif self.info.deliverable_type in ["rpm", "deb"]:
1882            if startserver and vbuckets == None:
1883                environment = ""
1884            else:
1885                environment = "INSTALL_DONT_START_SERVER=1 "
1886                start_server_after_install = False
1887            log.info('/tmp/{0} or /tmp/{1}'.format(build.name, build.product))
1888
1889            # set default swappiness to 10 unless specify in params in all unix environment
1890            if self.nonroot:
1891                log.info("**** use root to run script/ssh.py to execute /sbin/sysctl vm.swappiness=0 "
1892                              "enable coredump cbenable_core_dumps.sh /tmp")
1893            else:
1894                output, error = self.execute_command('/sbin/sysctl vm.swappiness={0}'.format(swappiness))
1895                success &= self.log_command_output(output, error, track_words, debug=False)
1896
1897            if self.info.deliverable_type == 'rpm':
1898                if self.nonroot:
1899                    op, er = self.execute_command('cd {0}; rpm2cpio {1} ' \
1900                        '|  cpio --extract --make-directories --no-absolute-filenames ' \
1901                                     .format(self.nr_home_path, build.name), debug=False)
1902                    self.log_command_output(op, er)
1903                    output, error = self.execute_command('cd {0}{1}; ./bin/install/reloc.sh `pwd` '\
1904                                            .format(self.nr_home_path, LINUX_CB_PATH), debug=False)
1905                    self.log_command_output(output, error)
1906                    op, er = self.execute_command('cd {0};pwd'.format(self.nr_home_path))
1907                    self.log_command_output(op, er)
1908                    """ command to start Couchbase Server in non root
1909                        /home/nonroot_user/opt/couchbase/bin/couchbase-server \-- -noinput -detached
1910                    """
1911                    output, error = self.execute_command("ls -lh ")
1912                    self.log_command_output(output, error)
1913                    if start_server_after_install:
1914                        output, error = self.execute_command('%s%scouchbase-server '\
1915                                                             '\-- -noinput -detached '\
1916                                                              % (self.nr_home_path,
1917                                                                 LINUX_COUCHBASE_BIN_PATH))
1918                else:
1919                    self.check_pkgconfig(self.info.deliverable_type, openssl)
1920                    if force:
1921                        output, error = self.execute_command('{0}rpm -Uvh --force /tmp/{1}'\
1922                                                .format(environment, build.name), debug=False)
1923                    else:
1924                        output, error = self.execute_command('{0}rpm -i /tmp/{1}'\
1925                                                .format(environment, build.name), debug=False)
1926            elif self.info.deliverable_type == 'deb':
1927                if self.nonroot:
1928                    op, er = self.execute_command('cd %s; dpkg-deb -x %s %s '
1929                                                % (self.nr_home_path, build.name,
1930                                                   self.nr_home_path))
1931                    self.log_command_output(op, er)
1932                    output, error = self.execute_command('cd {0}{1}; ./bin/install/reloc.sh `pwd`'\
1933                                                        .format(self.nr_home_path, LINUX_CB_PATH))
1934                    self.log_command_output(output, error)
1935                    op, er = self.execute_command('pwd')
1936                    self.log_command_output(op, er)
1937                    """ command to start Couchbase Server in non root in ubuntu the same
1938                        as in centos above
1939                    """
1940                    if start_server_after_install:
1941                        output, error = self.execute_command('%s%scouchbase-server '\
1942                                                             '\-- -noinput -detached '\
1943                                                               % (self.nr_home_path,
1944                                                                  LINUX_COUCHBASE_BIN_PATH))
1945                else:
1946                    self.install_missing_lib()
1947                    if force:
1948                        output, error = self.execute_command('{0}dpkg --force-all -i /tmp/{1}'\
1949                                                 .format(environment, build.name), debug=False)
1950                    else:
1951                        output, error = self.execute_command('{0}dpkg -i /tmp/{1}'\
1952                                                 .format(environment, build.name), debug=False)
1953
1954            if "SUSE" in self.info.distribution_type:
1955                if error and error[0] == 'insserv: Service network is missed in the runlevels 2'\
1956                                         ' 4 to use service couchbase-server':
1957                        log.info("Ignore this error for opensuse os")
1958                        error = []
1959            if output:
1960                server_ip = "\n\n**** Installing on server: {0} ****".format(self.ip)
1961                output.insert(0, server_ip)
1962            if error:
1963                server_ip = "\n\n**** Installing on server: {0} ****".format(self.ip)
1964                error.insert(0, server_ip)
1965            success &= self.log_command_output(output, error, track_words, debug=False)
1966            nonroot_path_start = ""
1967            if not self.nonroot:
1968                nonroot_path_start = "/"
1969                self.create_directory(path)
1970                output, error = self.execute_command('/opt/{0}/bin/{1}enable_core_dumps.sh  {2}'.
1971                                            format(server_type, abbr_product, path), debug=False)
1972                success &= self.log_command_output(output, error, track_words, debug=False)
1973
1974            if vbuckets:
1975                """
1976                   From spock, the file to edit is in /opt/couchbase/bin/couchbase-server
1977                """
1978                output, error = self.execute_command("sed -i 's/export ERL_FULLSWEEP_AFTER/"
1979                                                     "export ERL_FULLSWEEP_AFTER\\n"
1980                                                     "{0}_NUM_VBUCKETS={1}\\n"
1981                                                     "export {0}_NUM_VBUCKETS/' "
1982                                                     "{3}opt/{2}/bin/{2}-server".
1983                                                     format(server_type.upper(), vbuckets,
1984                                            server_type, nonroot_path_start), debug=False)
1985                success &= self.log_command_output(output, error, track_words)
1986            if upr is not None:
1987                protocol = "tap"
1988                if upr:
1989                    protocol = "upr"
1990                output, error = \
1991                self.execute_command("sed -i 's/END INIT INFO/END INIT INFO\\nexport"
1992                              " COUCHBASE_REPL_TYPE={1}/' {2}opt/{0}/etc/{0}_init.d"\
1993                                                      .format(server_type, protocol,
1994                                                              nonroot_path_start))
1995                success &= self.log_command_output(output, error, track_words)
1996            if xdcr_upr == False:
1997                output, error = \
1998                    self.execute_command("sed -i 's/ulimit -c unlimited/ulimit "
1999                             "-c unlimited\\n    export XDCR_USE_OLD_PATH={1}/'"
2000                                                     "{2}opt/{0}/etc/{0}_init.d"\
2001                                                    .format(server_type, "true",
2002                                                            nonroot_path_start))
2003                success &= self.log_command_output(output, error, track_words)
2004            if fts_query_limit:
2005                output, error = \
2006                    self.execute_command("sed -i 's/export PATH/export PATH\\n"
2007                            "export CBFT_ENV_OPTIONS=bleveMaxResultWindow={1},hideUI=false/'\
2008                            {2}opt/{0}/bin/{0}-server".format(server_type, int(fts_query_limit),
2009                                                              nonroot_path_start))
2010                success &= self.log_command_output(output, error, track_words)
2011                startserver = True
2012
2013            if enable_ipv6:
2014                output, error = \
2015                    self.execute_command("sed -i '/ipv6, /c \\{ipv6, true\}'. %s"
2016                        % testconstants.LINUX_STATIC_CONFIG)
2017                success &= self.log_command_output(output, error, track_words)
2018                startserver = True
2019
2020            # skip output: [WARNING] couchbase-server is already started
2021            # dirname error skipping for CentOS-6.6 (MB-12536)
2022            track_words = ("error", "fail", "dirname")
2023            if startserver:
2024                self.start_couchbase()
2025                if (build.product_version.startswith("1.") \
2026                        or build.product_version.startswith("2.0.0"))\
2027                        and build.deliverable_type == "deb":
2028                    # skip error '* Failed to start couchbase-server' for 1.* & 2.0.0 builds(MB-7288)
2029                    # fix in 2.0.1 branch Change-Id: I850ad9424e295bbbb79ede701495b018b5dfbd51
2030                    log.warn("Error '* Failed to start couchbase-server' for 1.* "
2031                                                          "builds will be skipped")
2032                    self.log_command_output(output, error, track_words)
2033                else:
2034                    success &= self.log_command_output(output, error, track_words, debug=False)
2035        elif self.info.deliverable_type in ["zip"]:
2036            """ close Safari browser before install """
2037            self.terminate_process(self.info, "/Applications/Safari.app/Contents/MacOS/Safari")
2038            o, r = self.execute_command("ps aux | grep Archive | awk '{print $2}' | xargs kill -9")
2039            self.sleep(20)
2040            output, error = self.execute_command("cd ~/Downloads ; open couchbase-server*.zip")
2041            self.log_command_output(output, error)
2042            self.sleep(20)
2043            cmd1 = "mv ~/Downloads/couchbase-server*/Couchbase\ Server.app /Applications/"
2044            cmd2 = "sudo xattr -d -r com.apple.quarantine /Applications/Couchbase\ Server.app"
2045            cmd3 = "open /Applications/Couchbase\ Server.app"
2046            output, error = self.execute_command(cmd1)
2047            self.log_command_output(output, error)
2048            output, error = self.execute_command(cmd2)
2049            self.log_command_output(output, error)
2050            output, error = self.execute_command(cmd3)
2051            self.log_command_output(output, error)
2052
2053        output, error = self.execute_command("rm -f *-diag.zip", debug=False)
2054        self.log_command_output(output, error, track_words, debug=False)
2055        return success
2056
2057    def install_server_win(self, build, version, startserver=True,
2058                           vbuckets=None, fts_query_limit=None,
2059                           enable_ipv6=None, windows_msi=False):
2060
2061
2062        log.info('******start install_server_win ********')
2063        if windows_msi:
2064            if enable_ipv6:
2065                output, error = self.execute_command("cd /cygdrive/c/tmp;"
2066                                                 "msiexec /i {0}.msi USE_IPV6=true /qn "\
2067                                                    .format(version))
2068            else:
2069                output, error = self.execute_command("cd /cygdrive/c/tmp;"
2070                                                     "msiexec /i {0}.msi /qn " \
2071                                                     .format(version))
2072            self.log_command_output(output, error)
2073            if fts_query_limit:
2074                self.set_fts_query_limit_win(
2075                    build = version,
2076                    name="CBFT_ENV_OPTIONS",
2077                    value="bleveMaxResultWindow={0}".format(int(fts_query_limit)),
2078                    ipv6=enable_ipv6
2079                )
2080            return len(error) == 0
2081        remote_path = None
2082        success = True
2083        track_words = ("warning", "error", "fail")
2084        if build.name.lower().find("membase") != -1:
2085            remote_path = testconstants.WIN_MB_PATH
2086            abbr_product = "mb"
2087        elif build.name.lower().find("couchbase") != -1:
2088            remote_path = WIN_CB_PATH
2089            abbr_product = "cb"
2090
2091        if remote_path is None:
2092            raise Exception("its not a membase or couchbase?")
2093        self.extract_remote_info()
2094        log.info('deliverable_type : {0}'.format(self.info.deliverable_type))
2095        if self.info.type.lower() == 'windows':
2096            task = "install"
2097            bat_file = "install.bat"
2098            dir_paths = ['/cygdrive/c/automation', '/cygdrive/c/tmp']
2099            capture_iss_file = ""
2100            # build = self.build_url(params)
2101            self.create_multiple_dir(dir_paths)
2102            self.copy_files_local_to_remote('resources/windows/automation', \
2103                                                    '/cygdrive/c/automation')
2104            #self.create_windows_capture_file(task, abbr_product, version)
2105            capture_iss_file = self.modify_bat_file('/cygdrive/c/automation', \
2106                                             bat_file, abbr_product, version, task)
2107            self.stop_schedule_tasks()
2108            self.remove_win_backup_dir()
2109            self.remove_win_collect_tmp()
2110            log.info('sleep for 5 seconds before running task '
2111                                'schedule uninstall on {0}'.format(self.ip))
2112
2113            """ the code below need to remove when bug MB-11985
2114                                                            is fixed in 3.0.1 """
2115            if task == "install" and (version[:5] in COUCHBASE_VERSION_2 or \
2116                                      version[:5] in COUCHBASE_FROM_VERSION_3):
2117                log.info("due to bug MB-11985, we need to delete below registry "
2118                                        "before install version 2.x.x and 3.x.x")
2119                output, error = self.execute_command("reg delete \
2120                         'HKLM\Software\Wow6432Node\Ericsson\Erlang\ErlSrv' /f ")
2121                self.log_command_output(output, error)
2122            """ end remove code """
2123
2124            """ run task schedule to install cb server """
2125            output, error = self.execute_command("cmd /c schtasks /run /tn installme")
2126            success &= self.log_command_output(output, error, track_words)
2127            file_check = 'VERSION.txt'
2128            self.wait_till_file_added(remote_path, file_check, timeout_in_seconds=600)
2129            if version[:3] != "2.5":
2130                ended = self.wait_till_process_ended(build.product_version[:10])
2131                if not ended:
2132                    sys.exit("*****  Node %s failed to install  *****" % (self.ip))
2133            if version[:3] == "2.5":
2134                self.sleep(20, "wait for server to start up completely")
2135            else:
2136                self.sleep(10, "wait for server to start up completely")
2137            output, error = self.execute_command("rm -f *-diag.zip")
2138            self.log_command_output(output, error, track_words)
2139            if vbuckets and int(vbuckets) != 1024:
2140                self.set_vbuckets_win(build.version_number, vbuckets)
2141            if fts_query_limit:
2142                self.set_fts_query_limit_win(
2143                    build = version,
2144                    name="CBFT_ENV_OPTIONS",
2145                    value="bleveMaxResultWindow={0}".format(int(fts_query_limit)),
2146                    ipv6=enable_ipv6
2147                )
2148
2149            if "4.0" in version[:5]:
2150                """  remove folder if it exists in work around of bub MB-13046 """
2151                self.execute_command("rm -rf \
2152                /cygdrive/c/Jenkins/workspace/sherlock-windows/couchbase/install/etc/security")
2153                """ end remove code for bug MB-13046 """
2154            if capture_iss_file:
2155                    log.info("****Delete {0} in windows automation directory" \
2156                                                          .format(capture_iss_file))
2157                    output, error = self.execute_command("rm -f \
2158                               /cygdrive/c/automation/{0}".format(capture_iss_file))
2159                    self.log_command_output(output, error)
2160                    log.info("Delete {0} in slave resources/windows/automation dir" \
2161                             .format(capture_iss_file))
2162                    os.system("rm -f resources/windows/automation/{0}" \
2163                                                          .format(capture_iss_file))
2164            self.delete_file(WIN_TMP_PATH, build.product_version[:10] + ".exe")
2165            log.info('***** done install_server_win *****')
2166            return success
2167
2168
2169    def install_server_via_repo(self, deliverable_type, cb_edition, remote_client):
2170        success = True
2171        track_words = ("warning", "error", "fail")
2172        if cb_edition:
2173            cb_edition = "-" + cb_edition
2174        if deliverable_type == "deb":
2175            self.update_couchbase_release(remote_client, deliverable_type)
2176            output, error = self.execute_command("yes \
2177                   | apt-get install couchbase-server{0}".format(cb_edition))
2178            self.log_command_output(output, error)
2179            success &= self.log_command_output(output, error, track_words)
2180        elif deliverable_type == "rpm":
2181            self.update_couchbase_release(remote_client, deliverable_type)
2182            output, error = self.execute_command("yes \
2183                  | yum install couchbase-server{0}".format(cb_edition))
2184            self.log_command_output(output, error, track_words)
2185            success &= self.log_command_output(output, error, track_words)
2186        return success
2187
2188    def update_couchbase_release(self, remote_client, deliverable_type):
2189        if deliverable_type == "deb":
2190            """ remove old couchbase-release package """
2191            log.info("remove couchbase-release at node {0}".format(self.ip))
2192            output, error = self.execute_command("dpkg --get-selections |\
2193                                                              grep couchbase")
2194            self.log_command_output(output, error)
2195            for str in output:
2196                if "couchbase-release" in str:
2197                    output, error = self.execute_command("apt-get \
2198                                                   purge -y couchbase-release")
2199            output, error = self.execute_command("dpkg --get-selections |\
2200                                                              grep couchbase")
2201            self.log_command_output(output, error)
2202            package_remove = True
2203            for str in output:
2204                if "couchbase-release" in str:
2205                    package_remove = False
2206                    log.info("couchbase-release is not removed at node {0}" \
2207                                                                  .format(self.ip))
2208                    sys.exit("***  Node %s failed to remove couchbase-release  ***"\
2209                                                                        % (self.ip))
2210            """ install new couchbase-release package """
2211            log.info("install new couchbase-release repo at node {0}" \
2212                                                             .format(self.ip))
2213            self.execute_command("rm -rf /tmp/couchbase-release*")
2214            self.execute_command("cd /tmp; wget {0}".format(CB_RELEASE_APT_GET_REPO))
2215            output, error = self.execute_command("yes | dpkg -i /tmp/couchbase-release*")
2216            self.log_command_output(output, error)
2217            output, error = self.execute_command("dpkg --get-selections |\
2218                                                              grep couchbase")
2219            package_updated = False
2220            for str in output:
2221                if "couchbase-release" in str:
2222                    package_updated = True
2223                    log.info("couchbase-release installed on node {0}" \
2224                                               .format(self.ip))
2225                    return package_updated
2226            if not package_updated:
2227                sys.exit("fail to install %s on node %s" % \
2228                                  (CB_RELEASE_APT_GET_REPO.rsplit("/",1)[-1], self.ip))
2229        elif deliverable_type == "rpm":
2230            """ remove old couchbase-release package """
2231            log.info("remove couchbase-release at node {0}".format(self.ip))
2232            output, error = self.execute_command("rpm -qa | grep couchbase")
2233            self.log_command_output(output, error)
2234            for str in output:
2235                if "couchbase-release" in str:
2236                    output, error = self.execute_command("rpm -e couchbase-release")
2237            output, error = self.execute_command("rpm -qa | grep couchbase")
2238            self.log_command_output(output, error)
2239            package_remove = True
2240            for str in output:
2241                if "couchbase-release" in str:
2242                    package_remove = False
2243                    log.info("couchbase-release is not removed at node {0}" \
2244                                                                  .format(self.ip))
2245                    sys.exit("***  Node %s failed to remove couchbase-release  ***"\
2246                                                                        % (self.ip))
2247            """ install new couchbase-release package """
2248            log.info("install new couchbase-release repo at node {0}" \
2249                                                             .format(self.ip))
2250            self.execute_command("rm -rf /tmp/couchbase-release*")
2251            self.execute_command("cd /tmp; wget {0}".format(CB_RELEASE_YUM_REPO))
2252            output, error = self.execute_command("yes | rpm -i /tmp/couchbase-release*")
2253            self.log_command_output(output, error)
2254            output, error = self.execute_command("rpm -qa | grep couchbase")
2255            package_updated = False
2256            for str in output:
2257                if "couchbase-release" in str:
2258                    package_updated = True
2259                    log.info("couchbase-release installed on node {0}" \
2260                                               .format(self.ip))
2261                    return package_updated
2262            if not package_updated:
2263                sys.exit("fail to install %s on node %s" % \
2264                                  (CB_RELEASE_YUM_REPO.rsplit("/",1)[-1], self.ip))
2265
2266    def install_moxi(self, build):
2267        success = True
2268        track_words = ("warning", "error", "fail")
2269        self.extract_remote_info()
2270        log.info('deliverable_type : {0}'.format(self.info.deliverable_type))
2271        if self.info.type.lower() == 'windows':
2272            log.error('Not implemented')
2273        elif self.info.deliverable_type in ["rpm"]:
2274            output, error = self.execute_command('rpm -i /tmp/{0}'.format(build.name))
2275            if error and ' '.join(error).find("ERROR") != -1:
2276                success = False
2277        elif self.info.deliverable_type == 'deb':
2278            output, error = self.execute_command('dpkg -i /tmp/{0}'.format(build.name))
2279            if error and ' '.join(error).find("ERROR") != -1:
2280                success = False
2281        success &= self.log_command_output(output, '', track_words)
2282        return success
2283
2284    def wait_till_file_deleted(self, remotepath, filename, timeout_in_seconds=180):
2285        end_time = time.time() + float(timeout_in_seconds)
2286        deleted = False
2287        log.info("file {0} checked at {1}".format(filename, remotepath))
2288        while time.time() < end_time and not deleted:
2289            # get the process list
2290            exists = self.file_exists(remotepath, filename)
2291            if exists:
2292                log.error('at {2} file {1} still exists' \
2293                          .format(remotepath, filename, self.ip))
2294                time.sleep(10)
2295            else:
2296                log.info('at {2} FILE {1} DOES NOT EXIST ANYMORE!' \
2297                         .format(remotepath, filename, self.ip))
2298                deleted = True
2299        return deleted
2300
2301    def wait_till_file_added(self, remotepath, filename, timeout_in_seconds=180):
2302        end_time = time.time() + float(timeout_in_seconds)
2303        added = False
2304        log.info("file {0} checked at {1}".format(filename, remotepath))
2305        while time.time() < end_time and not added:
2306            # get the process list
2307            exists = self.file_exists(remotepath, filename)
2308            if not exists:
2309                log.error('at {2} file {1} does not exist' \
2310                          .format(remotepath, filename, self.ip))
2311                time.sleep(10)
2312            else:
2313                log.info('at {2} FILE {1} EXISTS!' \
2314                         .format(remotepath, filename, self.ip))
2315                added = True
2316        return added
2317
2318
2319
2320
2321    def wait_till_compaction_end(self, rest, bucket, timeout_in_seconds=60):
2322        end_time = time.time() + float(timeout_in_seconds)
2323
2324        while time.time() < end_time:
2325            status, progress = rest.check_compaction_status(bucket)
2326            if status:
2327                log.info("compaction progress is %s" % progress)
2328                time.sleep(1)
2329            else:
2330                # the compaction task has completed
2331                return True
2332
2333        log.error("auto compaction has not ended in {0} sec.".format(str(timeout_in_seconds)))
2334        return False
2335
2336    def wait_till_process_ended(self, process_name, timeout_in_seconds=600):
2337        if process_name[-1:] == "-":
2338            process_name = process_name[:-1]
2339        end_time = time.time() + float(timeout_in_seconds)
2340        process_ended = False
2341        process_running = False
2342        count_process_not_run = 0
2343        while time.time() < end_time and not process_ended:
2344            output, error = self.execute_command("tasklist | grep {0}" \
2345                                                 .format(process_name))
2346            self.log_command_output(output, error)
2347            if output and process_name in output[0]:
2348                self.sleep(8, "wait for process ended!")
2349                process_running = True
2350            else:
2351                if process_running:
2352                    log.info("{1}: Alright, PROCESS {0} ENDED!" \
2353                             .format(process_name, self.ip))
2354                    process_ended = True
2355                else:
2356                    if count_process_not_run < 5:
2357                        log.error("{1}: process {0} may not run" \
2358                              .format(process_name, self.ip))
2359                        self.sleep(5)
2360                        count_process_not_run += 1
2361                    else:
2362                        log.error("{1}: process {0} did not run after 25 seconds"
2363                                                   .format(process_name, self.ip))
2364                        mesg = "kill in/uninstall job due to process was not run" \
2365                                                     .format(process_name, self.ip)
2366                        self.stop_current_python_running(mesg)
2367        if time.time() >= end_time and not process_ended:
2368            log.info("Process {0} on node {1} is still running"
2369                     " after 10 minutes VERSION.txt file was removed"
2370                                      .format(process_name, self.ip))
2371        return process_ended
2372
2373    def terminate_processes(self, info, list):
2374        for process in list:
2375            type = info.distribution_type.lower()
2376            if type == "windows":
2377                # set debug=False if does not want to show log
2378                self.execute_command("taskkill /F /T /IM {0}".format(process),
2379                                                                  debug=False)
2380            elif type in LINUX_DISTRIBUTION_NAME:
2381                self.terminate_process(info, process, force=True)
2382
2383    def remove_folders(self, list):
2384        for folder in list:
2385            output, error = self.execute_command("rm -rf {0}".format(folder),
2386                                                                debug=False)
2387            self.log_command_output(output, error)
2388
2389    def couchbase_uninstall(self, windows_msi=False, product=None):
2390        log.info('{0} *****In couchbase uninstall****'.format( self.ip))
2391        linux_folders = ["/var/opt/membase", "/opt/membase", "/etc/opt/membase",
2392                         "/var/membase/data/*", "/opt/membase/var/lib/membase/*",
2393                         "/opt/couchbase", "/data/*"]
2394        terminate_process_list = ["beam.smp", "memcached", "moxi", "vbucketmigrator",
2395                                  "couchdb", "epmd", "memsup", "cpu_sup", "goxdcr", "erlang", "eventing"]
2396        self.extract_remote_info()
2397        log.info(self.info.distribution_type)
2398        type = self.info.distribution_type.lower()
2399        fv, sv, bn = self.get_cbversion(type)
2400        if type == 'windows':
2401            product = "cb"
2402            query = BuildQuery()
2403            os_type = "exe"
2404            if windows_msi:
2405                os_type = "msi"
2406                self.info.deliverable_type = "msi"
2407            task = "uninstall"
2408            bat_file = "uninstall.bat"
2409            product_name = "couchbase-server-enterprise"
2410            version_path = "/cygdrive/c/Program Files/Couchbase/Server/"
2411            deleted = False
2412            capture_iss_file = ""
2413            log.info("kill any in/uninstall process from version 3 in node %s"
2414                                                                        % self.ip)
2415            self.terminate_processes(self.info, \
2416                                     [s + "-*" for s in COUCHBASE_FROM_VERSION_3])
2417            exist = self.file_exists(version_path, VERSION_FILE)
2418            log.info("Is VERSION file existed on {0}? {1}".format(self.ip, exist))
2419            if exist:
2420                cb_releases_version = []
2421                for x in CB_RELEASE_BUILDS:
2422                    if int(CB_RELEASE_BUILDS[x]):
2423                        cb_releases_version.append(x)
2424
2425                build_name, short_version, full_version = \
2426                    self.find_build_version(version_path, VERSION_FILE, product)
2427
2428                if "-" in full_version:
2429                    msi_build = full_version.split("-")
2430                    """
2431                       In spock from build 2924 and later release, we only support
2432                       msi installation method on windows
2433                    """
2434                    if msi_build[0] in COUCHBASE_FROM_SPOCK:
2435                        os_type = "msi"
2436                        windows_msi = True
2437                        self.info.deliverable_type = "msi"
2438                else:
2439                    mesg = " ***** ERROR: ***** \n" \
2440                           " Couchbase Server version format is not correct. \n" \
2441                           " It should be 0.0.0-DDDD format\n" \
2442                           % (self.ip, os.getpid())
2443                    self.stop_current_python_running(mesg)
2444
2445                build_repo = MV_LATESTBUILD_REPO
2446                if full_version[:5] not in COUCHBASE_VERSION_2 and \
2447                   full_version[:5] not in COUCHBASE_VERSION_3:
2448                    if full_version[:3] in CB_VERSION_NAME:
2449                        build_repo = CB_REPO + CB_VERSION_NAME[full_version[:3]] + "/"
2450                    else:
2451                        sys.exit("version is not support yet")
2452                log.info("*****VERSION file exists."
2453                                   "Start to uninstall {0} on {1} server"
2454                                   .format(product, self.ip))
2455                if full_version[:3] == "4.0":
2456                    build_repo = SHERLOCK_BUILD_REPO
2457                log.info('Build name: {0}'.format(build_name))
2458                build_name = build_name.rstrip() + ".%s" % os_type
2459                log.info('Check if {0} is in tmp directory on {1} server'\
2460                                                       .format(build_name, self.ip))
2461                exist = self.file_exists("/cygdrive/c/tmp/", build_name)
2462                if not exist:  # if not exist in tmp dir, start to download that version
2463                    if short_version[:5] in cb_releases_version:
2464                        build = query.find_couchbase_release_build(product_name,
2465                                                        self.info.deliverable_type,
2466                                                       self.info.architecture_type,
2467                                                                     short_version,
2468                                                                  is_amazon=False,
2469                                 os_version=self.info.distribution_version.lower())
2470                    elif short_version in COUCHBASE_RELEASE_VERSIONS_3:
2471                        build = query.find_membase_release_build(product_name,
2472                                                   self.info.deliverable_type,
2473                                                  self.info.architecture_type,
2474                                                                short_version,
2475                                                              is_amazon=False,
2476                            os_version=self.info.distribution_version.lower())
2477                    else:
2478                        builds, changes = query.get_all_builds(version=full_version,
2479                                      deliverable_type=self.info.deliverable_type,
2480                                      architecture_type=self.info.architecture_type,
2481                                      edition_type=product_name, repo=build_repo,
2482                                      distribution_version=\
2483                                            self.info.distribution_version.lower())
2484
2485                        build = query.find_build(builds, product_name, os_type,
2486                                                   self.info.architecture_type,
2487                                                                  full_version,
2488                            distribution_version=self.info.distribution_version.lower(),
2489                                  distribution_type=self.info.distribution_type.lower())
2490                    downloaded = self.download_binary_in_win(build.url, short_version)
2491                    if downloaded:
2492                        log.info('Successful download {0}.exe on {1} server'
2493                                             .format(short_version, self.ip))
2494                    else:
2495                        log.error('Download {0}.exe failed'.format(short_version))
2496                if not windows_msi:
2497                    dir_paths = ['/cygdrive/c/automation', '/cygdrive/c/tmp']
2498                    self.create_multiple_dir(dir_paths)
2499                    self.copy_files_local_to_remote('resources/windows/automation',
2500                                                      '/cygdrive/c/automation')
2501                    # modify bat file to run uninstall schedule task
2502                    #self.create_windows_capture_file(task, product, full_version)
2503                    capture_iss_file = self.modify_bat_file('/cygdrive/c/automation',
2504                                        bat_file, product, short_version, task)
2505                    self.stop_schedule_tasks()
2506
2507                """ Remove this workaround when bug MB-14504 is fixed """
2508                log.info("Kill any un/install process leftover in sherlock")
2509                log.info("Kill any cbq-engine.exe in sherlock")
2510                self.execute_command('taskkill /F /T /IM cbq-engine.exe')
2511                log.info('sleep for 5 seconds before running task '
2512                                    'schedule uninstall on {0}'.format(self.ip))
2513                """ End remove this workaround when bug MB-14504 is fixed """
2514
2515                """ Remove this workaround when bug MB-28775 is fixed """
2516                # log.info("Kill any eventing-consumer.exe in vulcan")
2517                # self.execute_command('taskkill /F /T /IM eventing*')
2518                """ End remove this workaround when bug MB-28775 is fixed """
2519
2520                self.stop_couchbase()
2521                time.sleep(5)
2522                # run schedule task uninstall couchbase server
2523                if windows_msi:
2524                    log.info("******** uninstall via msi method ***********")
2525                    output, error = \
2526                        self.execute_command("cd /cygdrive/c/tmp; msiexec /x %s /qn"\
2527                                             % build_name)
2528                    self.log_command_output(output, error)
2529                    var_dir = "/cygdrive/c/Program\ Files/Couchbase/Server/var"
2530                    self.execute_command("rm -rf %s" % var_dir)
2531                else:
2532                    output, error = \
2533                        self.execute_command("cmd /c schtasks /run /tn removeme")
2534                    self.log_command_output(output, error)
2535                deleted = self.wait_till_file_deleted(version_path, \
2536                                            VERSION_FILE, timeout_in_seconds=600)
2537                if not deleted:
2538                    if windows_msi:
2539                        log.info("******** repair via msi method ***********")
2540                        output, error = \
2541                            self.execute_command("cd /cygdrive/c/tmp; msiexec /fa %s /norestart"\
2542                                                 % build_name)
2543                        self.log_command_output(output, error)
2544                        log.info("******** uninstall via msi method ***********")
2545                        output, error = \
2546                            self.execute_command("cd /cygdrive/c/tmp; msiexec /x %s /qn" \
2547                                                 % build_name)
2548                        self.log_command_output(output, error)
2549                        var_dir = "/cygdrive/c/Program\ Files/Couchbase/"
2550                        self.execute_command("rm -rf %s" % var_dir)
2551                        deleted = self.wait_till_file_deleted(version_path, \
2552                                                              VERSION_FILE, timeout_in_seconds=300)
2553                        if not deleted:
2554                            log.error("Uninstall was failed at node {0}".format(self.ip))
2555                            sys.exit()
2556                    else:
2557                        log.error("Uninstall was failed at node {0}".format(self.ip))
2558                        sys.exit()
2559                if full_version[:3] != "2.5":
2560                    uninstall_process = full_version[:10]
2561                    if not windows_msi:
2562                        ended = self.wait_till_process_ended(uninstall_process)
2563                        if not ended:
2564                            sys.exit("****  Node %s failed to uninstall  ****"
2565                                                              % (self.ip))
2566                if full_version[:3] == "2.5":
2567                    self.sleep(20, "next step is to install")
2568                else:
2569                    self.sleep(10, "next step is to install")
2570                """ delete binary after uninstall """
2571                self.delete_file(WIN_TMP_PATH, build_name)
2572                """ the code below need to remove when bug MB-11328
2573                                                           is fixed in 3.0.1 """
2574                output, error = self.kill_erlang(os="windows")
2575                self.log_command_output(output, error)
2576                """ end remove code """
2577
2578
2579                """ the code below need to remove when bug MB-11985
2580                                                           is fixed in 3.0.1 """
2581                if full_version[:5] in COUCHBASE_VERSION_2 or \
2582                   full_version[:5] in COUCHBASE_FROM_VERSION_3:
2583                    log.info("due to bug MB-11985, we need to delete below registry")
2584                    output, error = self.execute_command("reg delete \
2585                            'HKLM\Software\Wow6432Node\Ericsson\Erlang\ErlSrv' /f ")
2586                    self.log_command_output(output, error)
2587                """ end remove code """
2588                if capture_iss_file and not windows_msi:
2589                    log.info("Delete {0} in windows automation directory" \
2590                                                          .format(capture_iss_file))
2591                    output, error = self.execute_command("rm -f \
2592                               /cygdrive/c/automation/{0}".format(capture_iss_file))
2593                    self.log_command_output(output, error)
2594                    log.info("Delete {0} in slave resources/windows/automation dir" \
2595                             .format(capture_iss_file))
2596                    os.system("rm -f resources/windows/automation/{0}" \
2597                                                          .format(capture_iss_file))
2598            else:
2599                log.info("*****No couchbase server on {0} server. Free to install" \
2600                                                                 .format(self.ip))
2601        elif type in LINUX_DISTRIBUTION_NAME:
2602            """ check if couchbase server installed by root """
2603            if self.nonroot:
2604                test = self.file_exists(LINUX_CB_PATH, VERSION_FILE)
2605                if self.file_exists(LINUX_CB_PATH, VERSION_FILE):
2606                    mesg = " ***** ERROR: ***** \n"\
2607                           " Couchbase Server was installed by root user. \n"\
2608                           " Use root user to uninstall it at %s \n"\
2609                           " This python process id: %d will be killed to stop the installation"\
2610                           % (self.ip, os.getpid())
2611                    self.stop_current_python_running(mesg)
2612                if self.file_exists(self.nr_home_path, NR_INSTALL_LOCATION_FILE):
2613                    output, error = self.execute_command("cat %s"
2614                                             % NR_INSTALL_LOCATION_FILE)
2615                    if output and output[0]:
2616                        log.info("Couchbase Server was installed in non default path %s"
2617                                                                            % output[0])
2618                        self.nr_home_path = output[0]
2619            # uninstallation command is different
2620            if type == "ubuntu":
2621                if self.nonroot:
2622                    """ check if old files from root install left in server """
2623                    if self.file_exists(LINUX_CB_PATH + "etc/couchdb/", "local.ini.debsave"):
2624                        print " ***** ERROR: ***** \n"\
2625                              "Couchbase Server files was left by root install at %s .\n"\
2626                              "Use root user to delete them all at server %s "\
2627                              " (rm -rf /opt/couchbase) to remove all couchbase folder.\n" \
2628                              % (LINUX_CB_PATH, self.ip)
2629                        sys.exit(1)
2630                    self.stop_server()
2631                else:
2632                    if sv in COUCHBASE_FROM_VERSION_4:
2633                        if self.is_enterprise(type):
2634                            if product is not None and product == "cbas":
2635                                uninstall_cmd = "dpkg -r {0};dpkg --purge {1};" \
2636                                        .format("couchbase-server-analytics", "couchbase-server-analytics")
2637                            else:
2638                                uninstall_cmd = "dpkg -r {0};dpkg --purge {1};" \
2639                                        .format("couchbase-server", "couchbase-server")
2640                        else:
2641                            uninstall_cmd = "dpkg -r {0};dpkg --purge {1};" \
2642                                         .format("couchbase-server-community",
2643                                                  "couchbase-server-community")
2644                    else:
2645                        uninstall_cmd = "dpkg -r {0};dpkg --purge {1};" \
2646                                     .format("couchbase-server", "couchbase-server")
2647                    output, error = self.execute_command(uninstall_cmd)
2648                    self.log_command_output(output, error)
2649            elif type in RPM_DIS_NAME:
2650                """ Sometimes, vm left with unfinish uninstall/install process.
2651                    We need to kill them before doing uninstall """
2652                if self.nonroot:
2653                    """ check if old files from root install left in server """
2654                    if self.file_exists(LINUX_CB_PATH + "etc/couchdb/", "local.ini.rpmsave"):
2655                        print "Couchbase Server files was left by root install at %s .\n"\
2656                               "Use root user to delete them all at server %s "\
2657                               " (rm -rf /opt/couchbase) to remove all couchbase folder.\n" \
2658                               % (LINUX_CB_PATH, self.ip)
2659                        sys.exit(1)
2660                    self.stop_server()
2661                else:
2662                    output, error = self.execute_command("killall -9 rpm")
2663                    self.log_command_output(output, error)
2664                    output, error = self.execute_command("rm -f /var/lib/rpm/.rpm.lock")
2665                    self.log_command_output(output, error)
2666                    if sv in COUCHBASE_FROM_VERSION_4:
2667                        if self.is_enterprise(type):
2668                            if product is not None and product == "cbas":
2669                                uninstall_cmd = 'rpm -e {0}'.format("couchbase-server-analytics")
2670                            else:
2671                                uninstall_cmd = 'rpm -e {0}'.format("couchbase-server")
2672                        else:
2673                            uninstall_cmd = 'rpm -e {0}' \
2674                                          .format("couchbase-server-community")
2675                    else:
2676                        uninstall_cmd = 'rpm -e {0}'.format("couchbase-server")
2677                    log.info('running rpm -e to remove couchbase-server')
2678                    output, error = self.execute_command(uninstall_cmd)
2679                    if output:
2680                        server_ip = "\n\n**** Uninstalling on server: {0} ****".format(self.ip)
2681                        output.insert(0, server_ip)
2682                    if error:
2683                        server_ip = "\n\n**** Uninstalling on server: {0} ****".format(self.ip)
2684                        error.insert(0, server_ip)
2685                    self.log_command_output(output, error)
2686            self.terminate_processes(self.info, terminate_process_list)
2687            if not self.nonroot:
2688                self.remove_folders(linux_folders)
2689                self.kill_memcached()
2690                output, error = self.execute_command("ipcrm")
2691                self.log_command_output(output, error)
2692        elif self.info.distribution_type.lower() == 'mac':
2693            self.stop_server(os='mac')
2694            """ close Safari browser before uninstall """
2695            self.terminate_process(self.info, "/Applications/Safari.app/Contents/MacOS/Safari")
2696            self.terminate_processes(self.info, terminate_process_list)
2697            output, error = self.execute_command("rm -rf /Applications/Couchbase\ Server.app")
2698            self.log_command_output(output, error)
2699            output, error = self.execute_command("rm -rf ~/Library/Application\ Support/Couchbase")
2700            self.log_command_output(output, error)
2701        if self.nonroot:
2702            if self.nr_home_path != "/home/%s/" % self.username:
2703                log.info("remove all non default install dir")
2704                output, error = self.execute_command("rm -rf %s"
2705                                                        % self.nr_home_path)
2706                self.log_command_output(output, error)
2707            else:
2708                log.info("Remove only Couchbase Server directories opt etc and usr ")
2709                output, error = self.execute_command("cd %s;rm -rf opt etc usr"
2710                                                           % self.nr_home_path)
2711                self.log_command_output(output, error)
2712                self.execute_command("cd %s;rm -rf couchbase-server-*"
2713                                                      % self.nr_home_path)
2714                output, error = self.execute_command("cd %s;ls -lh"
2715                                                      % self.nr_home_path)
2716                self.log_command_output(output, error)
2717            if "nr_install_dir" not in self.input.test_params:
2718                self.nr_home_path = "/home/%s/" % self.username
2719                output, error = self.execute_command(" :> %s"
2720                                             % NR_INSTALL_LOCATION_FILE)
2721                self.log_command_output(output, error)
2722            output, error = self.execute_command("ls -lh")
2723            self.log_command_output(output, error)
2724
2725    def couchbase_win_uninstall(self, product, version, os_name, query):
2726        log.info('*****couchbase_win_uninstall****')
2727        builds, changes = query.get_all_builds(version=version)
2728        bat_file = "uninstall.bat"
2729        task = "uninstall"
2730        deleted = False
2731
2732        self.extract_remote_info()
2733        ex_type = self.info.deliverable_type
2734        if self.info.architecture_type == "x86_64":
2735            os_type = "64"
2736        elif self.info.architecture_type == "x86":
2737            os_type = "32"
2738        if product == "cse":
2739            name = "couchbase-server-enterprise"
2740            version_path = "/cygdrive/c/Program Files/Couchbase/Server/"
2741
2742        exist = self.file_exists(version_path, VERSION_FILE)
2743        if exist:
2744            # call uninstall function to install couchbase server
2745            # Need to detect csse or cse when uninstall.
2746            log.info("Start uninstall cb server on this server")
2747            build_name, rm_version = self.find_build_version(version_path, VERSION_FILE)
2748            log.info('build needed to do auto uninstall {0}'.format(build_name))
2749            # find installed build in tmp directory to match with currently installed version
2750            build_name = build_name.rstrip() + ".exe"
2751            log.info('Check if {0} is in tmp directory'.format(build_name))
2752            exist = self.file_exists("/cygdrive/c/tmp/", build_name)
2753            if not exist:  # if not exist in tmp dir, start to download that version build
2754                build = query.find_build(builds, name, ex_type, self.info.architecture_type, rm_version)
2755                downloaded = self.download_binary_in_win(build.url, rm_version)
2756                if downloaded:
2757                    log.info('Successful download {0}.exe'.format(rm_version))
2758                else:
2759                    log.error('Download {0}.exe failed'.format(rm_version))
2760            # copy required files to automation directory
2761            dir_paths = ['/cygdrive/c/automation', '/cygdrive/c/tmp']
2762            self.create_multiple_dir(dir_paths)
2763            self.copy_files_local_to_remote('resources/windows/automation', '/cygdrive/c/automation')
2764            # modify bat file to run uninstall schedule task
2765            self.modify_bat_file('/cygdrive/c/automation', bat_file, product, rm_version, task)
2766            self.stop_couchbase()
2767
2768            """ the code below need to remove when bug MB-11328 is fixed in 3.0.1 """
2769            output, error = self.kill_erlang(os="windows")
2770            self.log_command_output(output, error)
2771            """ end remove code """
2772
2773            """ the code below need to removed when bug MB-28775 is fixed """
2774            output, error = self.kill_eventing_process(name="eventing-consumer")
2775            self.log_command_output(output, error)
2776            """ end remove code """
2777
2778            self.sleep(5, "before running task schedule uninstall")
2779            # run schedule task uninstall couchbase server
2780            output, error = self.execute_command("cmd /c schtasks /run /tn removeme")
2781            self.log_command_output(output, error)
2782            deleted = self.wait_till_file_deleted(version_path, VERSION_FILE, timeout_in_seconds=600)
2783            if not deleted:
2784                log.error("Uninstall was failed at node {0}".format(self.ip))
2785                sys.exit()
2786            ended = self.wait_till_process_ended(build_name[:10])
2787            if not ended:
2788                sys.exit("*****  Node %s failed to uninstall  *****" % (self.ip))
2789            self.sleep(10, "next step is to install")
2790            output, error = self.execute_command("rm -f \
2791                       /cygdrive/c/automation/*_{0}_uninstall.iss".format(self.ip))
2792            self.log_command_output(output, error)
2793            output, error = self.execute_command("cmd /c schtasks /Query /FO LIST /TN removeme /V")
2794            self.log_command_output(output, error)
2795            output, error = self.execute_command("rm -f /cygdrive/c/tmp/{0}".format(build_name))
2796            self.log_command_output(output, error)
2797
2798            """ the code below need to remove when bug MB-11328 is fixed in 3.0.1 """
2799            output, error = self.kill_erlang(os="windows")
2800            self.log_command_output(output, error)
2801            """ end remove code """
2802        else:
2803            log.info('No couchbase server on this server')
2804
2805    def membase_uninstall(self, save_upgrade_config=False):
2806        linux_folders = ["/var/opt/membase", "/opt/membase", "/etc/opt/membase",
2807                         "/var/membase/data/*", "/opt/membase/var/lib/membase/*"]
2808        terminate_process_list = ["beam", "memcached", "moxi", "vbucketmigrator",
2809                                  "couchdb", "epmd"]
2810        self.extract_remote_info()
2811        log.info(self.info.distribution_type)
2812        type = self.info.distribution_type.lower()
2813        if type == 'windows':
2814            product = "mb"
2815            query = BuildQuery()
2816            builds, changes = query.get_all_builds()
2817            os_type = "exe"
2818            task = "uninstall"
2819            bat_file = "uninstall.bat"
2820            deleted = False
2821            product_name = "membase-server-enterprise"
2822            version_path = "/cygdrive/c/Program Files/Membase/Server/"
2823
2824            exist = self.file_exists(version_path, VERSION_FILE)
2825            log.info("Is VERSION file existed? {0}".format(exist))
2826            if exist:
2827                log.info("VERSION file exists.  Start to uninstall")
2828                build_name, short_version, full_version = self.find_build_version(version_path, VERSION_FILE, product)
2829                if "1.8.0" in full_version or "1.8.1" in full_version:
2830                    product_name = "couchbase-server-enterprise"
2831                    product = "cb"
2832                log.info('Build name: {0}'.format(build_name))
2833                build_name = build_name.rstrip() + ".exe"
2834                log.info('Check if {0} is in tmp directory'.format(build_name))
2835                exist = self.file_exists("/cygdrive/c/tmp/", build_name)
2836                if not exist:  # if not exist in tmp dir, start to download that version build
2837                    build = query.find_build(builds, product_name, os_type, self.info.architecture_type, full_version)
2838                    downloaded = self.download_binary_in_win(build.url, short_version)
2839                    if downloaded:
2840                        log.info('Successful download {0}_{1}.exe'.format(product, short_version))
2841                    else:
2842                        log.error('Download {0}_{1}.exe failed'.format(product, short_version))
2843                dir_paths = ['/cygdrive/c/automation', '/cygdrive/c/tmp']
2844                self.create_multiple_dir(dir_paths)
2845                self.copy_files_local_to_remote('resources/windows/automation', '/cygdrive/c/automation')
2846                # modify bat file to run uninstall schedule task
2847                #self.create_windows_capture_file(task, product, full_version)
2848                self.modify_bat_file('/cygdrive/c/automation', bat_file, product, short_version, task)
2849                self.stop_schedule_tasks()
2850                self.sleep(5, "before running task schedule uninstall")
2851                # run schedule task uninstall Couchbase server
2852                output, error = self.execute_command("cmd /c schtasks /run /tn removeme")
2853                self.log_command_output(output, error)
2854                deleted = self.wait_till_file_deleted(version_path, VERSION_FILE, timeout_in_seconds=600)
2855                if not deleted:
2856                    log.error("Uninstall was failed at node {0}".format(self.ip))
2857                    sys.exit()
2858                ended = self.wait_till_process_ended(full_version[:10])
2859                if not ended:
2860                    sys.exit("****  Node %s failed to uninstall  ****" % (self.ip))
2861                self.sleep(10, "next step is to install")
2862                output, error = self.execute_command("rm -f \
2863                       /cygdrive/c/automation/*_{0}_uninstall.iss".format(self.ip))
2864                self.log_command_output(output, error)
2865                output, error = self.execute_command("cmd /c schtasks /Query /FO LIST /TN removeme /V")
2866                self.log_command_output(output, error)
2867            else:
2868                log.info("No membase server on this server.  Free to install")
2869        elif type in LINUX_DISTRIBUTION_NAME:
2870            # uninstallation command is different
2871            if type == "ubuntu":
2872                uninstall_cmd = 'dpkg -r {0};dpkg --purge {1};' \
2873                                 .format('membase-server', 'membase-server')
2874                output, error = self.execute_command(uninstall_cmd)
2875                self.log_command_output(output, error)
2876            elif type in RPM_DIS_NAME:
2877                """ Sometimes, vm left with unfinish uninstall/install process.
2878                    We need to kill them before doing uninstall """
2879                output, error = self.execute_command("killall -9 rpm")
2880                self.log_command_output(output, error)
2881                uninstall_cmd = 'rpm -e {0}'.format('membase-server')
2882                log.info('running rpm -e to remove membase-server')
2883                output, error = self.execute_command(uninstall_cmd)
2884                self.log_command_output(output, error)
2885            self.terminate_processes(self.info, terminate_process_list)
2886            if not save_upgrade_config:
2887                self.remove_folders(linux_folders)
2888
2889    def moxi_uninstall(self):
2890        terminate_process_list = ["moxi"]
2891        self.extract_remote_info()
2892        log.info(self.info.distribution_type)
2893        type = self.info.distribution_type.lower()
2894        if type == 'windows':
2895            log.error("Not implemented")
2896        elif type == "ubuntu":
2897            uninstall_cmd = "dpkg -r {0};dpkg --purge {1};".format("moxi-server", "moxi-server")
2898            output, error = self.execute_command(uninstall_cmd)
2899            self.log_command_output(output, error)
2900        elif type in LINUX_DISTRIBUTION_NAME:
2901            uninstall_cmd = 'rpm -e {0}'.format("moxi-server")
2902            log.info('running rpm -e to remove couchbase-server')
2903            output, error = self.execute_command(uninstall_cmd)
2904            self.log_command_output(output, error)
2905        self.terminate_processes(self.info, terminate_process_list)
2906
2907    def log_command_output(self, output, error, track_words=(), debug=True):
2908        # success means that there are no track_words in the output
2909        # and there are no errors at all, if track_words is not empty
2910        # if track_words=(), the result is not important, and we return True
2911        success = True
2912        for line in error:
2913            if debug:
2914                log.error(line)
2915            if track_words:
2916                if "Warning" in line and "hugepages" in line:
2917                    log.info("There is a warning about transparent_hugepage "
2918                             "may be in used when install cb server.\
2919                              So we will disable transparent_hugepage in this vm")
2920                    output, error = self.execute_command("echo never > \
2921                                        /sys/kernel/mm/transparent_hugepage/enabled")
2922                    self.log_command_output(output, error)
2923                    success = True
2924                elif "Warning" in line and "systemctl daemon-reload" in line:
2925                    log.info("Unit file of couchbase-server.service changed on disk,"
2926                             " we will run 'systemctl daemon-reload'")
2927                    output, error = self.execute_command("systemctl daemon-reload")
2928                    self.log_command_output(output, error)
2929                    success = True
2930                elif "Warning" in line and "RPMDB altered outside of yum" in line:
2931                    log.info("Warming: RPMDB altered outside of yum")
2932                    success = True
2933                elif "dirname" in line:
2934                    log.warning("Ignore dirname error message during couchbase "
2935                                "startup/stop/restart for CentOS 6.6 (MB-12536)")
2936                    success = True
2937                elif "Created symlink from /etc/systemd/system" in line:
2938                    log.info("This error is due to fix_failed_install.py script that only "
2939                             "happens in centos 7")
2940                    success = True
2941                elif "Created symlink /etc/systemd/system/multi-user.target.wants/couchbase-server.service" in line:
2942                    log.info(line)
2943                    log.info("This message comes only in debian8 and debian9 during installation. "
2944                             "This can be ignored.")
2945                    success = True
2946                else:
2947                    log.info("\nIf couchbase server is running with this error."
2948                              "\nGo to log_command_output to add error mesg to bypass it.")
2949                    success = False
2950        for line in output:
2951            if debug:
2952                log.info(line)
2953            if any(s.lower() in line.lower() for s in track_words):
2954                if "Warning" in line and "hugepages" in line:
2955                    log.info("There is a warning about transparent_hugepage may be in used when install cb server.\
2956                              So we will disable transparent_hugepage in this vm")
2957                    output, error = self.execute_command("echo never > /sys/kernel/mm/transparent_hugepage/enabled")
2958                    self.log_command_output(output, error, debug=debug)
2959                    success = True
2960                else:
2961                    success = False
2962                    log.error('something wrong happened on {0}!!! output:{1}, error:{2}, track_words:{3}'
2963                              .format(self.ip, output, error, track_words))
2964        return success
2965
2966    def execute_commands_inside(self, main_command,query, queries,bucket1,password,bucket2,source,subcommands=[], min_output_size=0,
2967                                end_msg='', timeout=250):
2968        self.extract_remote_info()
2969        filename = "/tmp/test2"
2970        iswin=False
2971
2972        if self.info.type.lower() == 'windows':
2973            iswin = True
2974            filename = "/cygdrive/c/tmp/test.txt"
2975
2976        filedata = ""
2977        if not(query==""):
2978            main_command = main_command + " -s=\"" + query+ '"'
2979        elif (self.remote and not(queries=="")):
2980            sftp = self._ssh_client.open_sftp()
2981            filein = sftp.open(filename, 'w')
2982            for query in queries:
2983                filein.write(query)
2984                filein.write('\n')
2985            fileout = sftp.open(filename,'r')
2986            filedata = fileout.read()
2987            #print filedata
2988            fileout.close()
2989        elif not(queries==""):
2990            f = open(filename, 'w')
2991            for query in queries:
2992                f.write(query)
2993                f.write('\n')
2994            f.close()
2995            fileout = open(filename,'r')
2996            filedata = fileout.read()
2997            fileout.close()
2998
2999        newdata = filedata.replace("bucketname",bucket2)
3000        newdata = newdata.replace("user",bucket1)
3001        newdata = newdata.replace("pass",password)
3002        newdata = newdata.replace("bucket1",bucket1)
3003
3004        newdata = newdata.replace("user1",bucket1)
3005        newdata = newdata.replace("pass1",password)
3006        newdata = newdata.replace("bucket2",bucket2)
3007        newdata = newdata.replace("user2",bucket2)
3008        newdata = newdata.replace("pass2",password)
3009
3010        if (self.remote and not(queries=="")) :
3011            f = sftp.open(filename,'w')
3012            f.write(newdata)
3013            f.close()
3014        elif not(queries==""):
3015            f = open(filename,'w')
3016            f.write(newdata)
3017            f.close()
3018        if not(queries==""):
3019            if (source):
3020                if iswin:
3021                    main_command = main_command + "  -s=\"\SOURCE " + 'c:\\\\tmp\\\\test.txt'
3022                else:
3023                    main_command = main_command + "  -s=\"\SOURCE " + filename+ '"'
3024            else:
3025                if iswin:
3026                    main_command = main_command + " -f=" + 'c:\\\\tmp\\\\test.txt'
3027                else:
3028                    main_command = main_command + " -f=" + filename
3029
3030        log.info("running command on {0}: {1}".format(self.ip, main_command))
3031        output=""
3032        if self.remote:
3033            (stdin, stdout, stderro) = self._ssh_client.exec_command(main_command)
3034            time.sleep(10)
3035            count = 0
3036            for line in stdout.readlines():
3037                if (count == 0) and line.lower().find("error") > 0:
3038                   output = "status:FAIL"
3039                   break
3040
3041                #if line.find("results") > 0 or line.find("status") > 0 or line.find("metrics") or line.find("elapsedTime")> 0 or  line.find("executionTime")> 0 or line.find("resultCount"):
3042                if (count > 0):
3043                    output+=line.strip()
3044                    output = output.strip()
3045                    if "Inputwasnotastatement" in output:
3046                        output = "status:FAIL"
3047                        break
3048                    if "timeout" in output:
3049                        output = "status:timeout"
3050                else:
3051                    count+=1
3052            stdin.close()
3053            stdout.close()
3054            stderro.close()
3055           # main_command = main_command + " < " + '/tmp/' + filename
3056           # stdin,stdout, ssh_stderr = ssh.exec_command(main_command)
3057           # stdin.close()
3058           # output = []
3059           # for line in stdout.read().splitlines():
3060           #   print(line)
3061           #   output = output.append(line)
3062           # f.close()
3063           # ssh.close()
3064
3065           #output = output + end_msg
3066
3067        else:
3068            p = Popen(main_command , shell=True, stdout=PIPE, stderr=PIPE)
3069            stdout, stderro = p.communicate()
3070            output = stdout
3071            print output
3072            time.sleep(1)
3073        # for cmd in subcommands:
3074        #       log.info("running command {0} inside {1} ({2})".format(
3075        #                                                 main_command, cmd, self.ip))
3076        #       stdin.channel.send("{0}\n".format(cmd))
3077        #       end_time = time.time() + float(timeout)
3078        #       while True:
3079        #           if time.time() > end_time:
3080        #               raise Exception("no output in {3} sec running command \
3081        #                                {0} inside {1} ({2})".format(main_command,
3082        #                                                             cmd, self.ip,
3083        #                                                             timeout))
3084        #           output = stdout.channel.recv(1024)
3085        #           if output.strip().endswith(end_msg) and len(output) >= min_output_size:
3086        #                   break
3087        #           time.sleep(2)
3088        #       log.info("{0}:'{1}' -> '{2}' output\n: {3}".format(self.ip, main_command, cmd, output))
3089        # stdin.close()
3090        # stdout.close()
3091        # stderro.close()
3092        if (self.remote and not(queries=="")) :
3093            sftp.remove(filename)
3094            sftp.close()
3095        elif not(queries==""):
3096            os.remove(filename)
3097
3098        output = re.sub('\s+', '', output)
3099        return (output)
3100
3101    def execute_command(self, command, info=None, debug=True, use_channel=False):
3102        if getattr(self, "info", None) is None and info is not None :
3103            self.info = info
3104        else:
3105            self.extract_remote_info()
3106
3107        if self.info.type.lower() == 'windows':
3108            self.use_sudo = False
3109
3110        if self.use_sudo:
3111            command = "sudo " + command
3112
3113        return self.execute_command_raw(command, debug=debug, use_channel=use_channel)
3114
3115    def execute_command_raw(self, command, debug=True, use_channel=False):
3116        if debug:
3117            log.info("running command.raw on {0}: {1}".format(self.ip, command))
3118        output = []
3119        error = []
3120        temp = ''
3121        if self.remote and self.use_sudo or use_channel:
3122            channel = self._ssh_client.get_transport().open_session()
3123            channel.get_pty()
3124            channel.settimeout(900)
3125            stdin = channel.makefile('wb')
3126            stdout = channel.makefile('rb')
3127            stderro = channel.makefile_stderr('rb')
3128            channel.exec_command(command)
3129            data = channel.recv(1024)
3130            while data:
3131                temp += data
3132                data = channel.recv(1024)
3133            channel.close()
3134            stdin.close()
3135        elif self.remote:
3136            stdin, stdout, stderro = self._ssh_client.exec_command(command)
3137            stdin.close()
3138
3139        if not self.remote:
3140            p = Popen(command , shell=True, stdout=PIPE, stderr=PIPE)
3141            output, error = p.communicate()
3142
3143        if self.remote:
3144            for line in stdout.read().splitlines():
3145                output.append(line)
3146            for line in stderro.read().splitlines():
3147                error.append(line)
3148            if temp:
3149                line = temp.splitlines()
3150                output.extend(line)
3151            stdout.close()
3152            stderro.close()
3153        if debug:
3154            log.info('command executed successfully')
3155        return output, error
3156
3157    def execute_non_sudo_command(self, command, info=None, debug=True, use_channel=False):
3158        info = info or self.extract_remote_info()
3159        self.info = info
3160
3161        return self.execute_command_raw(command, debug=debug, use_channel=use_channel)
3162
3163    def terminate_process(self, info=None, process_name='',force=False):
3164        self.extract_remote_info()
3165        if self.info.type.lower() == 'windows':
3166            o, r = self.execute_command("taskkill /F /T /IM {0}*"\
3167                                            .format(process_name), debug=False)
3168            self.log_command_output(o, r)
3169        else:
3170            if (force == True):
3171                o, r = self.execute_command("kill -9 "
3172                   "$(ps aux | grep '{0}' |  awk '{{print $2}}') "\
3173                               .format(process_name), debug=False)
3174                self.log_command_output(o, r)
3175            else:
3176                o, r = self.execute_command("kill "
3177                    "$(ps aux | grep '{0}' |  awk '{{print $2}}') "\
3178                                 .format(process_name), debug=False)
3179                self.log_command_output(o, r)
3180
3181    def disconnect(self):
3182        self._ssh_client.close()
3183
3184    def extract_remote_info(self):
3185        # initialize params
3186        os_distro = "linux"
3187        os_version = "default"
3188        is_linux_distro = True
3189        self.use_sudo = False
3190        is_mac = False
3191        arch = "local"
3192        ext = "local"
3193        # use ssh to extract remote machine info
3194        # use sftp to if certain types exists or not
3195        if getattr(self, "info", None) is not None and isinstance(self.info, RemoteMachineInfo):
3196            return self.info
3197        mac_check_cmd = "sw_vers | grep ProductVersion | awk '{ print $2 }'"
3198        if self.remote:
3199            stdin, stdout, stderro = self._ssh_client.exec_command(mac_check_cmd)
3200            stdin.close()
3201            ver, err = stdout.read(), stderro.read()
3202        else:
3203            p = Popen(mac_check_cmd , shell=True, stdout=PIPE, stderr=PIPE)
3204            ver, err = p.communicate()
3205        if not err and ver:
3206            os_distro = "Mac"
3207            os_version = ver
3208            is_linux_distro = True
3209            is_mac = True
3210            self.use_sudo = False
3211        elif self.remote:
3212            is_mac = False
3213            sftp = self._ssh_client.open_sftp()
3214            filenames = sftp.listdir('/etc/')
3215            os_distro = ""
3216            os_version = ""
3217            is_linux_distro = False
3218            for name in filenames:
3219                if name == 'issue':
3220                    # it's a linux_distro . let's downlaod this file
3221                    # format Ubuntu 10.04 LTS \n \l
3222                    filename = 'etc-issue-{0}'.format(uuid.uuid4())
3223                    sftp.get(localpath=filename, remotepath='/etc/issue')
3224                    file = open(filename)
3225                    etc_issue = ''
3226                    # let's only read the first line
3227                    for line in file.xreadlines():
3228                        # for SuSE that has blank first line
3229                        if line.rstrip('\n'):
3230                            etc_issue = line
3231                            break
3232                        # strip all extra characters
3233                    etc_issue = etc_issue.rstrip('\n').rstrip(' ').rstrip('\\l').rstrip(' ').rstrip('\\n').rstrip(' ')
3234                    if etc_issue.lower().find('ubuntu') != -1:
3235                        os_distro = 'Ubuntu'
3236                        os_version = etc_issue
3237                        tmp_str = etc_issue.split()
3238                        if tmp_str and tmp_str[1][:2].isdigit():
3239                            os_version = "Ubuntu %s" % tmp_str[1][:5]
3240                        is_linux_distro = True
3241                    elif etc_issue.lower().find('debian') != -1:
3242                        os_distro = 'Ubuntu'
3243                        os_version = etc_issue
3244                        is_linux_distro = True
3245                    elif etc_issue.lower().find('mint') != -1:
3246                        os_distro = 'Ubuntu'
3247                        os_version = etc_issue
3248                        is_linux_distro = True
3249                    elif etc_issue.lower().find('amazon linux ami') != -1:
3250                        os_distro = 'CentOS'
3251                        os_version = etc_issue
3252                        is_linux_distro = True
3253                    elif etc_issue.lower().find('centos') != -1:
3254                        os_distro = 'CentOS'
3255                        os_version = etc_issue
3256                        is_linux_distro = True
3257                    elif etc_issue.lower().find('red hat') != -1:
3258                        os_distro = 'Red Hat'
3259                        os_version = etc_issue
3260                        is_linux_distro = True
3261                    elif etc_issue.lower().find('opensuse') != -1:
3262                        os_distro = 'openSUSE'
3263                        os_version = etc_issue
3264                        is_linux_distro = True
3265                    elif etc_issue.lower().find('suse linux') != -1:
3266                        os_distro = 'SUSE'
3267                        os_version = etc_issue
3268                        tmp_str = etc_issue.split()
3269                        if tmp_str and tmp_str[6].isdigit():
3270                            os_version = "SUSE %s" % tmp_str[6]
3271                        is_linux_distro = True
3272                    elif etc_issue.lower().find('oracle linux') != -1:
3273                        os_distro = 'Oracle Linux'
3274                        os_version = etc_issue
3275                        is_linux_distro = True
3276                    else:
3277                        log.info("It could be other operating system."
3278                                 "  Go to check at other location")
3279                    file.close()
3280                    # now remove this file
3281                    os.remove(filename)
3282                    break
3283            else:
3284                os_distro = "linux"
3285                os_version = "default"
3286                is_linux_distro = True
3287                self.use_sudo = False
3288                is_mac = False
3289                arch = "local"
3290                ext = "local"
3291                filenames = []
3292            """ for centos 7 only """
3293            for name in filenames:
3294                if name == "redhat-release":
3295                    filename = 'redhat-release-{0}'.format(uuid.uuid4())
3296                    if self.remote:
3297                        sftp.get(localpath=filename, remotepath='/etc/redhat-release')
3298                    else:
3299                        p = Popen("cat /etc/redhat-release > {0}".format(filename) , shell=True, stdout=PIPE, stderr=PIPE)
3300                        var, err = p.communicate()
3301                    file = open(filename)
3302                    redhat_release = ''
3303                    for line in file.xreadlines():
3304                        redhat_release = line
3305                        break
3306                    redhat_release = redhat_release.rstrip('\n').rstrip('\\l').rstrip('\\n')
3307                    """ in ec2: Red Hat Enterprise Linux Server release 7.2 """
3308                    if redhat_release.lower().find('centos') != -1 \
3309                         or redhat_release.lower().find('linux server') != -1:
3310                        if redhat_release.lower().find('release 7') != -1:
3311                            os_distro = 'CentOS'
3312                            os_version = "CentOS 7"
3313                            is_linux_distro = True
3314                    else:
3315                        log.error("Could not find OS name."
3316                                 "It could be unsupport OS")
3317                    file.close()
3318                    os.remove(filename)
3319                    break
3320
3321        if self.remote:
3322            if self.find_file("/cygdrive/c/Windows", "win.ini"):
3323                log.info("This is windows server!")
3324                is_linux_distro = False
3325        if not is_linux_distro:
3326            arch = ''
3327            os_version = 'unknown windows'
3328            win_info = self.find_windows_info()
3329            info = RemoteMachineInfo()
3330            info.type = win_info['os']
3331            info.windows_name = win_info['os_name']
3332            info.distribution_type = win_info['os']
3333            info.architecture_type = win_info['os_arch']
3334            info.ip = self.ip
3335            info.distribution_version = win_info['os']
3336            info.deliverable_type = 'exe'
3337            info.cpu = self.get_cpu_info(win_info)
3338            info.disk = self.get_disk_info(win_info)
3339            info.ram = self.get_ram_info(win_info)
3340            info.hostname = self.get_hostname(win_info)
3341            info.domain = self.get_domain(win_info)
3342            self.info = info
3343            return info
3344        else:
3345            # now run uname -m to get the architechtre type
3346            if self.remote:
3347                stdin, stdout, stderro = self._ssh_client.exec_command('uname -m')
3348                stdin.close()
3349                os_arch = ''
3350                text = stdout.read().splitlines()
3351            else:
3352                p = Popen('uname -m' , shell=True, stdout=PIPE, stderr=PIPE)
3353                text, err = p.communicate()
3354                os_arch = ''
3355            for line in text:
3356                os_arch += line
3357                # at this point we should know if its a linux or windows ditro
3358            ext = { 'Ubuntu' : "deb",
3359                   'CentOS'  : "rpm",
3360                   'Red Hat' : "rpm",
3361                   "Mac"     : "zip",
3362                   "Debian"  : "deb",
3363                   "openSUSE": "rpm",
3364                   "SUSE"    : "rpm",
3365                   "Oracle Linux": "rpm"}.get(os_distro, '')
3366            arch = {'i686': 'x86',
3367                    'i386': 'x86'}.get(os_arch, os_arch)
3368
3369            info = RemoteMachineInfo()
3370            info.type = "Linux"
3371            info.distribution_type = os_distro
3372            info.architecture_type = arch
3373            info.ip = self.ip
3374            info.distribution_version = os_version
3375            info.deliverable_type = ext
3376            info.cpu = self.get_cpu_info(mac=is_mac)
3377            info.disk = self.get_disk_info(mac=is_mac)
3378            info.ram = self.get_ram_info(mac=is_mac)
3379            info.hostname = self.get_hostname()
3380            info.domain = self.get_domain()
3381            self.info = info
3382            return info
3383
3384    def get_extended_windows_info(self):
3385        info = {}
3386        win_info = self.extend_windows_info()
3387        info['ram'] = self.get_ram_info(win_info)
3388        info['disk'] = self.get_disk_info(win_info)
3389        info['cpu'] = self.get_cpu_info(win_info)
3390        info['hostname'] = self.get_hostname()
3391        return info
3392
3393    def get_hostname(self, win_info=None):
3394        if win_info:
3395            if 'Host Name' not in win_info:
3396                win_info = self.create_windows_info()
3397            o = win_info['Host Name']
3398        o, r = self.execute_command_raw('hostname', debug=False)
3399        if o:
3400            return o
3401
3402    def get_domain(self, win_info=None):
3403        if win_info:
3404            o, _ = self.execute_batch_command('ipconfig')
3405            suffix_dns_row = [row for row in o if row.find(" Connection-specific DNS Suffix") != -1 and \
3406                             len(row.split(':')[1]) > 1]
3407            if suffix_dns_row ==[]:
3408                #'   Connection-specific DNS Suffix  . : '
3409                ret=""
3410            else:
3411                ret = suffix_dns_row[0].split(':')[1].strip()
3412        else:
3413            ret = self.execute_command_raw('hostname -d', debug=False)
3414        return ret
3415
3416    def get_full_hostname(self):
3417        info = self.extract_remote_info()
3418        if not info.domain:
3419            return None
3420        log.info("hostname of this {0} is {1}"
3421                 .format(self.ip, info.hostname[0]))
3422        if info.type.lower() == 'windows':
3423            log.info("domain name of this {0} is {1}"
3424                 .format(self.ip, info.domain))
3425            return '%s.%s' % (info.hostname[0], info.domain)
3426        else:
3427            if info.domain[0]:
3428                if info.domain[0][0]:
3429                    log.info("domain name of this {0} is {1}"
3430                         .format(self.ip, info.domain[0][0]))
3431                    return "{0}.{1}".format(info.hostname[0], info.domain[0][0])
3432                else:
3433                    mesg = "Need to set domain name in server {0} like 'sc.couchbase.com'"\
3434                                                                           .format(self.ip)
3435                    raise Exception(mesg)
3436
3437    def get_cpu_info(self, win_info=None, mac=False):
3438        if win_info:
3439            if 'Processor(s)' not in win_info:
3440                win_info = self.create_windows_info()
3441            o = win_info['Processor(s)']
3442        elif mac:
3443            o, r = self.execute_command_raw('/sbin/sysctl -n machdep.cpu.brand_string')
3444        else:
3445            o, r = self.execute_command_raw('cat /proc/cpuinfo', debug=False)
3446        if o:
3447            return o
3448
3449    def get_ram_info(self, win_info=None, mac=False):
3450        if win_info:
3451            if 'Virtual Memory Max Size' not in win_info:
3452                win_info = self.create_windows_info()
3453            o = "Virtual Memory Max Size =" + win_info['Virtual Memory Max Size'] + '\n'
3454            o += "Virtual Memory Available =" + win_info['Virtual Memory Available'] + '\n'
3455            o += "Virtual Memory In Use =" + win_info['Virtual Memory In Use']
3456        elif mac:
3457            o, r = self.execute_command_raw('/sbin/sysctl -n hw.memsize',debug=False)
3458        else:
3459            o, r = self.execute_command_raw('cat /proc/meminfo', debug=False)
3460        if o:
3461            return o
3462
3463    def get_disk_info(self, win_info=None, mac=False):
3464        if win_info:
3465            if 'Total Physical Memory' not in win_info:
3466                win_info = self.create_windows_info()
3467            o = "Total Physical Memory =" + win_info['Total Physical Memory'] + '\n'
3468            o += "Available Physical Memory =" + win_info['Available Physical Memory']
3469        elif mac:
3470            o, r = self.execute_command_raw('df -h', debug=False)
3471        else:
3472            o, r = self.execute_command_raw('df -Th', debug=False)
3473        if o:
3474            return o
3475
3476    def get_memcache_pid(self):
3477         self.extract_remote_info()
3478         if self.info.type == 'Linux':
3479             o, _ = self.execute_command("ps -eo comm,pid | awk '$1 == \"memcached\" { print $2 }'")
3480             return o[0]
3481         elif self.info.type == 'windows':
3482             output, error = self.execute_command('tasklist| grep memcache', debug=False)
3483             if error or output == [""] or output == []:
3484                  return None
3485             words = output[0].split(" ")
3486             words = filter(lambda x: x != "", words)
3487             return words[1]
3488
3489    def cleanup_data_config(self, data_path):
3490        self.extract_remote_info()
3491        if self.info.type.lower() == 'windows':
3492            if "c:/Program Files" in data_path:
3493                data_path = data_path.replace("c:/Program Files",
3494                                              "/cygdrive/c/Program\ Files")
3495            o, r = self.execute_command("rm -rf ""{0}""/*".format(data_path))
3496            self.log_command_output(o, r)
3497            o, r = self.execute_command("rm -rf ""{0}""/*"\
3498                                             .format(data_path.replace("data","config")))
3499            self.log_command_output(o, r)
3500        else:
3501            o, r = self.execute_command("rm -rf {0}/*".format(data_path))
3502            self.log_command_output(o, r)
3503            o, r = self.execute_command("rm -rf {0}/*"\
3504                                             .format(data_path.replace("data","config")))
3505            self.log_command_output(o, r)
3506
3507    def