xref: /6.0.3/testrunner/lib/remote/remote_util.py (revision 741ca48f)
1import os
2import re
3import sys
4import copy
5import urllib
6import uuid
7import time
8import logging
9import stat
10import json
11import TestInput
12from subprocess import Popen, PIPE
13
14import logger
15from builds.build_query import BuildQuery
16import testconstants
17from testconstants import VERSION_FILE
18from testconstants import WIN_REGISTER_ID
19from testconstants import MEMBASE_VERSIONS
20from testconstants import COUCHBASE_VERSIONS
21from testconstants import MISSING_UBUNTU_LIB
22from testconstants import MV_LATESTBUILD_REPO
23from testconstants import SHERLOCK_BUILD_REPO
24from testconstants import COUCHBASE_VERSIONS
25from testconstants import WIN_CB_VERSION_3
26from testconstants import COUCHBASE_VERSION_2
27from testconstants import COUCHBASE_VERSION_3
28from testconstants import COUCHBASE_FROM_VERSION_3,\
29                          COUCHBASE_FROM_SPOCK, SYSTEMD_SERVER
30from testconstants import COUCHBASE_RELEASE_VERSIONS_3, CB_RELEASE_BUILDS
31from testconstants import SHERLOCK_VERSION, WIN_PROCESSES_KILLED
32from testconstants import COUCHBASE_FROM_VERSION_4, COUCHBASE_FROM_WATSON,\
33                          COUCHBASE_FROM_SPOCK
34from testconstants import RPM_DIS_NAME
35from testconstants import LINUX_DISTRIBUTION_NAME, LINUX_CB_PATH, \
36                          LINUX_COUCHBASE_BIN_PATH
37from testconstants import WIN_COUCHBASE_BIN_PATH,\
38                          WIN_CB_PATH
39from testconstants import WIN_COUCHBASE_BIN_PATH_RAW
40from testconstants import WIN_TMP_PATH, WIN_TMP_PATH_RAW
41from testconstants import WIN_UNZIP, WIN_PSSUSPEND
42
43from testconstants import MAC_CB_PATH, MAC_COUCHBASE_BIN_PATH
44
45from testconstants import CB_VERSION_NAME
46from testconstants import CB_REPO
47from testconstants import CB_RELEASE_APT_GET_REPO
48from testconstants import CB_RELEASE_YUM_REPO
49
50from testconstants import LINUX_NONROOT_CB_BIN_PATH,\
51                          NR_INSTALL_LOCATION_FILE, LINUX_DIST_CONFIG
52
53from membase.api.rest_client import RestConnection, RestHelper
54
55log = logger.Logger.get_logger()
56logging.getLogger("paramiko").setLevel(logging.WARNING)
57
58try:
59    import paramiko
60except ImportError:
61    log.warn("{0} {1} {2}".format("Warning: proceeding without importing",
62                                  "paramiko due to import error.",
63                                  "ssh connections to remote machines will fail!\n"))
64
65class RemoteMachineInfo(object):
66    def __init__(self):
67        self.type = ''
68        self.ip = ''
69        self.distribution_type = ''
70        self.architecture_type = ''
71        self.distribution_version = ''
72        self.deliverable_type = ''
73        self.ram = ''
74        self.cpu = ''
75        self.disk = ''
76        self.hostname = ''
77
78
79class RemoteMachineProcess(object):
80    def __init__(self):
81        self.pid = ''
82        self.name = ''
83        self.vsz = 0
84        self.rss = 0
85        self.args = ''
86
87
88class RemoteMachineHelper(object):
89    remote_shell = None
90
91    def __init__(self, remote_shell):
92        self.remote_shell = remote_shell
93
94    def monitor_process(self, process_name,
95                        duration_in_seconds=120):
96        # monitor this process and return if it crashes
97        end_time = time.time() + float(duration_in_seconds)
98        last_reported_pid = None
99        while time.time() < end_time:
100            # get the process list
101            process = self.is_process_running(process_name)
102            if process:
103                if not last_reported_pid:
104                    last_reported_pid = process.pid
105                elif not last_reported_pid == process.pid:
106                    message = 'process {0} was restarted. old pid : {1} new pid : {2}'
107                    log.info(message.format(process_name, last_reported_pid, process.pid))
108                    return False
109                    # check if its equal
110            else:
111                # we should have an option to wait for the process
112                # to start during the timeout
113                # process might have crashed
114                log.info("{0}:process {1} is not running or it might have crashed!"
115                                       .format(self.remote_shell.ip, process_name))
116                return False
117            time.sleep(1)
118        #            log.info('process {0} is running'.format(process_name))
119        return True
120
121    def monitor_process_memory(self, process_name,
122                               duration_in_seconds=180,
123                               end=False):
124        # monitor this process and return list of memories in 7 seconds interval
125        end_time = time.time() + float(duration_in_seconds)
126        count = 0
127        vsz = []
128        rss = []
129        while time.time() < end_time and not end:
130            # get the process list
131            process = self.is_process_running(process_name)
132            if process:
133                vsz.append(process.vsz)
134                rss.append(process.rss)
135            else:
136                log.info("{0}:process {1} is not running.  Wait for 2 seconds"
137                                   .format(self.remote_shell.ip, process_name))
138                count += 1
139                time.sleep(2)
140                if count == 5:
141                    log.error("{0}:process {1} is not running at all."
142                                   .format(self.remote_shell.ip, process_name))
143                    exit(1)
144            log.info("sleep for 7 seconds before poll new processes")
145            time.sleep(7)
146        return vsz, rss
147
148    def is_process_running(self, process_name):
149        if getattr(self.remote_shell, "info", None) is None:
150            self.remote_shell.info = self.remote_shell.extract_remote_info()
151
152        if self.remote_shell.info.type.lower() == 'windows':
153             output, error = self.remote_shell.execute_command('tasklist| grep {0}'
154                                                .format(process_name), debug=False)
155             if error or output == [""] or output == []:
156                 return None
157             words = output[0].split(" ")
158             words = filter(lambda x: x != "", words)
159             process = RemoteMachineProcess()
160             process.pid = words[1]
161             process.name = words[0]
162             log.info("process is running on {0}: {1}".format(self.remote_shell.ip, words))
163             return process
164        else:
165            processes = self.remote_shell.get_running_processes()
166            for process in processes:
167                if process.name == process_name:
168                    return process
169                elif process_name in process.args:
170                    return process
171            return None
172
173
174class RemoteMachineShellConnection:
175    _ssh_client = None
176
177    def __init__(self, username='root',
178                 pkey_location='',
179                 ip='', port=''):
180        self.username = username
181        self.use_sudo = True
182        self.nonroot = False
183        """ in nonroot, we could extract Couchbase Server at
184            any directory that non root user could create
185        """
186        self.nr_home_path = "/home/%s/" % self.username
187        if self.username == 'root':
188            self.use_sudo = False
189        elif self.username != "Administrator":
190            self.use_sudo = False
191            self.nonroot = True
192        # let's create a connection
193        self._ssh_client = paramiko.SSHClient()
194        self.ip = ip
195        self.port = port
196        self.remote = (self.ip != "localhost" and self.ip != "127.0.0.1")
197        self._ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
198        log.info('connecting to {0} with username : {1} pem key : {2}'.format(ip, username, pkey_location))
199        try:
200            if self.remote:
201                self._ssh_client.connect(hostname=ip, username=username, key_filename=pkey_location)
202        except paramiko.AuthenticationException:
203            log.info("Authentication failed for {0}".format(self.ip))
204            exit(1)
205        except paramiko.BadHostKeyException:
206            log.info("Invalid Host key for {0}".format(self.ip))
207            exit(1)
208        except Exception:
209            log.info("Can't establish SSH session with {0}".format(self.ip))
210            exit(1)
211
212    def __init__(self, serverInfo):
213        # let's create a connection
214        self.username = serverInfo.ssh_username
215        self.password = serverInfo.ssh_password
216        self.ssh_key = serverInfo.ssh_key
217        self.input = TestInput.TestInputParser.get_test_input(sys.argv)
218        self.use_sudo = True
219        self.nonroot = False
220        self.nr_home_path = "/home/%s/" % self.username
221        if self.username == 'root':
222            self.use_sudo = False
223        elif self.username != "Administrator":
224            self.use_sudo = False
225            self.nonroot = True
226        self._ssh_client = paramiko.SSHClient()
227        self.ip = serverInfo.ip
228        self.remote = (self.ip != "localhost" and self.ip != "127.0.0.1")
229        self.port = serverInfo.port
230        self._ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
231        msg = 'connecting to {0} with username:{1} '
232        log.info(msg.format(serverInfo.ip, serverInfo.ssh_username))
233        # added attempts for connection because of PID check failed.
234        # RNG must be re-initialized after fork() error
235        # That's a paramiko bug
236        max_attempts_connect = 2
237        attempt = 0
238        while True:
239            try:
240                if self.remote and serverInfo.ssh_key == '':
241                    self._ssh_client.connect(hostname=serverInfo.ip.replace('[', '').replace(']',''),
242                                             username=serverInfo.ssh_username,
243                                             password=serverInfo.ssh_password)
244                elif self.remote:
245                    self._ssh_client.connect(hostname=serverInfo.ip.replace('[', '').replace(']',''),
246                                             username=serverInfo.ssh_username,
247                                             key_filename=serverInfo.ssh_key)
248                break
249            except paramiko.AuthenticationException:
250                log.error("Authentication failed")
251                exit(1)
252            except paramiko.BadHostKeyException:
253                log.error("Invalid Host key")
254                exit(1)
255            except Exception as e:
256                if str(e).find('PID check failed. RNG must be re-initialized') != -1 and\
257                        attempt != max_attempts_connect:
258                    log.error("Can't establish SSH session to node {1} :\
259                              {0}. Will try again in 1 sec".format(e, self.ip))
260                    attempt += 1
261                    time.sleep(1)
262                else:
263                    log.error("Can't establish SSH session to node {1} :\
264                                                   {0}".format(e, self.ip))
265                    exit(1)
266        log.info("Connected to {0}".format(serverInfo.ip))
267        """ self.info.distribution_type.lower() == "ubuntu" """
268        self.cmd_ext = ""
269        self.bin_path = LINUX_COUCHBASE_BIN_PATH
270        self.msi = False
271        self.extract_remote_info()
272        os_type = self.info.distribution_type.lower()
273        if os_type == "windows":
274            self.cmd_ext = ".exe"
275            self.bin_path = WIN_COUCHBASE_BIN_PATH
276        elif os_type == "mac":
277            self.nonroot = False
278
279        if self.nonroot:
280            self.bin_path = self.nr_home_path + self.bin_path
281
282    """
283        In case of non root user, we need to switch to root to
284        run command
285    """
286    def connect_with_user(self, user="root"):
287        if self.info.distribution_type.lower() == "mac":
288            log.info("This is Mac Server.  Skip re-connect to it as %s" % user)
289            return
290        max_attempts_connect = 2
291        attempt = 0
292        while True:
293            try:
294                log.info("Connect to node: %s as user: %s" % (self.ip, user))
295                if self.remote and self.ssh_key == '':
296                    self._ssh_client.connect(hostname=self.ip.replace('[', '').replace(']',''),
297                                             username=user,
298                                             password=self.password)
299                break
300            except paramiko.AuthenticationException:
301                log.error("Authentication for root failed")
302                exit(1)
303            except paramiko.BadHostKeyException:
304                log.error("Invalid Host key")
305                exit(1)
306            except Exception as e:
307                if str(e).find('PID check failed. RNG must be re-initialized') != -1 and\
308                        attempt != max_attempts_connect:
309                    log.error("Can't establish SSH session to node {1} as root:\
310                              {0}. Will try again in 1 sec".format(e, self.ip))
311                    attempt += 1
312                    time.sleep(1)
313                else:
314                    log.error("Can't establish SSH session to node {1} :\
315                                                   {0}".format(e, self.ip))
316                    exit(1)
317        log.info("Connected to {0} as {1}".format(self.ip, user))
318
319    def sleep(self, timeout=1, message=""):
320        log.info("{0}:sleep for {1} secs. {2} ...".format(self.ip, timeout, message))
321        time.sleep(timeout)
322
323    def get_running_processes(self):
324        # if its linux ,then parse each line
325        # 26989 ?        00:00:51 pdflush
326        # ps -Ao pid,comm
327        processes = []
328        output, error = self.execute_command('ps -Ao pid,comm,vsz,rss,args', debug=False)
329        if output:
330            for line in output:
331                # split to words
332                words = line.strip().split(' ')
333                words = filter(None, words)
334                if len(words) >= 2:
335                    process = RemoteMachineProcess()
336                    process.pid = words[0]
337                    process.name = words[1]
338                    if words[2].isdigit():
339                        process.vsz = int(words[2])/1024
340                    else:
341                        process.vsz = words[2]
342                    if words[3].isdigit():
343                        process.rss = int(words[3])/1024
344                    else:
345                        process.rss = words[3]
346                    process.args = " ".join(words[4:])
347                    processes.append(process)
348        return processes
349
350    def get_mem_usage_by_process(self, process_name):
351        """Now only linux"""
352        output, error = self.execute_command('ps -e -o %mem,cmd|grep {0}'.format(process_name),
353                                             debug=False)
354        if output:
355            for line in output:
356                if not 'grep' in line.strip().split(' '):
357                    return float(line.strip().split(' ')[0])
358
359    def stop_network(self, stop_time):
360        """
361        Stop the network for given time period and then restart the network
362        on the machine.
363        :param stop_time: Time duration for which the network service needs
364        to be down in the machine
365        :return: Nothing
366        """
367        self.extract_remote_info()
368        os_type = self.info.type.lower()
369        if os_type == "unix" or os_type == "linux":
370            if self.info.distribution_type.lower() == "ubuntu":
371                command = "ifdown -a && sleep {} && ifup -a"
372            else:
373                command = "service network stop && sleep {} && service network " \
374                          "start"
375            output, error = self.execute_command(command.format(stop_time))
376            self.log_command_output(output, error)
377        elif os_type == "windows":
378            command = "net stop Netman && timeout {} && net start Netman"
379            output, error = self.execute_command(command.format(stop_time))
380            self.log_command_output(output, error)
381
382    def stop_network(self, stop_time):
383        """
384        Stop the network for given time period and then restart the network
385        on the machine.
386        :param stop_time: Time duration for which the network service needs
387        to be down in the machine
388        :return: Nothing
389        """
390        self.extract_remote_info()
391        os_type = self.info.type.lower()
392        if os_type == "unix" or os_type == "linux":
393            if self.info.distribution_type.lower() == "ubuntu":
394                command = "ifdown -a && sleep {} && ifup -a"
395            else:
396                command = "nohup service network stop && sleep {} && service network " \
397                          "start &"
398            output, error = self.execute_command(command.format(stop_time))
399            self.log_command_output(output, error)
400        elif os_type == "windows":
401            command = "net stop Netman && timeout {} && net start Netman"
402            output, error = self.execute_command(command.format(stop_time))
403            self.log_command_output(output, error)
404
405    def stop_membase(self):
406        self.extract_remote_info()
407        if self.info.type.lower() == 'windows':
408            log.info("STOP SERVER")
409            o, r = self.execute_command("net stop membaseserver")
410            self.log_command_output(o, r)
411            o, r = self.execute_command("net stop couchbaseserver")
412            self.log_command_output(o, r)
413            self.sleep(10, "Wait to stop service completely")
414        if self.info.type.lower() == "linux":
415            o, r = self.execute_command("/etc/init.d/membase-server stop")
416            self.log_command_output(o, r)
417
418    def start_membase(self):
419        self.extract_remote_info()
420        if self.info.type.lower() == 'windows':
421            o, r = self.execute_command("net start membaseserver")
422            self.log_command_output(o, r)
423        if self.info.type.lower() == "linux":
424            o, r = self.execute_command("/etc/init.d/membase-server start")
425            self.log_command_output(o, r)
426
427    def start_server(self, os="unix"):
428        self.extract_remote_info()
429        os = self.info.type.lower()
430        if not os or os == "centos":
431            os = "unix"
432        if os == "windows":
433            o, r = self.execute_command("net start couchbaseserver")
434            self.log_command_output(o, r)
435        elif os == "unix" or os == "linux":
436            if self.is_couchbase_installed():
437                if self.nonroot:
438                    log.info("Start Couchbase Server with non root method")
439                    o, r = self.execute_command('%s%scouchbase-server \-- -noinput -detached '\
440                                              % (self.nr_home_path, LINUX_COUCHBASE_BIN_PATH))
441                    self.log_command_output(o, r)
442                else:
443                    fv, sv, bn = self.get_cbversion("linux")
444                    if self.info.distribution_version.lower() in SYSTEMD_SERVER \
445                                                 and sv in COUCHBASE_FROM_WATSON:
446                        """from watson, systemd is used in centos 7, suse 12 """
447                        log.info("Running systemd command on this server")
448                        o, r = self.execute_command("systemctl start couchbase-server.service")
449                        self.log_command_output(o, r)
450                    else:
451                        o, r = self.execute_command("/etc/init.d/couchbase-server start")
452                        self.log_command_output(o, r)
453        elif os == "mac":
454            o, r = self.execute_command("open /Applications/Couchbase\ Server.app")
455            self.log_command_output(o, r)
456        else:
457            log.error("don't know operating system or product version")
458
459    def stop_server(self, os="unix"):
460        self.extract_remote_info()
461        os = self.info.distribution_type.lower()
462        if not os or os == "centos":
463            os = "unix"
464        if os == "windows":
465            o, r = self.execute_command("net stop couchbaseserver")
466            self.log_command_output(o, r)
467        elif os == "unix" or os == "linux":
468            if self.is_couchbase_installed():
469                if self.nonroot:
470                    o, r = self.execute_command("%s%scouchbase-server -k"
471                                                % (self.nr_home_path,
472                                                   LINUX_COUCHBASE_BIN_PATH))
473                    self.log_command_output(o, r)
474                else:
475                    fv, sv, bn = self.get_cbversion("linux")
476                    if self.info.distribution_version.lower() in SYSTEMD_SERVER \
477                                                 and sv in COUCHBASE_FROM_WATSON:
478                        """from watson, systemd is used in centos 7, suse 12 """
479                        log.info("Running systemd command on this server")
480                        o, r = self.execute_command("systemctl stop couchbase-server.service")
481                        self.log_command_output(o, r)
482                    else:
483                        o, r = self.execute_command("/etc/init.d/couchbase-server stop"\
484                                                                     , use_channel=True)
485            else:
486                log.info("No Couchbase Server installed yet!")
487        elif os == "mac":
488            cb_process = '/Applications/Couchbase\ Server.app/Contents/MacOS/Couchbase\ Server'
489            cmd = "ps aux | grep {0} | awk '{{print $2}}' | xargs kill -9 ".format(cb_process)
490            o, r = self.execute_command(cmd)
491            self.log_command_output(o, r)
492            o, r = self.execute_command("killall -9 epmd")
493            self.log_command_output(o, r)
494        else:
495            log.error("don't know operating system or product version")
496
497    def restart_couchbase(self):
498        """
499        Restart the couchbase server on the machine.
500        :return: Nothing
501        """
502        self.extract_remote_info()
503        if self.info.type.lower() == 'windows':
504            o, r = self.execute_command("net stop couchbaseserver")
505            self.log_command_output(o, r)
506            o, r = self.execute_command("net start couchbaseserver")
507            self.log_command_output(o, r)
508        if self.info.type.lower() == "linux":
509            fv, sv, bn = self.get_cbversion("linux")
510            if "centos 7" in self.info.distribution_version.lower() \
511                    and sv in COUCHBASE_FROM_WATSON:
512                """from watson, systemd is used in centos 7 """
513                log.info("this node is centos 7.x")
514                o, r = self.execute_command("service couchbase-server restart")
515                self.log_command_output(o, r)
516            else:
517                o, r = self.execute_command(
518                    "/etc/init.d/couchbase-server restart")
519                self.log_command_output(o, r)
520        if self.info.distribution_type.lower() == "mac":
521            o, r = self.execute_command(
522                "open /Applications/Couchbase\ Server.app")
523            self.log_command_output(o, r)
524
525    def stop_schedule_tasks(self):
526        log.info("STOP ALL SCHEDULE TASKS: installme, removeme and upgrademe")
527        output, error = self.execute_command("cmd /c schtasks /end /tn installme")
528        self.log_command_output(output, error)
529        output, error = self.execute_command("cmd /c schtasks /end /tn removeme")
530        self.log_command_output(output, error)
531        output, error = self.execute_command("cmd /c schtasks /end /tn upgrademe")
532        self.log_command_output(output, error)
533
534    def kill_erlang(self, os="unix"):
535        if os == "windows":
536            o, r = self.execute_command("taskkill /F /T /IM epmd.exe*")
537            self.log_command_output(o, r)
538            o, r = self.execute_command("taskkill /F /T /IM erl.exe*")
539            self.log_command_output(o, r)
540            o, r = self.execute_command("tasklist | grep erl.exe")
541            kill_all = False
542            count = 0
543            while len(o) >= 1 and not kill_all:
544                if o and "erl.exe" in o[0]:
545                    self.execute_command("taskkill /F /T /IM erl.exe*")
546                    self.sleep(1)
547                    o, r = self.execute_command("tasklist | grep erl.exe")
548                if len(o) == 0:
549                    kill_all = True
550                    log.info("all erlang processes were killed")
551                else:
552                    count += 1
553                if count == 5:
554                    log.error("erlang process is not killed")
555                    break
556        else:
557            o, r = self.execute_command("killall -9 beam.smp")
558            self.log_command_output(o, r)
559            all_killed = False
560            count = 0
561            while not all_killed and count < 6:
562                process_count = 0
563                self.sleep(2, "wait for erlang processes terminated")
564                out, _ = self.execute_command("ps aux | grep beam.smp")
565                for idx, val in enumerate(out):
566                    if "/opt/couchbase" in val:
567                        process_count += 1
568                if process_count == 0:
569                    all_killed = True
570                if count == 3:
571                    o, r = self.execute_command("killall -9 beam.smp")
572                count += 1
573            if not all_killed:
574                raise Exception("Could not kill erlang process")
575        return o, r
576
577    def kill_cbft_process(self):
578        self.extract_remote_info()
579        if self.info.type.lower() == 'windows':
580            o, r = self.execute_command("taskkill /F /T /IM cbft.exe*")
581            self.log_command_output(o, r)
582        else:
583            o, r = self.execute_command("killall -9 cbft")
584            self.log_command_output(o, r)
585        return o, r
586
587    def kill_memcached(self):
588        self.extract_remote_info()
589        if self.info.type.lower() == 'windows':
590            o, r = self.execute_command("taskkill /F /T /IM memcached*")
591            self.log_command_output(o, r)
592        else:
593            # Changed from kill -9 $(ps aux | grep 'memcached' | awk '{print $2}' as grep was also returning eventing
594            # process which was using memcached-cert
595            o, r = self.execute_command("kill -9 $(ps aux | pgrep 'memcached')")
596            self.log_command_output(o, r)
597        return o, r
598
599    def stop_memcached(self):
600        self.extract_remote_info()
601        if self.info.type.lower() == 'windows':
602            o, r = self.execute_command("taskkill /F /T /IM memcached*")
603            self.log_command_output(o, r)
604        else:
605            o, r = self.execute_command("kill -SIGSTOP $(pgrep memcached)")
606            self.log_command_output(o, r)
607        return o, r
608
609    def start_memcached(self):
610        self.extract_remote_info()
611        if self.info.type.lower() == 'windows':
612            o, r = self.execute_command("taskkill /F /T /IM memcached*")
613            self.log_command_output(o, r)
614        else:
615            o, r = self.execute_command("kill -SIGCONT $(pgrep memcached)")
616            self.log_command_output(o, r)
617        return o, r
618
619    def kill_goxdcr(self):
620        self.extract_remote_info()
621        if self.info.type.lower() == 'windows':
622            o, r = self.execute_command("taskkill /F /T /IM goxdcr*")
623            self.log_command_output(o, r)
624        else:
625            o, r = self.execute_command("killall -9 goxdcr")
626            self.log_command_output(o, r)
627        return o, r
628
629    def kill_eventing_process(self, name):
630        self.extract_remote_info()
631        if self.info.type.lower() == 'windows':
632            o, r = self.execute_command(command="taskkill /F /T /IM {0}*".format(name))
633            self.log_command_output(o, r)
634        else:
635            o, r = self.execute_command(command="killall -9 {0}".format(name))
636            self.log_command_output(o, r)
637        return o, r
638
639    def reboot_node(self):
640        self.extract_remote_info()
641        if self.extract_remote_info().type.lower() == 'windows':
642            o, r = self.execute_command("shutdown -r -f -t 0")
643            self.log_command_output(o, r)
644        elif self.extract_remote_info().type.lower() == 'linux':
645            o, r = self.execute_command("reboot")
646            self.log_command_output(o, r)
647        return o, r
648
649    def change_log_level(self, new_log_level):
650        log.info("CHANGE LOG LEVEL TO %s".format(new_log_level))
651        # ADD NON_ROOT user config_details
652        output, error = self.execute_command("sed -i '/loglevel_default, /c \\{loglevel_default, %s\}'. %s"
653                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
654        self.log_command_output(output, error)
655        output, error = self.execute_command("sed -i '/loglevel_ns_server, /c \\{loglevel_ns_server, %s\}'. %s"
656                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
657        self.log_command_output(output, error)
658        output, error = self.execute_command("sed -i '/loglevel_stats, /c \\{loglevel_stats, %s\}'. %s"
659                                             % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
660        self.log_command_output(output, error)
661        output, error = self.execute_command("sed -i '/loglevel_rebalance, /c \\{loglevel_rebalance, %s\}'. %s"
662                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
663        self.log_command_output(output, error)
664        output, error = self.execute_command("sed -i '/loglevel_cluster, /c \\{loglevel_cluster, %s\}'. %s"
665                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
666        self.log_command_output(output, error)
667        output, error = self.execute_command("sed -i '/loglevel_views, /c \\{loglevel_views, %s\}'. %s"
668                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
669        self.log_command_output(output, error)
670        output, error = self.execute_command("sed -i '/loglevel_error_logger, /c \\{loglevel_error_logger, %s\}'. %s"
671                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
672        self.log_command_output(output, error)
673        output, error = self.execute_command("sed -i '/loglevel_mapreduce_errors, /c \\{loglevel_mapreduce_errors, %s\}'. %s"
674                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
675        self.log_command_output(output, error)
676        output, error = self.execute_command("sed -i '/loglevel_user, /c \\{loglevel_user, %s\}'. %s"
677                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
678        self.log_command_output(output, error)
679        output, error = self.execute_command("sed -i '/loglevel_xdcr, /c \\{loglevel_xdcr, %s\}'. %s"
680                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
681        self.log_command_output(output, error)
682        output, error = self.execute_command("sed -i '/loglevel_menelaus, /c \\{loglevel_menelaus, %s\}'. %s"
683                                            % (new_log_level, testconstants.LINUX_STATIC_CONFIG))
684        self.log_command_output(output, error)
685
686    def configure_log_location(self, new_log_location):
687        mv_logs = testconstants.LINUX_LOG_PATH + '/' + new_log_location
688        print " MV LOGS %s" % mv_logs
689        error_log_tag = "error_logger_mf_dir"
690        # ADD NON_ROOT user config_details
691        log.info("CHANGE LOG LOCATION TO %s".format(mv_logs))
692        output, error = self.execute_command("rm -rf %s" % mv_logs)
693        self.log_command_output(output, error)
694        output, error = self.execute_command("mkdir %s" % mv_logs)
695        self.log_command_output(output, error)
696        output, error = self.execute_command("chown -R couchbase %s" % mv_logs)
697        self.log_command_output(output, error)
698        output, error = self.execute_command("sed -i '/%s, /c \\{%s, \"%s\"\}.' %s"
699                                        % (error_log_tag, error_log_tag, mv_logs, testconstants.LINUX_STATIC_CONFIG))
700        self.log_command_output(output, error)
701
702    def change_stat_periodicity(self, ticks):
703        # ADD NON_ROOT user config_details
704        log.info("CHANGE STAT PERIODICITY TO every %s seconds" % ticks)
705        output, error = self.execute_command("sed -i '$ a\{grab_stats_every_n_ticks, %s}.'  %s"
706        % (ticks, testconstants.LINUX_STATIC_CONFIG))
707        self.log_command_output(output, error)
708
709    def change_port_static(self, new_port):
710        # ADD NON_ROOT user config_details
711        log.info("=========CHANGE PORTS for REST: %s, MCCOUCH: %s,MEMCACHED: %s, MOXI: %s, CAPI: %s==============="
712                % (new_port, new_port + 1, new_port + 2, new_port + 3, new_port + 4))
713        output, error = self.execute_command("sed -i '/{rest_port/d' %s" % testconstants.LINUX_STATIC_CONFIG)
714        self.log_command_output(output, error)
715        output, error = self.execute_command("sed -i '$ a\{rest_port, %s}.' %s"
716                                             % (new_port, testconstants.LINUX_STATIC_CONFIG))
717        self.log_command_output(output, error)
718        output, error = self.execute_command("sed -i '/{mccouch_port/d' %s" % testconstants.LINUX_STATIC_CONFIG)
719        self.log_command_output(output, error)
720        output, error = self.execute_command("sed -i '$ a\{mccouch_port, %s}.' %s"
721                                             % (new_port + 1, testconstants.LINUX_STATIC_CONFIG))
722        self.log_command_output(output, error)
723        output, error = self.execute_command("sed -i '/{memcached_port/d' %s" % testconstants.LINUX_STATIC_CONFIG)
724        self.log_command_output(output, error)
725        output, error = self.execute_command("sed -i '$ a\{memcached_port, %s}.' %s"
726                                             % (new_port + 2, testconstants.LINUX_STATIC_CONFIG))
727        self.log_command_output(output, error)
728        output, error = self.execute_command("sed -i '/{moxi_port/d' %s" % testconstants.LINUX_STATIC_CONFIG)
729        self.log_command_output(output, error)
730        output, error = self.execute_command("sed -i '$ a\{moxi_port, %s}.' %s"
731                                             % (new_port + 3, testconstants.LINUX_STATIC_CONFIG))
732        self.log_command_output(output, error)
733        output, error = self.execute_command("sed -i '/port = /c\port = %s' %s"
734                                             % (new_port + 4, testconstants.LINUX_CAPI_INI))
735        self.log_command_output(output, error)
736        output, error = self.execute_command("rm %s" % testconstants.LINUX_CONFIG_FILE)
737        self.log_command_output(output, error)
738        output, error = self.execute_command("cat %s" % testconstants.LINUX_STATIC_CONFIG)
739        self.log_command_output(output, error)
740
741    def is_couchbase_installed(self):
742        self.extract_remote_info()
743        if self.info.type.lower() == 'windows':
744            if self.file_exists(WIN_CB_PATH, VERSION_FILE):
745                log.info("{0} **** The version file {1} {2}  exists"\
746                          .format(self.ip, WIN_CB_PATH, VERSION_FILE ))
747                # print running process on windows
748                RemoteMachineHelper(self).is_process_running('memcached')
749                RemoteMachineHelper(self).is_process_running('erl')
750                return True
751        elif self.info.distribution_type.lower() == 'mac':
752            output, error = self.execute_command('ls %s%s' % (MAC_CB_PATH, VERSION_FILE))
753            self.log_command_output(output, error)
754            for line in output:
755                if line.find('No such file or directory') == -1:
756                    return True
757        elif self.info.type.lower() == "linux":
758            if self.nonroot:
759                if self.file_exists("/home/%s/" % self.username, NR_INSTALL_LOCATION_FILE):
760                    output, error = self.execute_command("cat %s" % NR_INSTALL_LOCATION_FILE)
761                    if output and output[0]:
762                        log.info("Couchbase Server was installed in non default path %s"
763                                                                            % output[0])
764                        self.nr_home_path = output[0]
765                file_path = self.nr_home_path + LINUX_CB_PATH
766                if self.file_exists(file_path, VERSION_FILE):
767                    log.info("non root couchbase installed at %s " % self.ip)
768                    return True
769            else:
770                if self.file_exists(LINUX_CB_PATH, VERSION_FILE):
771                    log.info("{0} **** The version file {1} {2}  exists".format(self.ip,
772                                                      LINUX_CB_PATH, VERSION_FILE ))
773                    return True
774        return False
775
776    def is_moxi_installed(self):
777        self.extract_remote_info()
778        if self.info.type.lower() == 'windows':
779            log.error('Not implemented')
780        elif self.info.distribution_type.lower() == 'mac':
781            log.error('Not implemented')
782        elif self.info.type.lower() == "linux":
783            if self.file_exists(testconstants.LINUX_MOXI_PATH, 'moxi'):
784                return True
785        return False
786
787    # /opt/moxi/bin/moxi -Z port_listen=11211 -u root -t 4 -O /var/log/moxi/moxi.log
788    def start_moxi(self, ip, bucket, port, user=None, threads=4, log_file="/var/log/moxi.log"):
789        if self.is_couchbase_installed():
790            prod = "couchbase"
791        else:
792            prod = "membase"
793        cli_path = "/opt/" + prod + "/bin/moxi"
794        args = ""
795        args += "http://{0}:8091/pools/default/bucketsStreaming/{1} ".format(ip, bucket)
796        args += "-Z port_listen={0} -u {1} -t {2} -O {3} -d".format(port,
797                                                                    user or prod,
798                                                                    threads, log_file)
799        self.extract_remote_info()
800        if self.info.type.lower() == "linux":
801            o, r = self.execute_command("{0} {1}".format(cli_path, args))
802            self.log_command_output(o, r)
803        else:
804            raise Exception("running standalone moxi is not supported for windows")
805
806    def stop_moxi(self):
807        self.extract_remote_info()
808        if self.info.type.lower() == "linux":
809            o, r = self.execute_command("killall -9 moxi")
810            self.log_command_output(o, r)
811        else:
812            raise Exception("stopping standalone moxi is not supported on windows")
813    def is_url_live(self, url):
814        live_url = False
815        log.info("Check if url {0} is ok".format(url))
816        status = urllib.urlopen(url).getcode()
817        if status == 200:
818            log.info("This url {0} is live".format(url))
819            live_url = True
820        else:
821            mesg = "\n===============\n"\
822                   "        This url {0} \n"\
823                   "        is failed to connect.\n"\
824                   "        Check version in params to make sure it correct pattern or build number.\n"\
825                   "===============\n".format(url)
826            self.stop_current_python_running(mesg)
827        return live_url
828
829    def is_ntp_installed(self):
830        ntp_installed = False
831        do_install = False
832        os_version = ""
833        log.info("Check if ntp is installed")
834        self.extract_remote_info()
835        if self.info.type.lower() == 'linux':
836            log.info("\nThis OS version %s" % self.info.distribution_version.lower())
837            if "centos 7" in self.info.distribution_version.lower():
838                os_version = "centos 7"
839                output, e = self.execute_command("systemctl status ntpd")
840                for line in output:
841                    if "Active: active (running)" in line:
842                        ntp_installed = True
843                if not ntp_installed:
844                    log.info("ntp not installed yet or not run.\n"\
845                             "Let remove any old one and install ntp")
846                    self.execute_command("yum erase -y ntp", debug=False)
847                    self.execute_command("yum install -y ntp", debug=False)
848                    self.execute_command("systemctl start ntpd", debug=False)
849                    self.execute_command("systemctl enable ntpd", debug=False)
850                    self.execute_command("firewall-cmd --add-service=ntp --permanent",
851                                                                          debug=False)
852                    self.execute_command("firewall-cmd --reload", debug=False)
853                    do_install = True
854                timezone, _ = self.execute_command("date")
855                if "PST" not in timezone[0]:
856                    self.execute_command("timedatectl set-timezone America/Los_Angeles",
857                                                                            debug=False)
858            elif "centos release 6" in self.info.distribution_version.lower():
859                os_version = "centos 6"
860                output, e = self.execute_command("/etc/init.d/ntpd status")
861                if not output:
862                    log.info("ntp was not installed on {0} server yet.  "\
863                             "Let install ntp on this server ".format(self.ip))
864                    self.execute_command("yum install -y ntp ntpdate", debug=False)
865                    self.execute_command("chkconfig ntpd on", debug=False)
866                    self.execute_command("ntpdate pool.ntp.org", debug=False)
867                    self.execute_command("/etc/init.d/ntpd start", debug=False)
868                    do_install = True
869                elif output and "ntpd is stopped" in output[0]:
870                    log.info("ntp is not running.  Let remove it and install again in {0}"\
871                                                            .format(self.ip))
872                    self.execute_command("yum erase -y ntp", debug=False)
873                    self.execute_command("yum install -y ntp ntpdate", debug=False)
874                    self.execute_command("chkconfig ntpd on", debug=False)
875                    self.execute_command("ntpdate pool.ntp.org", debug=False)
876                    self.execute_command("/etc/init.d/ntpd start", debug=False)
877                    do_install = True
878                elif output and "is running..." in output[0]:
879                    ntp_installed = True
880                timezone, _ = self.execute_command("date")
881                if "PST" not in timezone[0]:
882                    self.execute_command("cp /etc/localtime /root/old.timezone",
883                                                                    debug=False)
884                    self.execute_command("rm -rf /etc/localtime", debug=False)
885                    self.execute_command("ln -s /usr/share/zoneinfo/America/Los_Angeles "
886                                         "/etc/localtime", debug=False)
887            else:
888                log.info("will add install in other os later, no set do install")
889
890        if do_install:
891            if os_version == "centos 7":
892                output, e = self.execute_command("systemctl status ntpd")
893                for line in output:
894                    if "Active: active (running)" in line:
895                        log.info("ntp is installed and running on this server %s" % self.ip)
896                        ntp_installed = True
897                        break;
898            if os_version == "centos 6":
899                output, e = self.execute_command("/etc/init.d/ntpd status")
900                if output and " is running..." in output[0]:
901                    log.info("ntp is installed and running on this server %s" % self.ip)
902                    ntp_installed = True
903
904        output, _ = self.execute_command("date", debug=False)
905        log.info("\n{0} IP: {1}".format(output, self.ip))
906        if not ntp_installed and "centos" in os_version:
907            mesg = "\n===============\n"\
908                   "        This server {0} \n"\
909                   "        failed to install ntp service.\n"\
910                   "===============\n".format(self.ip)
911            self.stop_current_python_running(mesg)
912
913    def download_build(self, build):
914        return self.download_binary(build.url, build.deliverable_type, build.name,
915                                                latest_url=build.url_latest_build,
916                                                version=build.product_version)
917
918    def disable_firewall(self):
919        self.extract_remote_info()
920        if self.info.type.lower() == "windows":
921            output, error = self.execute_command('netsh advfirewall set publicprofile state off')
922            self.log_command_output(output, error)
923            output, error = self.execute_command('netsh advfirewall set privateprofile state off')
924            self.log_command_output(output, error)
925            # for details see RemoteUtilHelper.enable_firewall for windows
926            output, error = self.execute_command('netsh advfirewall firewall delete rule name="block erl.exe in"')
927            self.log_command_output(output, error)
928            output, error = self.execute_command('netsh advfirewall firewall delete rule name="block erl.exe out"')
929            self.log_command_output(output, error)
930        else:
931            command_1 = "/sbin/iptables -F"
932            command_2 = "/sbin/iptables -t nat -F"
933            if self.nonroot:
934                log.info("\n Non root or non sudo has no right to disable firewall")
935                return
936            output, error = self.execute_command(command_1)
937            self.log_command_output(output, error)
938            output, error = self.execute_command(command_2)
939            self.log_command_output(output, error)
940            self.connect_with_user(user=self.username)
941
942    def download_binary(self, url, deliverable_type, filename, latest_url=None,
943                                              version="", skip_md5_check=True):
944        self.extract_remote_info()
945        self.disable_firewall()
946        file_status = False
947        if self.info.type.lower() == 'windows':
948            self.terminate_processes(self.info, [s for s in WIN_PROCESSES_KILLED])
949            self.terminate_processes(self.info, \
950                                     [s + "-*" for s in COUCHBASE_FROM_VERSION_3])
951            self.disable_firewall()
952            remove_words = ["-rel", ".exe"]
953            for word in remove_words:
954                filename = filename.replace(word, "")
955            """couchbase-server-enterprise_3.5.0-968-windows_amd64
956               couchbase-server-enterprise_4.0.0-1655-windows_amd64
957               sherlock changed from 3.5. to 4.0 """
958            filename_version = ""
959            if len(filename) > 40:
960                if "enterprise" in filename:
961                    filename_version = filename[28:33]
962                elif "community" in filename:
963                    filename_version = filename[27:32]
964            if filename_version in COUCHBASE_FROM_VERSION_4:
965                log.info("This version is {0}".format(filename_version))
966                tmp = filename.split("_")
967                version = tmp[1].replace("-windows", "")
968            else:
969                tmp = filename.split("-")
970                tmp.reverse()
971                version = tmp[1] + "-" + tmp[0]
972
973            exist = self.file_exists('/cygdrive/c/tmp/', '{0}.exe'.format(version))
974            command = "cd /cygdrive/c/tmp;cmd /c 'c:\\automation\\wget.exe "\
975                                        "--no-check-certificate"\
976                                        " -q {0} -O {1}.exe';ls -lh;".format(url, version)
977            file_location = "/cygdrive/c/tmp/"
978            deliverable_type = "exe"
979            if not exist:
980                output, error = self.execute_command(command)
981                self.log_command_output(output, error)
982                if not self.file_exists(file_location, '{0}.exe'.format(version)):
983                    file_status = self.check_and_retry_download_binary(command,
984                                                             file_location, version)
985                return file_status
986            else:
987                log.info('File {0}.exe exist in tmp directory'.format(version))
988                return True
989
990        elif self.info.distribution_type.lower() == 'mac':
991            command = "cd ~/Downloads ; rm -rf couchbase-server* ;"\
992                      " rm -rf Couchbase\ Server.app ; curl -O {0}".format(url)
993            output, error = self.execute_command(command)
994            self.log_command_output(output, error)
995            output, error = self.execute_command('ls -lh  ~/Downloads/%s' % filename)
996            self.log_command_output(output, error)
997            for line in output:
998                if line.find('No such file or directory') == -1:
999                    return True
1000            return False
1001        else:
1002        # try to push this build into
1003        # depending on the os
1004        # build.product has the full name
1005        # first remove the previous file if it exist ?
1006        # fix this :
1007            command1 = "cd /tmp ; D=$(mktemp -d cb_XXXX) ; mv {0} $D ; mv core.* $D ;"\
1008                                     " rm -f * ; mv $D/* . ; rmdir $D".format(filename)
1009            command_root = "cd /tmp;wget -q -O {0} {1};cd /tmp;ls -lh".format(filename, url)
1010            file_location = "/tmp"
1011            output, error = self.execute_command_raw(command1)
1012            self.log_command_output(output, error)
1013            if skip_md5_check:
1014                if self.nonroot:
1015                    output, error = self.execute_command("ls -lh ")
1016                    self.log_command_output(output, error)
1017                    log.info("remove old couchbase server binary ")
1018                    if self.file_exists("/home/%s/" % self.username, NR_INSTALL_LOCATION_FILE):
1019                        output, error = self.execute_command("cat %s"
1020                                             % NR_INSTALL_LOCATION_FILE)
1021                        if output and output[0]:
1022                            log.info("Couchbase Server was installed in non default path %s"
1023                                                                            % output[0])
1024                        self.nr_home_path = output[0]
1025                    self.execute_command_raw('cd %s;rm couchbase-server-*'
1026                                                      % self.nr_home_path)
1027                    if "nr_install_dir" in self.input.test_params and \
1028                                           self.input.test_params["nr_install_dir"]:
1029                        self.nr_home_path = self.nr_home_path + self.input.test_params["nr_install_dir"]
1030                        op, er = self.execute_command("echo %s > %s" % (self.nr_home_path,
1031                                                              NR_INSTALL_LOCATION_FILE))
1032                        self.log_command_output(op, er)
1033                        op, er = self.execute_command("rm -rf %s" % self.nr_home_path)
1034                        self.log_command_output(op, er)
1035                        op, er = self.execute_command("mkdir %s" % self.nr_home_path)
1036                        self.log_command_output(op, er)
1037                    output, error = self.execute_command("ls -lh ")
1038                    self.log_command_output(output, error)
1039                    output, error = self.execute_command_raw('cd {2}; pwd;'
1040                                                             ' wget -q -O {0} {1};ls -lh'
1041                                                             .format(filename, url,
1042                                                                     self.nr_home_path))
1043                    self.log_command_output(output, error)
1044                else:
1045                    output, error = self.execute_command_raw(command_root)
1046                    self.log_command_output(output, error)
1047            else:
1048                log.info('get md5 sum for local and remote')
1049                output, error = self.execute_command_raw('cd /tmp ; rm -f *.md5 *.md5l ; wget -q -O {1}.md5 {0}.md5 ; md5sum {1} > {1}.md5l'.format(url, filename))
1050                self.log_command_output(output, error)
1051                if str(error).find('No such file or directory') != -1 and latest_url != '':
1052                    url = latest_url
1053                    output, error = self.execute_command_raw('cd /tmp ; rm -f *.md5 *.md5l ; wget -q -O {1}.md5 {0}.md5 ; md5sum {1} > {1}.md5l'.format(url, filename))
1054                log.info('comparing md5 sum and downloading if needed')
1055                output, error = self.execute_command_raw('cd /tmp;diff {0}.md5 {0}.md5l || wget -q -O {0} {1};rm -f *.md5 *.md5l'.format(filename, url))
1056                self.log_command_output(output, error)
1057            # check if the file exists there now ?
1058            if self.nonroot:
1059                """ binary is saved at current user directory """
1060                return self.file_exists(self.nr_home_path, filename)
1061            else:
1062                file_status = self.file_exists(file_location, filename)
1063                if not file_status:
1064                    file_status = self.check_and_retry_download_binary(command_root,
1065                                                                      file_location,
1066                                                                      version)
1067                return file_status
1068            # for linux environment we can just
1069            # figure out what version , check if /tmp/ has the
1070            # binary and then return True if binary is installed
1071
1072
1073    def get_file(self, remotepath, filename, todir):
1074        if self.file_exists(remotepath, filename):
1075            if self.remote:
1076                sftp = self._ssh_client.open_sftp()
1077                try:
1078                    filenames = sftp.listdir(remotepath)
1079                    for name in filenames:
1080                        if name == filename:
1081                            sftp.get('{0}/{1}'.format(remotepath, filename), todir)
1082                            sftp.close()
1083                            return True
1084                    sftp.close()
1085                    return False
1086                except IOError:
1087                    return False
1088        else:
1089            os.system("cp {0} {1}".format('{0}/{1}'.format(remotepath, filename), todir))
1090
1091    def read_remote_file(self, remote_path, filename):
1092        if self.file_exists(remote_path, filename):
1093            if self.remote:
1094                sftp = self._ssh_client.open_sftp()
1095                remote_file = sftp.open('{0}/{1}'.format(remote_path, filename))
1096                try:
1097                    out = remote_file.readlines()
1098                finally:
1099                    remote_file.close()
1100                return out
1101            else:
1102                txt = open('{0}/{1}'.format(remote_path, filename))
1103                return txt.read()
1104        return None
1105
1106    def write_remote_file(self, remote_path, filename, lines):
1107        cmd = 'echo "%s" > %s/%s' % (''.join(lines), remote_path, filename)
1108        self.execute_command(cmd)
1109
1110    def write_remote_file_single_quote(self, remote_path, filename, lines):
1111        cmd = 'echo \'%s\' > %s/%s' % (''.join(lines), remote_path, filename)
1112        self.execute_command(cmd)
1113
1114    def create_whitelist(self, path, whitelist):
1115        if not os.path.exists(path):
1116            self.execute_command("mkdir %s" % path)
1117        filepath = os.path.join(path, "curl_whitelist.json")
1118        if os.path.exists(filepath):
1119            os.remove(filepath)
1120        file_data = json.dumps(whitelist)
1121        self.create_file(filepath, file_data)
1122
1123    def remove_directory(self, remote_path):
1124        if self.remote:
1125            sftp = self._ssh_client.open_sftp()
1126            try:
1127                log.info("removing {0} directory...".format(remote_path))
1128                sftp.rmdir(remote_path)
1129            except IOError:
1130                return False
1131            finally:
1132                sftp.close()
1133        else:
1134            try:
1135                p = Popen("rm -rf {0}".format(remote_path) , shell=True, stdout=PIPE, stderr=PIPE)
1136                stdout, stderro = p.communicate()
1137            except IOError:
1138                return False
1139        return True
1140
1141    def rmtree(self, sftp, remote_path, level=0):
1142        for f in sftp.listdir_attr(remote_path):
1143            rpath = remote_path + "/" + f.filename
1144            if stat.S_ISDIR(f.st_mode):
1145                self.rmtree(sftp, rpath, level=(level + 1))
1146            else:
1147                rpath = remote_path + "/" + f.filename
1148                print('removing %s' % (rpath))
1149                sftp.remove(rpath)
1150        print('removing %s' % (remote_path))
1151        sftp.rmdir(remote_path)
1152
1153    def remove_directory_recursive(self, remote_path):
1154        if self.remote:
1155            sftp = self._ssh_client.open_sftp()
1156            try:
1157                log.info("removing {0} directory...".format(remote_path))
1158                self.rmtree(sftp, remote_path)
1159            except IOError:
1160                return False
1161            finally:
1162                sftp.close()
1163        else:
1164            try:
1165                p = Popen("rm -rf {0}".format(remote_path) , shell=True, stdout=PIPE, stderr=PIPE)
1166                p.communicate()
1167            except IOError:
1168                return False
1169        return True
1170
1171    def list_files(self, remote_path):
1172        if self.remote:
1173            sftp = self._ssh_client.open_sftp()
1174            files = []
1175            try:
1176                file_names = sftp.listdir(remote_path)
1177                for name in file_names:
1178                    files.append({'path': remote_path, 'file': name})
1179                sftp.close()
1180            except IOError:
1181                return []
1182            return files
1183        else:
1184            p = Popen("ls {0}".format(remote_path) , shell=True, stdout=PIPE, stderr=PIPE)
1185            files, stderro = p.communicate()
1186            return files
1187
1188    def file_ends_with(self, remotepath, pattern):
1189        """
1190         Check if file ending with this pattern is present in remote machine
1191        """
1192        sftp = self._ssh_client.open_sftp()
1193        files_matched = []
1194        try:
1195            file_names = sftp.listdir(remotepath)
1196            for name in file_names:
1197                if name.endswith(pattern):
1198                    files_matched.append("{0}/{1}".format(remotepath, name))
1199        except IOError:
1200            # ignore this error
1201            pass
1202        sftp.close()
1203        if len(files_matched) > 0:
1204            log.info("found these files : {0}".format(files_matched))
1205        return files_matched
1206
1207    # check if this file exists in the remote
1208    # machine or not
1209    def file_starts_with(self, remotepath, pattern):
1210        sftp = self._ssh_client.open_sftp()
1211        files_matched = []
1212        try:
1213            file_names = sftp.listdir(remotepath)
1214            for name in file_names:
1215                if name.startswith(pattern):
1216                    files_matched.append("{0}/{1}".format(remotepath, name))
1217        except IOError:
1218            # ignore this error
1219            pass
1220        sftp.close()
1221        if len(files_matched) > 0:
1222            log.info("found these files : {0}".format(files_matched))
1223        return files_matched
1224
1225    def file_exists(self, remotepath, filename, pause_time=30):
1226        sftp = self._ssh_client.open_sftp()
1227        try:
1228            filenames = sftp.listdir_attr(remotepath)
1229            for name in filenames:
1230                if filename in name.filename and int(name.st_size) > 0:
1231                    sftp.close()
1232                    return True
1233                elif filename in name.filename and int(name.st_size) == 0:
1234                    if name.filename == NR_INSTALL_LOCATION_FILE:
1235                        continue
1236                    log.info("File {0} will be deleted".format(filename))
1237                    if not remotepath.endswith("/"):
1238                        remotepath += "/"
1239                    self.execute_command("rm -rf {0}*{1}*".format(remotepath,filename))
1240                    self.sleep(pause_time, "** Network or sever may be busy. **"\
1241                                           "\nWait {0} seconds before executing next instrucion"\
1242                                                                             .format(pause_time))
1243
1244            sftp.close()
1245            return False
1246        except IOError:
1247            return False
1248
1249    def delete_file(self, remotepath, filename):
1250        sftp = self._ssh_client.open_sftp()
1251        delete_file = False
1252        try:
1253            filenames = sftp.listdir_attr(remotepath)
1254            for name in filenames:
1255                if name.filename == filename:
1256                    log.info("File {0} will be deleted".format(filename))
1257                    sftp.remove(remotepath + filename)
1258                    delete_file = True
1259                    break
1260            if delete_file:
1261                """ verify file is deleted """
1262                filenames = sftp.listdir_attr(remotepath)
1263                for name in filenames:
1264                    if name.filename == filename:
1265                        log.error("fail to remove file %s " % filename)
1266                        delete_file = False
1267                        break
1268            sftp.close()
1269            return delete_file
1270        except IOError:
1271            return False
1272
1273    def download_binary_in_win(self, url, version, msi_install=False):
1274        self.terminate_processes(self.info, [s for s in WIN_PROCESSES_KILLED])
1275        self.terminate_processes(self.info, \
1276                                 [s + "-*" for s in COUCHBASE_FROM_VERSION_3])
1277        self.disable_firewall()
1278        version = version.replace("-rel", "")
1279        deliverable_type = "exe"
1280        if url and url.split(".")[-1] == "msi":
1281            deliverable_type = "msi"
1282        exist = self.file_exists('/cygdrive/c/tmp/', version)
1283        log.info('**** about to do the wget ****')
1284        command = "cd /cygdrive/c/tmp;cmd /c 'c:\\automation\\wget.exe "\
1285                        " --no-check-certificate"\
1286                        " -q {0} -O {1}.{2}';ls -lh;".format(url, version, deliverable_type)
1287        file_location = "/cygdrive/c/tmp/"
1288        if not exist:
1289            output, error = self.execute_command(command)
1290            self.log_command_output(output, error)
1291        else:
1292            log.info('File {0}.{1} exist in tmp directory'.format(version,
1293                                                                  deliverable_type))
1294        file_status = self.file_exists('/cygdrive/c/tmp/', version)
1295        if not file_status:
1296            file_status = self.check_and_retry_download_binary(command, file_location, version)
1297        return file_status
1298
1299    def check_and_retry_download_binary(self, command, file_location,
1300                                        version, time_of_try=3):
1301        count = 1
1302        file_status = self.file_exists(file_location, version, pause_time=60)
1303        while count <= time_of_try and not file_status:
1304            log.info(" *** try to download binary again {0} time(s)".format(count))
1305            output, error = self.execute_command(command)
1306            self.log_command_output(output, error)
1307            file_status = self.file_exists(file_location, version, pause_time=60)
1308            count += 1
1309            if not file_status and count == 3:
1310                log.error("build {0} did not download completely at server {1}"
1311                                                   .format(version, self.ip))
1312                mesg = "stop job due to failure download build {0} at {1} " \
1313                                                     .format(version, self.ip)
1314                self.stop_current_python_running(mesg)
1315        return file_status
1316
1317    def copy_file_local_to_remote(self, src_path, des_path):
1318        sftp = self._ssh_client.open_sftp()
1319        try:
1320            sftp.put(src_path, des_path)
1321        except IOError:
1322            log.error('Can not copy file')
1323        finally:
1324            sftp.close()
1325
1326    def copy_file_remote_to_local(self, rem_path, des_path):
1327        sftp = self._ssh_client.open_sftp()
1328        try:
1329            sftp.get(rem_path, des_path)
1330        except IOError as e:
1331            if e:
1332                print e
1333            log.error('Can not copy file')
1334        finally:
1335            sftp.close()
1336
1337
1338    # copy multi files from local to remote server
1339    def copy_files_local_to_remote(self, src_path, des_path):
1340        files = os.listdir(src_path)
1341        log.info("copy files from {0} to {1}".format(src_path, des_path))
1342        # self.execute_batch_command("cp -r  {0}/* {1}".format(src_path, des_path))
1343        for file in files:
1344            if file.find("wget") != 1:
1345                a = ""
1346            full_src_path = os.path.join(src_path, file)
1347            full_des_path = os.path.join(des_path, file)
1348            self.copy_file_local_to_remote(full_src_path, full_des_path)
1349
1350
1351    # create a remote file from input string
1352    def create_file(self, remote_path, file_data):
1353        output, error = self.execute_command("echo '{0}' > {1}".format(file_data, remote_path))
1354
1355    def find_file(self, remote_path, file):
1356        sftp = self._ssh_client.open_sftp()
1357        try:
1358            files = sftp.listdir(remote_path)
1359            for name in files:
1360                if name == file:
1361                    found_it = os.path.join(remote_path, name)
1362                    log.info("File {0} was found".format(found_it))
1363                    return found_it
1364            else:
1365                log.error('File(s) name in {0}'.format(remote_path))
1366                for name in files:
1367                    log.info(name)
1368                log.error('Can not find {0}'.format(file))
1369        except IOError:
1370            pass
1371        sftp.close()
1372
1373    def find_build_version(self, path_to_version, version_file, product):
1374        sftp = self._ssh_client.open_sftp()
1375        ex_type = "exe"
1376        if self.file_exists(WIN_CB_PATH, VERSION_FILE):
1377            path_to_version = WIN_CB_PATH
1378        else:
1379            path_to_version = testconstants.WIN_MB_PATH
1380        try:
1381            log.info(path_to_version)
1382            f = sftp.open(os.path.join(path_to_version, VERSION_FILE), 'r+')
1383            tmp_str = f.read().strip()
1384            full_version = tmp_str.replace("-rel", "")
1385            if full_version == "1.6.5.4-win64":
1386                full_version = "1.6.5.4"
1387            build_name = short_version = full_version
1388            return build_name, short_version, full_version
1389        except IOError:
1390            log.error('Can not read version file')
1391        sftp.close()
1392
1393    def find_windows_info(self):
1394        if self.remote:
1395            found = self.find_file("/cygdrive/c/tmp", "windows_info.txt")
1396            if isinstance(found, basestring):
1397                if self.remote:
1398
1399                    sftp = self._ssh_client.open_sftp()
1400                    try:
1401                        f = sftp.open(found)
1402                        log.info("get windows information")
1403                        info = {}
1404                        for line in f:
1405                            (key, value) = line.split('=')
1406                            key = key.strip(' \t\n\r')
1407                            value = value.strip(' \t\n\r')
1408                            info[key] = value
1409                        return info
1410                    except IOError:
1411                        log.error("can not find windows info file")
1412                    sftp.close()
1413            else:
1414                return self.create_windows_info()
1415        else:
1416            try:
1417                txt = open("{0}/{1}".format("/cygdrive/c/tmp", "windows_info.txt"))
1418                log.info("get windows information")
1419                info = {}
1420                for line in txt.read():
1421                    (key, value) = line.split('=')
1422                    key = key.strip(' \t\n\r')
1423                    value = value.strip(' \t\n\r')
1424                    info[key] = value
1425                return info
1426            except IOError:
1427                log.error("can not find windows info file")
1428
1429    def create_windows_info(self):
1430            systeminfo = self.get_windows_system_info()
1431            info = {}
1432            info["os_name"] = "2k8"
1433            if "OS Name" in systeminfo:
1434                info["os"] = systeminfo["OS Name"].find("indows") and "windows" or "NONE"
1435            if systeminfo["OS Name"].find("2008 R2") != -1:
1436                info["os_name"] = 2008
1437            elif systeminfo["OS Name"].find("2016") != -1:
1438                info["os_name"] = 2016
1439            if "System Type" in systeminfo:
1440                info["os_arch"] = systeminfo["System Type"].find("64") and "x86_64" or "NONE"
1441            info.update(systeminfo)
1442            self.execute_batch_command("rm -rf  /cygdrive/c/tmp/windows_info.txt")
1443            self.execute_batch_command("touch  /cygdrive/c/tmp/windows_info.txt")
1444            sftp = self._ssh_client.open_sftp()
1445            try:
1446                f = sftp.open('/cygdrive/c/tmp/windows_info.txt', 'w')
1447                content = ''
1448                for key in sorted(info.keys()):
1449                    content += '{0} = {1}\n'.format(key, info[key])
1450                f.write(content)
1451                log.info("/cygdrive/c/tmp/windows_info.txt was created with content: {0}".format(content))
1452            except IOError:
1453                log.error('Can not write windows_info.txt file')
1454            finally:
1455                sftp.close()
1456            return info
1457
1458    # Need to add new windows register ID in testconstant file when
1459    # new couchbase server version comes out.
1460    def create_windows_capture_file(self, task, product, version):
1461        src_path = "resources/windows/automation"
1462        des_path = "/cygdrive/c/automation"
1463
1464        # remove dot in version (like 2.0.0 ==> 200)
1465        reg_version = version[0:5:2]
1466        reg_id = WIN_REGISTER_ID[reg_version]
1467        uuid_name = uuid.uuid4()
1468
1469        if task == "install":
1470            template_file = "cb-install.wct"
1471            file = "{0}_{1}_install.iss".format(uuid_name, self.ip)
1472            #file = "{0}_install.iss".format(self.ip)
1473        elif task == "uninstall":
1474            template_file = "cb-uninstall.wct"
1475            file = "{0}_{1}_uninstall.iss".format(uuid_name, self.ip)
1476            #file = "{0}_uninstall.iss".format(self.ip)
1477
1478        # create in/uninstall file from windows capture template (wct) file
1479        full_src_path_template = os.path.join(src_path, template_file)
1480        full_src_path = os.path.join(src_path, file)
1481        full_des_path = os.path.join(des_path, file)
1482
1483        f1 = open(full_src_path_template, 'r')
1484        f2 = open(full_src_path, 'w')
1485        """ replace ####### with reg ID to install/uninstall """
1486        if "2.2.0-837" in version:
1487            reg_id = "2B630EB8-BBC7-6FE4-C9B8-D8843EB1EFFA"
1488        log.info("register ID: {0}".format(reg_id))
1489        for line in f1:
1490            line = line.replace("#######", reg_id)
1491            if product == "mb" and task == "install":
1492                line = line.replace("Couchbase", "Membase")
1493            f2.write(line)
1494        f1.close()
1495        f2.close()
1496
1497        self.copy_file_local_to_remote(full_src_path, full_des_path)
1498        """ remove capture file from source after copy to destination """
1499        self.sleep(4, "wait for remote copy completed")
1500        """ need to implement to remove only at the end
1501            of installation """
1502        #os.remove(full_src_path)
1503        return uuid_name
1504
1505    def get_windows_system_info(self):
1506        try:
1507            info = {}
1508            o, _ = self.execute_batch_command('systeminfo')
1509            for line in o:
1510                line_list = line.split(':')
1511                if len(line_list) > 2:
1512                    if line_list[0] == 'Virtual Memory':
1513                        key = "".join(line_list[0:2])
1514                        value = " ".join(line_list[2:])
1515                    else:
1516                        key = line_list[0]
1517                        value = " ".join(line_list[1:])
1518                elif len(line_list) == 2:
1519                    (key, value) = line_list
1520                else:
1521                    continue
1522                key = key.strip(' \t\n\r')
1523                if key.find("[") != -1:
1524                    info[key_prev] += '|' + key + value.strip(' |')
1525                else:
1526                    value = value.strip(' |')
1527                    info[key] = value
1528                    key_prev = key
1529            return info
1530        except Exception as ex:
1531            log.error("error {0} appeared during getting  windows info".format(ex))
1532
1533    # this function used to modify bat file to run task schedule in windows
1534    def modify_bat_file(self, remote_path, file_name, name, version, task):
1535        found = self.find_file(remote_path, file_name)
1536        sftp = self._ssh_client.open_sftp()
1537        capture_iss_file = ""
1538
1539        product_version = ""
1540        if version[:5] in MEMBASE_VERSIONS:
1541            product_version = version[:5]
1542            name = "mb"
1543        elif version[:5] in COUCHBASE_VERSIONS:
1544            product_version = version[:5]
1545            name = "cb"
1546        else:
1547            log.error('Windows automation does not support {0} version yet'\
1548                                                               .format(version))
1549            sys.exit()
1550
1551        if "upgrade" not in task:
1552            uuid_name = self.create_windows_capture_file(task, name, version)
1553        try:
1554            f = sftp.open(found, 'w')
1555            name = name.strip()
1556            version = version.strip()
1557            if task == "upgrade":
1558                content = 'c:\\tmp\{3}.exe /s -f1c:\\automation\{0}_{1}_{2}.iss' \
1559                                .format(name, product_version, task, version)
1560            else:
1561                content = 'c:\\tmp\{0}.exe /s -f1c:\\automation\{3}_{2}_{1}.iss' \
1562                             .format(version, task, self.ip, uuid_name)
1563            log.info("create {0} task with content:{1}".format(task, content))
1564            f.write(content)
1565            log.info('Successful write to {0}'.format(found))
1566            if "upgrade" not in task:
1567                capture_iss_file = '{0}_{1}_{2}.iss'.format(uuid_name, self.ip, task)
1568        except IOError:
1569            log.error('Can not write build name file to bat file {0}'.format(found))
1570        sftp.close()
1571        return capture_iss_file
1572
1573    def compact_vbuckets(self, vbuckets, nodes, upto_seq, cbadmin_user="cbadminbucket",
1574                                                          cbadmin_password="password"):
1575        """
1576            compact each vbucket with cbcompact tools
1577        """
1578        for node in nodes:
1579            log.info("Purge 'deleted' keys in all %s vbuckets in node %s.  It will take times "
1580                                                                         % (vbuckets, node.ip))
1581            for vbucket in range(0, vbuckets):
1582                cmd = "%scbcompact%s %s:11210 compact %d -u %s -p %s "\
1583                      "--dropdeletes --purge-only-upto-seq=%d" \
1584                                                  % (self.bin_path,
1585                                                     self.cmd_ext,
1586                                                     node.ip, vbucket,
1587                                                     cbadmin_user,
1588                                                     cbadmin_password,
1589                                                     upto_seq)
1590                self.execute_command(cmd, debug=False)
1591        log.info("done compact")
1592
1593    def set_vbuckets_win(self, build, vbuckets):
1594        bin_path = WIN_COUCHBASE_BIN_PATH
1595        bin_path = bin_path.replace("\\", "")
1596        build = build.replace('-', '.')
1597        src_file = bin_path + "install/cb_winsvc_start_{0}.bat".format(build)
1598        des_file = "/tmp/cb_winsvc_start_{0}_{1}.bat".format(build, self.ip)
1599        local_file = "/tmp/cb_winsvc_start_{0}_{1}.bat_tmp".format(build, self.ip)
1600
1601        self.copy_file_remote_to_local(src_file, des_file)
1602        f1 = open(des_file, "r")
1603        f2 = open(local_file, "w")
1604        """ when install new cb server on windows, there is not
1605            env COUCHBASE_NUM_VBUCKETS yet.  We need to insert this
1606            env to service_register.bat right after  ERL_FULLSWEEP_AFTER 512
1607            like -env ERL_FULLSWEEP_AFTER 512 -env COUCHBASE_NUM_VBUCKETS vbuckets
1608            where vbucket is params passed to function when run install scripts """
1609        for line in f1:
1610            if "-env COUCHBASE_NUM_VBUCKETS " in line:
1611                tmp1 = line.split("COUCHBASE_NUM_VBUCKETS")
1612                tmp2 = tmp1[1].strip().split(" ")
1613                log.info("set vbuckets of node {0} to {1}" \
1614                                 .format(self.ip, vbuckets))
1615                tmp2[0] = vbuckets
1616                tmp1[1] = " ".join(tmp2)
1617                line = "COUCHBASE_NUM_VBUCKETS ".join(tmp1)
1618            elif "-env ERL_FULLSWEEP_AFTER 512" in line:
1619                log.info("set vbuckets of node {0} to {1}" \
1620                                 .format(self.ip, vbuckets))
1621                line = line.replace("-env ERL_FULLSWEEP_AFTER 512", \
1622                  "-env ERL_FULLSWEEP_AFTER 512 -env COUCHBASE_NUM_VBUCKETS {0}" \
1623                                 .format(vbuckets))
1624            f2.write(line)
1625        f1.close()
1626        f2.close()
1627        self.copy_file_local_to_remote(local_file, src_file)
1628
1629        """ re-register new setup to cb server """
1630        o, r = self.execute_command(WIN_COUCHBASE_BIN_PATH + "install/cb_winsvc_stop_{0}.bat".format(build))
1631        self.log_command_output(o, r)
1632        o, r = self.execute_command(WIN_COUCHBASE_BIN_PATH + "install/cb_winsvc_start_{0}.bat".format(build))
1633        self.log_command_output(o, r)
1634        self.sleep(20, "wait for cb server start completely after reset vbuckets!")
1635
1636        """ remove temporary files on slave """
1637        os.remove(local_file)
1638        os.remove(des_file)
1639
1640    def set_fts_query_limit_win(self, build, name, value, ipv6=False):
1641        bin_path = WIN_COUCHBASE_BIN_PATH
1642        bin_path = bin_path.replace("\\", "")
1643        build = build.replace('-', '.')
1644        src_file = bin_path + "install/cb_winsvc_start_{0}.bat".format(build)
1645        des_file = "/tmp/cb_winsvc_start_{0}_{1}.bat".format(build, self.ip)
1646        local_file = "/tmp/cb_winsvc_start_{0}_{1}.bat_tmp".format(build, self.ip)
1647
1648        self.copy_file_remote_to_local(src_file, des_file)
1649        f1 = open(des_file, "r")
1650        f2 = open(local_file, "w")
1651        """ when install new cb server on windows, there is not
1652            env CBFT_ENV_OPTIONS yet.  We need to insert this
1653            env to service_register.bat right after  ERL_FULLSWEEP_AFTER 512
1654            like -env ERL_FULLSWEEP_AFTER 512 -env CBFT_ENV_OPTIONS value
1655            where value is params passed to function when run install scripts """
1656        for line in f1:
1657            if "-env CBFT_ENV_OPTIONS " in line:
1658                tmp1 = line.split("CBFT_ENV_OPTIONS")
1659                tmp2 = tmp1[1].strip().split(" ")
1660                log.info("set CBFT_ENV_OPTIONS of node {0} to {1}" \
1661                                 .format(self.ip, value))
1662                tmp2[0] = value
1663                tmp1[1] = " ".join(tmp2)
1664                line = "CBFT_ENV_OPTIONS ".join(tmp1)
1665            elif "-env ERL_FULLSWEEP_AFTER 512" in line:
1666                log.info("set CBFT_ENV_OPTIONS of node {0} to {1}" \
1667                                 .format(self.ip, value))
1668                line = line.replace("-env ERL_FULLSWEEP_AFTER 512", \
1669                  "-env ERL_FULLSWEEP_AFTER 512 -env {0} {1}" \
1670                                 .format(name, value))
1671            f2.write(line)
1672        f1.close()
1673        f2.close()
1674        self.copy_file_local_to_remote(local_file, src_file)
1675
1676        """ re-register new setup to cb server """
1677        o, r = self.execute_command(WIN_COUCHBASE_BIN_PATH + "install/cb_winsvc_stop_{0}.bat ".format(build))
1678        self.log_command_output(o, r)
1679        cmd = WIN_COUCHBASE_BIN_PATH + "install/cb_winsvc_start_{0}.bat".format(build)
1680        if ipv6:
1681            cmd += " true"
1682        o, r = self.execute_command(cmd)
1683        self.log_command_output(o, r)
1684        self.sleep(20, "wait for cb server start completely after setting CBFT_ENV_OPTIONS")
1685
1686        """ remove temporary files on slave """
1687        os.remove(local_file)
1688        os.remove(des_file)
1689
1690    def create_directory(self, remote_path):
1691        sftp = self._ssh_client.open_sftp()
1692        try:
1693            sftp.stat(remote_path)
1694        except IOError, e:
1695            if e[0] == 2:
1696                log.info("Directory at {0} DOES NOT exist. We will create on here".format(remote_path))
1697                sftp.mkdir(remote_path)
1698                sftp.close()
1699                return False
1700            raise
1701        else:
1702            log.error("Directory at {0} DOES exist. Fx returns True".format(remote_path))
1703            return True
1704
1705    # this function will remove the automation directory in windows
1706    def create_multiple_dir(self, dir_paths):
1707        sftp = self._ssh_client.open_sftp()
1708        try:
1709            for dir_path in dir_paths:
1710                if dir_path != '/cygdrive/c/tmp':
1711                    output = self.remove_directory('/cygdrive/c/automation')
1712                    if output:
1713                        log.info("{0} directory is removed.".format(dir_path))
1714                    else:
1715                        log.error("Can not delete {0} directory or directory {0} does not exist.".format(dir_path))
1716                self.create_directory(dir_path)
1717            sftp.close()
1718        except IOError:
1719            pass
1720
1721    def couchbase_upgrade(self, build, save_upgrade_config=False, forcefully=False,
1722                                            fts_query_limit=None, debug_logs=False):
1723        server_type = None
1724        success = True
1725        if fts_query_limit is None:
1726            fts_query_limit = 10000000
1727        start_server_after_install = True
1728        track_words = ("warning", "error", "fail")
1729        if build.name.lower().find("membase") != -1:
1730            server_type = 'membase'
1731            abbr_product = "mb"
1732        elif build.name.lower().find("couchbase") != -1:
1733            server_type = 'couchbase'
1734            abbr_product = "cb"
1735        else:
1736            raise Exception("its not a membase or couchbase?")
1737        # upgrade couchbase server
1738        nonroot_path_start = "/"
1739        self.extract_remote_info()
1740        log.info('deliverable_type : {0}'.format(self.info.deliverable_type))
1741        log.info('/tmp/{0} or /tmp/{1}'.format(build.name, build.product))
1742        command = ''
1743        if self.info.type.lower() == 'windows':
1744                print "build name in couchbase upgrade    ", build.product_version
1745                self.couchbase_upgrade_win(self.info.architecture_type, \
1746                                     self.info.windows_name, build.product_version)
1747                log.info('********* continue upgrade process **********')
1748
1749        elif self.info.deliverable_type == 'rpm':
1750            # run rpm -i to install
1751            if save_upgrade_config:
1752                self.couchbase_uninstall()
1753                install_command = 'rpm -i /tmp/{0}'.format(build.name)
1754                command = 'INSTALL_UPGRADE_CONFIG_DIR=/opt/couchbase/var/lib/membase/config {0}'\
1755                                             .format(install_command)
1756            else:
1757                command = 'rpm -U /tmp/{0}'.format(build.name)
1758                if forcefully:
1759                    command = 'rpm -U --force /tmp/{0}'.format(build.name)
1760        elif self.info.deliverable_type == 'deb':
1761            if save_upgrade_config:
1762                self.couchbase_uninstall()
1763                install_command = 'dpkg -i /tmp/{0}'.format(build.name)
1764                command = 'INSTALL_UPGRADE_CONFIG_DIR=/opt/couchbase/var/lib/membase/config {0}'\
1765                                             .format(install_command)
1766            else:
1767                command = 'dpkg -i /tmp/{0}'.format(build.name)
1768                if forcefully:
1769                    command = 'dpkg -i --force /tmp/{0}'.format(build.name)
1770        output, error = self.execute_command(command, use_channel=True)
1771        if debug_logs:
1772            self.log_command_output(output, error)
1773        else:
1774            mesg = "You have successfully installed Couchbase Server."
1775            success_upgrade = self._check_output(mesg, output)
1776            if success_upgrade:
1777                output = []
1778                output.append("You have successfully installed Couchbase Server.")
1779        linux = ["deb", "rpm"]
1780        if float(build.product_version[:3]) >= 5.1:
1781            if self.info.deliverable_type in linux:
1782                o, e = \
1783                    self.execute_command("sed -i 's/export PATH/export PATH\\n"
1784                            "export CBFT_ENV_OPTIONS=bleveMaxResultWindow={1}/'\
1785                            {2}opt/{0}/bin/{0}-server".format(server_type, int(fts_query_limit),
1786                                                              nonroot_path_start))
1787                success &= self.log_command_output(o, e, track_words)
1788                self.sleep(5, "wait for server up before stop it.")
1789                self.stop_couchbase()
1790                self.start_couchbase()
1791        return output, error
1792
1793    def _check_output(self, word_check, output):
1794        found = False
1795        if len(output) >=1 :
1796            for x in output:
1797                if word_check.lower() in x.lower():
1798                    log.info("Found '{0}' in output".format(word_check))
1799                    found = True
1800                    break
1801        return found
1802
1803    def couchbase_upgrade_win(self, architecture, windows_name, version):
1804        task = "upgrade"
1805        bat_file = "upgrade.bat"
1806        deleted = False
1807        self.modify_bat_file('/cygdrive/c/automation', bat_file, 'cb',\
1808                                                         version, task)
1809        self.stop_schedule_tasks()
1810        self.remove_win_backup_dir()
1811        self.remove_win_collect_tmp()
1812        output, error = self.execute_command("cat "
1813                        "'/cygdrive/c/Program Files/Couchbase/Server/VERSION.txt'")
1814        log.info("version to upgrade from: {0} to {1}".format(output, version))
1815        log.info('before running task schedule upgrademe')
1816        if '1.8.0' in str(output):
1817            """ run installer in second time as workaround for upgrade 1.8.0 only:
1818            #   Installer needs to update registry value in order to upgrade
1819            #   from the previous version.
1820            #   Please run installer again to continue."""
1821            output, error = \
1822                        self.execute_command("cmd /c schtasks /run /tn upgrademe")
1823            self.log_command_output(output, error)
1824            self.sleep(200, "because upgrade version is {0}".format(output))
1825            output, error = self.execute_command("cmd /c "
1826                                      "schtasks /Query /FO LIST /TN upgrademe /V")
1827            self.log_command_output(output, error)
1828            self.stop_schedule_tasks()
1829        # run task schedule to upgrade Membase server
1830        output, error = self.execute_command("cmd /c schtasks /run /tn upgrademe")
1831        self.log_command_output(output, error)
1832        deleted = self.wait_till_file_deleted(WIN_CB_PATH, \
1833                       VERSION_FILE, timeout_in_seconds=600)
1834        if not deleted:
1835            log.error("Uninstall was failed at node {0}".format(self.ip))
1836            sys.exit()
1837        self.wait_till_file_added(WIN_CB_PATH, VERSION_FILE, \
1838                                       timeout_in_seconds=600)
1839        log.info("installed version: {0}".format(version))
1840        output, error = self.execute_command("cat "
1841                       "'/cygdrive/c/Program Files/Couchbase/Server/VERSION.txt'")
1842        ended = self.wait_till_process_ended(version[:10])
1843        if not ended:
1844            sys.exit("*****  Node %s failed to upgrade  *****" % (self.ip))
1845        self.sleep(10, "wait for server to start up completely")
1846        ct = time.time()
1847        while time.time() - ct < 10800:
1848            output, error = self.execute_command("cmd /c "
1849                      "schtasks /Query /FO LIST /TN upgrademe /V| findstr Status ")
1850            if "Ready" in str(output):
1851                log.info("upgrademe task complteted")
1852                break
1853            elif "Could not start":
1854                log.exception("Ugrade failed!!!")
1855                break
1856            else:
1857                log.info("upgrademe task still running:{0}".format(output))
1858                self.sleep(30)
1859        output, error = self.execute_command("cmd /c "
1860                                       "schtasks /Query /FO LIST /TN upgrademe /V")
1861        self.log_command_output(output, error)
1862        """ need to remove binary after done upgrade.  Since watson, current binary
1863            could not reused to uninstall or install cb server """
1864        self.delete_file(WIN_TMP_PATH, version[:10] + ".exe")
1865
1866    """
1867        This method install Couchbase Server
1868    """
1869    def install_server(self, build, startserver=True, path='/tmp', vbuckets=None,
1870                       swappiness=10, force=False, openssl='', upr=None, xdcr_upr=None,
1871                       fts_query_limit=None, cbft_env_options=None, enable_ipv6=None):
1872
1873        log.info('*****install server ***')
1874        server_type = None
1875        success = True
1876        start_server_after_install = True
1877        track_words = ("warning", "error", "fail")
1878        if build.name.lower().find("membase") != -1:
1879            server_type = 'membase'
1880            abbr_product = "mb"
1881        elif build.name.lower().find("couchbase") != -1:
1882            server_type = 'couchbase'
1883            abbr_product = "cb"
1884        else:
1885            raise Exception("its not a membase or couchbase?")
1886        self.extract_remote_info()
1887
1888        log.info('deliverable_type : {0}'.format(self.info.deliverable_type))
1889        if self.info.type.lower() == 'windows':
1890            log.info('***** Doing the windows install')
1891            self.terminate_processes(self.info, [s for s in WIN_PROCESSES_KILLED])
1892            self.terminate_processes(self.info, \
1893                                     [s + "-*" for s in COUCHBASE_FROM_VERSION_3])
1894            # to prevent getting full disk let's delete some large files
1895            self.remove_win_backup_dir()
1896            self.remove_win_collect_tmp()
1897            output, error = self.execute_command("cmd /c schtasks /run /tn installme")
1898            success &= self.log_command_output(output, error, track_words)
1899            file_check = 'VERSION.txt'
1900            self.wait_till_file_added("/cygdrive/c/Program Files/{0}/Server/".format(server_type.title()), file_check,
1901                                          timeout_in_seconds=600)
1902            output, error = self.execute_command("cmd /c schtasks /Query /FO LIST /TN installme /V")
1903            self.log_command_output(output, error)
1904            ended = self.wait_till_process_ended(build.product_version[:10])
1905            if not ended:
1906                sys.exit("*****  Node %s failed to install  *****" % (self.ip))
1907            self.sleep(10, "wait for server to start up completely")
1908            if vbuckets and int(vbuckets) != 1024:
1909                self.set_vbuckets_win(build.version_number, vbuckets)
1910            if fts_query_limit:
1911                self.set_environment_variable(
1912                    name="CBFT_ENV_OPTIONS",
1913                    value="bleveMaxResultWindow={0}".format(int(fts_query_limit))
1914                )
1915
1916
1917            output, error = self.execute_command("rm -f \
1918                       /cygdrive/c/automation/*_{0}_install.iss".format(self.ip))
1919            self.log_command_output(output, error)
1920            self.delete_file(WIN_TMP_PATH, build.product_version[:10] + ".exe")
1921            # output, error = self.execute_command("cmd rm /cygdrive/c/tmp/{0}*.exe".format(build_name))
1922            # self.log_command_output(output, error)
1923        elif self.info.deliverable_type in ["rpm", "deb"]:
1924            if startserver and vbuckets == None:
1925                environment = ""
1926            else:
1927                environment = "INSTALL_DONT_START_SERVER=1 "
1928                start_server_after_install = False
1929            log.info('/tmp/{0} or /tmp/{1}'.format(build.name, build.product))
1930
1931            # set default swappiness to 10 unless specify in params in all unix environment
1932            if self.nonroot:
1933                log.info("**** use root to run script/ssh.py to execute /sbin/sysctl vm.swappiness=0 "
1934                              "enable coredump cbenable_core_dumps.sh /tmp")
1935            else:
1936                output, error = self.execute_command('/sbin/sysctl vm.swappiness={0}'.format(swappiness))
1937                success &= self.log_command_output(output, error, track_words, debug=False)
1938
1939            if self.info.deliverable_type == 'rpm':
1940                if self.nonroot:
1941                    op, er = self.execute_command('cd {0}; rpm2cpio {1} ' \
1942                        '|  cpio --extract --make-directories --no-absolute-filenames ' \
1943                                     .format(self.nr_home_path, build.name), debug=False)
1944                    self.log_command_output(op, er)
1945                    output, error = self.execute_command('cd {0}{1}; ./bin/install/reloc.sh `pwd` '\
1946                                            .format(self.nr_home_path, LINUX_CB_PATH), debug=False)
1947                    self.log_command_output(output, error)
1948                    op, er = self.execute_command('cd {0};pwd'.format(self.nr_home_path))
1949                    self.log_command_output(op, er)
1950                    """ command to start Couchbase Server in non root
1951                        /home/nonroot_user/opt/couchbase/bin/couchbase-server \-- -noinput -detached
1952                    """
1953                    output, error = self.execute_command("ls -lh ")
1954                    self.log_command_output(output, error)
1955                    if start_server_after_install:
1956                        output, error = self.execute_command('%s%scouchbase-server '\
1957                                                             '\-- -noinput -detached '\
1958                                                              % (self.nr_home_path,
1959                                                                 LINUX_COUCHBASE_BIN_PATH))
1960                else:
1961                    self.check_pkgconfig(self.info.deliverable_type, openssl)
1962                    if force:
1963                        output, error = self.execute_command('{0}rpm -Uvh --force /tmp/{1}'\
1964                                                .format(environment, build.name), debug=False)
1965                    else:
1966                        output, error = self.execute_command('{0}rpm -i /tmp/{1}'\
1967                                                .format(environment, build.name), debug=False)
1968            elif self.info.deliverable_type == 'deb':
1969                if self.nonroot:
1970                    op, er = self.execute_command('cd %s; dpkg-deb -x %s %s '
1971                                                % (self.nr_home_path, build.name,
1972                                                   self.nr_home_path))
1973                    self.log_command_output(op, er)
1974                    output, error = self.execute_command('cd {0}{1}; ./bin/install/reloc.sh `pwd`'\
1975                                                        .format(self.nr_home_path, LINUX_CB_PATH))
1976                    self.log_command_output(output, error)
1977                    op, er = self.execute_command('pwd')
1978                    self.log_command_output(op, er)
1979                    """ command to start Couchbase Server in non root in ubuntu the same
1980                        as in centos above
1981                    """
1982                    if start_server_after_install:
1983                        output, error = self.execute_command('%s%scouchbase-server '\
1984                                                             '\-- -noinput -detached '\
1985                                                               % (self.nr_home_path,
1986                                                                  LINUX_COUCHBASE_BIN_PATH))
1987                else:
1988                    self.install_missing_lib()
1989                    if force:
1990                        output, error = self.execute_command('{0}dpkg --force-all -i /tmp/{1}'\
1991                                                 .format(environment, build.name), debug=False)
1992                    else:
1993                        output, error = self.execute_command('{0}dpkg -i /tmp/{1}'\
1994                                                 .format(environment, build.name), debug=False)
1995
1996            if "SUSE" in self.info.distribution_type:
1997                if error and error[0] == 'insserv: Service network is missed in the runlevels 2'\
1998                                         ' 4 to use service couchbase-server':
1999                        log.info("Ignore this error for opensuse os")
2000                        error = []
2001            if output:
2002                server_ip = "\n\n**** Installing on server: {0} ****".format(self.ip)
2003                output.insert(0, server_ip)
2004            if error:
2005                server_ip = "\n\n**** Installing on server: {0} ****".format(self.ip)
2006                error.insert(0, server_ip)
2007            success &= self.log_command_output(output, error, track_words, debug=False)
2008            nonroot_path_start = ""
2009            if not self.nonroot:
2010                nonroot_path_start = "/"
2011                self.create_directory(path)
2012                output, error = self.execute_command('/opt/{0}/bin/{1}enable_core_dumps.sh  {2}'.
2013                                            format(server_type, abbr_product, path), debug=False)
2014                success &= self.log_command_output(output, error, track_words, debug=False)
2015
2016            if vbuckets:
2017                """
2018                   From spock, the file to edit is in /opt/couchbase/bin/couchbase-server
2019                """
2020                output, error = self.execute_command("sed -i 's/export ERL_FULLSWEEP_AFTER/"
2021                                                     "export ERL_FULLSWEEP_AFTER\\n"
2022                                                     "{0}_NUM_VBUCKETS={1}\\n"
2023                                                     "export {0}_NUM_VBUCKETS/' "
2024                                                     "{3}opt/{2}/bin/{2}-server".
2025                                                     format(server_type.upper(), vbuckets,
2026                                            server_type, nonroot_path_start), debug=False)
2027                success &= self.log_command_output(output, error, track_words)
2028            if upr is not None:
2029                protocol = "tap"
2030                if upr:
2031                    protocol = "upr"
2032                output, error = \
2033                self.execute_command("sed -i 's/END INIT INFO/END INIT INFO\\nexport"
2034                              " COUCHBASE_REPL_TYPE={1}/' {2}opt/{0}/etc/{0}_init.d"\
2035                                                      .format(server_type, protocol,
2036                                                              nonroot_path_start))
2037                success &= self.log_command_output(output, error, track_words)
2038            if xdcr_upr == False:
2039                output, error = \
2040                    self.execute_command("sed -i 's/ulimit -c unlimited/ulimit "
2041                             "-c unlimited\\n    export XDCR_USE_OLD_PATH={1}/'"
2042                                                     "{2}opt/{0}/etc/{0}_init.d"\
2043                                                    .format(server_type, "true",
2044                                                            nonroot_path_start))
2045                success &= self.log_command_output(output, error, track_words)
2046            if fts_query_limit:
2047                if cbft_env_options:
2048                    output, error = \
2049                        self.execute_command("sed -i 's/export PATH/export PATH\\n"
2050                                             "export CBFT_ENV_OPTIONS=bleveMaxResultWindow={1},{2}/'\
2051                                             {3}opt/{0}/bin/{0}-server".format(server_type,
2052                                                                               int(fts_query_limit),
2053                                                                               cbft_env_options.replace(':','='),
2054                                                                               nonroot_path_start))
2055                else:
2056                    output, error = \
2057                        self.execute_command("sed -i 's/export PATH/export PATH\\n"
2058                                "export CBFT_ENV_OPTIONS=bleveMaxResultWindow={1}/'\
2059                                {2}opt/{0}/bin/{0}-server".format(server_type, int(fts_query_limit),
2060                                                                  nonroot_path_start))
2061                success &= self.log_command_output(output, error, track_words)
2062                startserver = True
2063
2064
2065                success &= self.log_command_output(output, error, track_words)
2066                startserver = True
2067
2068            if enable_ipv6:
2069                cb_version = build.product_version[:5]
2070                if cb_version == "6.0.0" or cb_version < "5.5.3":
2071                    output, error = \
2072                        self.execute_command("sed -i '/ipv6, /c \\{ipv6, true\}'. %s"
2073                            % testconstants.LINUX_STATIC_CONFIG)
2074                else:
2075                    """ dist_cfg contains {dist_type,inet_tcp}. in IPv4.
2076                        We need to change it to {dist_type,inet6_tcp}. in IPv6 server
2077                    """
2078                    output, error = self.execute_command("echo '{{dist_type,inet6_tcp}}.' > {0}"\
2079                                                                      .format(LINUX_DIST_CONFIG))
2080                success &= self.log_command_output(output, error, track_words)
2081                startserver = True
2082
2083            # skip output: [WARNING] couchbase-server is already started
2084            # dirname error skipping for CentOS-6.6 (MB-12536)
2085            track_words = ("error", "fail", "dirname")
2086            if startserver:
2087                self.start_couchbase()
2088                if (build.product_version.startswith("1.") \
2089                        or build.product_version.startswith("2.0.0"))\
2090                        and build.deliverable_type == "deb":
2091                    # skip error '* Failed to start couchbase-server' for 1.* & 2.0.0 builds(MB-7288)
2092                    # fix in 2.0.1 branch Change-Id: I850ad9424e295bbbb79ede701495b018b5dfbd51
2093                    log.warn("Error '* Failed to start couchbase-server' for 1.* "
2094                                                          "builds will be skipped")
2095                    self.log_command_output(output, error, track_words)
2096                else:
2097                    success &= self.log_command_output(output, error, track_words, debug=False)
2098        elif self.info.deliverable_type in ["zip"]:
2099            """ close Safari browser before install """
2100            self.terminate_process(self.info, "/Applications/Safari.app/Contents/MacOS/Safari")
2101            o, r = self.execute_command("ps aux | grep Archive | awk '{print $2}' | xargs kill -9")
2102            self.sleep(20)
2103            output, error = self.execute_command("cd ~/Downloads ; open couchbase-server*.zip")
2104            self.log_command_output(output, error)
2105            self.sleep(20)
2106            cmd1 = "mv ~/Downloads/couchbase-server*/Couchbase\ Server.app /Applications/"
2107            cmd2 = "sudo xattr -d -r com.apple.quarantine /Applications/Couchbase\ Server.app"
2108            cmd3 = "open /Applications/Couchbase\ Server.app"
2109            output, error = self.execute_command(cmd1)
2110            self.log_command_output(output, error)
2111            output, error = self.execute_command(cmd2)
2112            self.log_command_output(output, error)
2113            output, error = self.execute_command(cmd3)
2114            self.log_command_output(output, error)
2115
2116        output, error = self.execute_command("rm -f *-diag.zip", debug=False)
2117        self.log_command_output(output, error, track_words, debug=False)
2118        return success
2119
2120    def install_server_win(self, build, version, startserver=True,
2121                           vbuckets=None, fts_query_limit=None,
2122                           enable_ipv6=None, windows_msi=False, cbft_env_options=None):
2123
2124
2125        log.info('******start install_server_win ********')
2126        if windows_msi:
2127            self.remove_win_backup_dir()
2128            self.remove_win_collect_tmp()
2129            if enable_ipv6:
2130                output, error = self.execute_command("cd /cygdrive/c/tmp;"
2131                                                 "msiexec /i {0}.msi USE_IPV6=true /qn "\
2132                                                    .format(version))
2133            else:
2134                output, error = self.execute_command("cd /cygdrive/c/tmp;"
2135                                                     "msiexec /i {0}.msi /qn " \
2136                                                     .format(version))
2137            self.log_command_output(output, error)
2138            if fts_query_limit:
2139                self.set_fts_query_limit_win(
2140                    build = version,
2141                    name="CBFT_ENV_OPTIONS",
2142                    value="bleveMaxResultWindow={0}".format(int(fts_query_limit)),
2143                    ipv6=enable_ipv6
2144                )
2145            return len(error) == 0
2146        remote_path = None
2147        success = True
2148        track_words = ("warning", "error", "fail")
2149        if build.name.lower().find("membase") != -1:
2150            remote_path = testconstants.WIN_MB_PATH
2151            abbr_product = "mb"
2152        elif build.name.lower().find("couchbase") != -1:
2153            remote_path = WIN_CB_PATH
2154            abbr_product = "cb"
2155
2156        if remote_path is None:
2157            raise Exception("its not a membase or couchbase?")
2158        self.extract_remote_info()
2159        log.info('deliverable_type : {0}'.format(self.info.deliverable_type))
2160        if self.info.type.lower() == 'windows':
2161            task = "install"
2162            bat_file = "install.bat"
2163            dir_paths = ['/cygdrive/c/automation', '/cygdrive/c/tmp']
2164            capture_iss_file = ""
2165            # build = self.build_url(params)
2166            self.create_multiple_dir(dir_paths)
2167            self.copy_files_local_to_remote('resources/windows/automation', \
2168                                                    '/cygdrive/c/automation')
2169            #self.create_windows_capture_file(task, abbr_product, version)
2170            capture_iss_file = self.modify_bat_file('/cygdrive/c/automation', \
2171                                             bat_file, abbr_product, version, task)
2172            self.stop_schedule_tasks()
2173            self.remove_win_backup_dir()
2174            self.remove_win_collect_tmp()
2175            log.info('sleep for 5 seconds before running task '
2176                                'schedule uninstall on {0}'.format(self.ip))
2177
2178            """ the code below need to remove when bug MB-11985
2179                                                            is fixed in 3.0.1 """
2180            if task == "install" and (version[:5] in COUCHBASE_VERSION_2 or \
2181                                      version[:5] in COUCHBASE_FROM_VERSION_3):
2182                log.info("due to bug MB-11985, we need to delete below registry "
2183                                        "before install version 2.x.x and 3.x.x")
2184                output, error = self.execute_command("reg delete \
2185                         'HKLM\Software\Wow6432Node\Ericsson\Erlang\ErlSrv' /f ")
2186                self.log_command_output(output, error)
2187            """ end remove code """
2188
2189            """ run task schedule to install cb server """
2190            output, error = self.execute_command("cmd /c schtasks /run /tn installme")
2191            success &= self.log_command_output(output, error, track_words)
2192            file_check = 'VERSION.txt'
2193            self.wait_till_file_added(remote_path, file_check, timeout_in_seconds=600)
2194            if version[:3] != "2.5":
2195                ended = self.wait_till_process_ended(build.product_version[:10])
2196                if not ended:
2197                    sys.exit("*****  Node %s failed to install  *****" % (self.ip))
2198            if version[:3] == "2.5":
2199                self.sleep(20, "wait for server to start up completely")
2200            else:
2201                self.sleep(10, "wait for server to start up completely")
2202            output, error = self.execute_command("rm -f *-diag.zip")
2203            self.log_command_output(output, error, track_words)
2204            if vbuckets and int(vbuckets) != 1024:
2205                self.set_vbuckets_win(build.version_number, vbuckets)
2206            if fts_query_limit:
2207                self.set_fts_query_limit_win(
2208                    build = version,
2209                    name="CBFT_ENV_OPTIONS",
2210                    value="bleveMaxResultWindow={0}".format(int(fts_query_limit)),
2211                    ipv6=enable_ipv6
2212                )
2213
2214            if "4.0" in version[:5]:
2215                """  remove folder if it exists in work around of bub MB-13046 """
2216                self.execute_command("rm -rf \
2217                /cygdrive/c/Jenkins/workspace/sherlock-windows/couchbase/install/etc/security")
2218                """ end remove code for bug MB-13046 """
2219            if capture_iss_file:
2220                    log.info("****Delete {0} in windows automation directory" \
2221                                                          .format(capture_iss_file))
2222                    output, error = self.execute_command("rm -f \
2223                               /cygdrive/c/automation/{0}".format(capture_iss_file))
2224                    self.log_command_output(output, error)
2225                    log.info("Delete {0} in slave resources/windows/automation dir" \
2226                             .format(capture_iss_file))
2227                    os.system("rm -f resources/windows/automation/{0}" \
2228                                                          .format(capture_iss_file))
2229            self.delete_file(WIN_TMP_PATH, build.product_version[:10] + ".exe")
2230            log.info('***** done install_server_win *****')
2231            return success
2232
2233
2234    def install_server_via_repo(self, deliverable_type, cb_edition, remote_client):
2235        success = True
2236        track_words = ("warning", "error", "fail")
2237        if cb_edition:
2238            cb_edition = "-" + cb_edition
2239        if deliverable_type == "deb":
2240            self.update_couchbase_release(remote_client, deliverable_type)
2241            output, error = self.execute_command("yes \
2242                   | apt-get install couchbase-server{0}".format(cb_edition))
2243            self.log_command_output(output, error)
2244            success &= self.log_command_output(output, error, track_words)
2245        elif deliverable_type == "rpm":
2246            self.update_couchbase_release(remote_client, deliverable_type)
2247            output, error = self.execute_command("yes \
2248                  | yum install couchbase-server{0}".format(cb_edition))
2249            self.log_command_output(output, error, track_words)
2250            success &= self.log_command_output(output, error, track_words)
2251        return success
2252
2253    def update_couchbase_release(self, remote_client, deliverable_type):
2254        if deliverable_type == "deb":
2255            """ remove old couchbase-release package """
2256            log.info("remove couchbase-release at node {0}".format(self.ip))
2257            output, error = self.execute_command("dpkg --get-selections |\
2258                                                              grep couchbase")
2259            self.log_command_output(output, error)
2260            for str in output:
2261                if "couchbase-release" in str:
2262                    output, error = self.execute_command("apt-get \
2263                                                   purge -y couchbase-release")
2264            output, error = self.execute_command("dpkg --get-selections |\
2265                                                              grep couchbase")
2266            self.log_command_output(output, error)
2267            package_remove = True
2268            for str in output:
2269                if "couchbase-release" in str:
2270                    package_remove = False
2271                    log.info("couchbase-release is not removed at node {0}" \
2272                                                                  .format(self.ip))
2273                    sys.exit("***  Node %s failed to remove couchbase-release  ***"\
2274                                                                        % (self.ip))
2275            """ install new couchbase-release package """
2276            log.info("install new couchbase-release repo at node {0}" \
2277                                                             .format(self.ip))
2278            self.execute_command("rm -rf /tmp/couchbase-release*")
2279            self.execute_command("cd /tmp; wget {0}".format(CB_RELEASE_APT_GET_REPO))
2280            output, error = self.execute_command("yes | dpkg -i /tmp/couchbase-release*")
2281            self.log_command_output(output, error)
2282            output, error = self.execute_command("dpkg --get-selections |\
2283                                                              grep couchbase")
2284            package_updated = False
2285            for str in output:
2286                if "couchbase-release" in str:
2287                    package_updated = True
2288                    log.info("couchbase-release installed on node {0}" \
2289                                               .format(self.ip))
2290                    return package_updated
2291            if not package_updated:
2292                sys.exit("fail to install %s on node %s" % \
2293                                  (CB_RELEASE_APT_GET_REPO.rsplit("/",1)[-1], self.ip))
2294        elif deliverable_type == "rpm":
2295            """ remove old couchbase-release package """
2296            log.info("remove couchbase-release at node {0}".format(self.ip))
2297            output, error = self.execute_command("rpm -qa | grep couchbase")
2298            self.log_command_output(output, error)
2299            for str in output:
2300                if "couchbase-release" in str:
2301                    output, error = self.execute_command("rpm -e couchbase-release")
2302            output, error = self.execute_command("rpm -qa | grep couchbase")
2303            self.log_command_output(output, error)
2304            package_remove = True
2305            for str in output:
2306                if "couchbase-release" in str:
2307                    package_remove = False
2308                    log.info("couchbase-release is not removed at node {0}" \
2309                                                                  .format(self.ip))
2310                    sys.exit("***  Node %s failed to remove couchbase-release  ***"\
2311                                                                        % (self.ip))
2312            """ install new couchbase-release package """
2313            log.info("install new couchbase-release repo at node {0}" \
2314                                                             .format(self.ip))
2315            self.execute_command("rm -rf /tmp/couchbase-release*")
2316            self.execute_command("cd /tmp; wget {0}".format(CB_RELEASE_YUM_REPO))
2317            output, error = self.execute_command("yes | rpm -i /tmp/couchbase-release*")
2318            self.log_command_output(output, error)
2319            output, error = self.execute_command("rpm -qa | grep couchbase")
2320            package_updated = False
2321            for str in output:
2322                if "couchbase-release" in str:
2323                    package_updated = True
2324                    log.info("couchbase-release installed on node {0}" \
2325                                               .format(self.ip))
2326                    return package_updated
2327            if not package_updated:
2328                sys.exit("fail to install %s on node %s" % \
2329                                  (CB_RELEASE_YUM_REPO.rsplit("/",1)[-1], self.ip))
2330
2331    def install_moxi(self, build):
2332        success = True
2333        track_words = ("warning", "error", "fail")
2334        self.extract_remote_info()
2335        log.info('deliverable_type : {0}'.format(self.info.deliverable_type))
2336        if self.info.type.lower() == 'windows':
2337            log.error('Not implemented')
2338        elif self.info.deliverable_type in ["rpm"]:
2339            output, error = self.execute_command('rpm -i /tmp/{0}'.format(build.name))
2340            if error and ' '.join(error).find("ERROR") != -1:
2341                success = False
2342        elif self.info.deliverable_type == 'deb':
2343            output, error = self.execute_command('dpkg -i /tmp/{0}'.format(build.name))
2344            if error and ' '.join(error).find("ERROR") != -1:
2345                success = False
2346        success &= self.log_command_output(output, '', track_words)
2347        return success
2348
2349    def wait_till_file_deleted(self, remotepath, filename, timeout_in_seconds=180):
2350        end_time = time.time() + float(timeout_in_seconds)
2351        deleted = False
2352        log.info("file {0} checked at {1}".format(filename, remotepath))
2353        while time.time() < end_time and not deleted:
2354            # get the process list
2355            exists = self.file_exists(remotepath, filename)
2356            if exists:
2357                log.error('at {2} file {1} still exists' \
2358                          .format(remotepath, filename, self.ip))
2359                time.sleep(10)
2360            else:
2361                log.info('at {2} FILE {1} DOES NOT EXIST ANYMORE!' \
2362                         .format(remotepath, filename, self.ip))
2363                deleted = True
2364        return deleted
2365
2366    def wait_till_file_added(self, remotepath, filename, timeout_in_seconds=180):
2367        end_time = time.time() + float(timeout_in_seconds)
2368        added = False
2369        log.info("file {0} checked at {1}".format(filename, remotepath))
2370        while time.time() < end_time and not added:
2371            # get the process list
2372            exists = self.file_exists(remotepath, filename)
2373            if not exists:
2374                log.error('at {2} file {1} does not exist' \
2375                          .format(remotepath, filename, self.ip))
2376                time.sleep(10)
2377            else:
2378                log.info('at {2} FILE {1} EXISTS!' \
2379                         .format(remotepath, filename, self.ip))
2380                added = True
2381        return added
2382
2383
2384
2385
2386    def wait_till_compaction_end(self, rest, bucket, timeout_in_seconds=60):
2387        end_time = time.time() + float(timeout_in_seconds)
2388
2389        while time.time() < end_time:
2390            status, progress = rest.check_compaction_status(bucket)
2391            if status:
2392                log.info("compaction progress is %s" % progress)
2393                time.sleep(1)
2394            else:
2395                # the compaction task has completed
2396                return True
2397
2398        log.error("auto compaction has not ended in {0} sec.".format(str(timeout_in_seconds)))
2399        return False
2400
2401    def wait_till_process_ended(self, process_name, timeout_in_seconds=600):
2402        if process_name[-1:] == "-":
2403            process_name = process_name[:-1]
2404        end_time = time.time() + float(timeout_in_seconds)
2405        process_ended = False
2406        process_running = False
2407        count_process_not_run = 0
2408        while time.time() < end_time and not process_ended:
2409            output, error = self.execute_command("tasklist | grep {0}" \
2410                                                 .format(process_name))
2411            self.log_command_output(output, error)
2412            if output and process_name in output[0]:
2413                self.sleep(8, "wait for process ended!")
2414                process_running = True
2415            else:
2416                if process_running:
2417                    log.info("{1}: Alright, PROCESS {0} ENDED!" \
2418                             .format(process_name, self.ip))
2419                    process_ended = True
2420                else:
2421                    if count_process_not_run < 5:
2422                        log.error("{1}: process {0} may not run" \
2423                              .format(process_name, self.ip))
2424                        self.sleep(5)
2425                        count_process_not_run += 1
2426                    else:
2427                        log.error("{1}: process {0} did not run after 25 seconds"
2428                                                   .format(process_name, self.ip))
2429                        mesg = "kill in/uninstall job due to process was not run" \
2430                                                     .format(process_name, self.ip)
2431                        self.stop_current_python_running(mesg)
2432        if time.time() >= end_time and not process_ended:
2433            log.info("Process {0} on node {1} is still running"
2434                     " after 10 minutes VERSION.txt file was removed"
2435                                      .format(process_name, self.ip))
2436        return process_ended
2437
2438    def terminate_processes(self, info, list):
2439        for process in list:
2440            type = info.distribution_type.lower()
2441            if type == "windows":
2442                # set debug=False if does not want to show log
2443                self.execute_command("taskkill /F /T /IM {0}".format(process),
2444                                                                  debug=False)
2445            elif type in LINUX_DISTRIBUTION_NAME:
2446                self.terminate_process(info, process, force=True)
2447
2448    def remove_folders(self, list):
2449        for folder in list:
2450            output, error = self.execute_command("rm -rf {0}".format(folder),
2451                                                                debug=False)
2452            self.log_command_output(output, error)
2453
2454    def couchbase_uninstall(self, windows_msi=False, product=None):
2455        log.info('{0} *****In couchbase uninstall****'.format( self.ip))
2456        linux_folders = ["/var/opt/membase", "/opt/membase", "/etc/opt/membase",
2457                         "/var/membase/data/*", "/opt/membase/var/lib/membase/*",
2458                         "/opt/couchbase", "/data/*"]
2459        terminate_process_list = ["beam.smp", "memcached", "moxi", "vbucketmigrator",
2460                                  "couchdb", "epmd", "memsup", "cpu_sup", "goxdcr", "erlang", "eventing", "erl", "godu",
2461                                  "goport", "gosecrets", "projector"]
2462        self.extract_remote_info()
2463        log.info(self.info.distribution_type)
2464        type = self.info.distribution_type.lower()
2465        fv, sv, bn = self.get_cbversion(type)
2466        if type == 'windows':
2467            product = "cb"
2468            query = BuildQuery()
2469            os_type = "exe"
2470            if windows_msi:
2471                os_type = "msi"
2472                self.info.deliverable_type = "msi"
2473            task = "uninstall"
2474            bat_file = "uninstall.bat"
2475            product_name = "couchbase-server-enterprise"
2476            version_path = "/cygdrive/c/Program Files/Couchbase/Server/"
2477            deleted = False
2478            capture_iss_file = ""
2479            log.info("kill any in/uninstall process from version 3 in node %s"
2480                                                                        % self.ip)
2481            self.terminate_processes(self.info, \
2482                                     [s + "-*" for s in COUCHBASE_FROM_VERSION_3])
2483            exist = self.file_exists(version_path, VERSION_FILE)
2484            log.info("Is VERSION file existed on {0}? {1}".format(self.ip, exist))
2485            if exist:
2486                cb_releases_version = []
2487                for x in CB_RELEASE_BUILDS:
2488                    if int(CB_RELEASE_BUILDS[x]):
2489                        cb_releases_version.append(x)
2490
2491                build_name, short_version, full_version = \
2492                    self.find_build_version(version_path, VERSION_FILE, product)
2493
2494                if "-" in full_version:
2495                    msi_build = full_version.split("-")
2496                    """
2497                       In spock from build 2924 and later release, we only support
2498                       msi installation method on windows
2499                    """
2500                    if msi_build[0] in COUCHBASE_FROM_SPOCK:
2501                        os_type = "msi"
2502                        windows_msi = True
2503                        self.info.deliverable_type = "msi"
2504                else:
2505                    mesg = " ***** ERROR: ***** \n" \
2506                           " Couchbase Server version format is not correct. \n" \
2507                           " It should be 0.0.0-DDDD format\n" \
2508                           % (self.ip, os.getpid())
2509                    self.stop_current_python_running(mesg)
2510
2511                build_repo = MV_LATESTBUILD_REPO
2512                if full_version[:5] not in COUCHBASE_VERSION_2 and \
2513                   full_version[:5] not in COUCHBASE_VERSION_3:
2514                    if full_version[:3] in CB_VERSION_NAME:
2515                        build_repo = CB_REPO + CB_VERSION_NAME[full_version[:3]] + "/"
2516                    else:
2517                        sys.exit("version is not support yet")
2518                log.info("*****VERSION file exists."
2519                                   "Start to uninstall {0} on {1} server"
2520                                   .format(product, self.ip))
2521                if full_version[:3] == "4.0":
2522                    build_repo = SHERLOCK_BUILD_REPO
2523                log.info('Build name: {0}'.format(build_name))
2524                build_name = build_name.rstrip() + ".%s" % os_type
2525                log.info('Check if {0} is in tmp directory on {1} server'\
2526                                                       .format(build_name, self.ip))
2527                exist = self.file_exists("/cygdrive/c/tmp/", build_name)
2528                if not exist:  # if not exist in tmp dir, start to download that version
2529                    if short_version[:5] in cb_releases_version:
2530                        build = query.find_couchbase_release_build(product_name,
2531                                                        self.info.deliverable_type,
2532                                                       self.info.architecture_type,
2533                                                                     short_version,
2534                                                                  is_amazon=False,
2535                                 os_version=self.info.distribution_version.lower())
2536                    elif short_version in COUCHBASE_RELEASE_VERSIONS_3:
2537                        build = query.find_membase_release_build(product_name,
2538                                                   self.info.deliverable_type,
2539                                                  self.info.architecture_type,
2540                                                                short_version,
2541                                                              is_amazon=False,
2542                            os_version=self.info.distribution_version.lower())
2543                    else:
2544                        builds, changes = query.get_all_builds(version=full_version,
2545                                      deliverable_type=self.info.deliverable_type,
2546                                      architecture_type=self.info.architecture_type,
2547                                      edition_type=product_name, repo=build_repo,
2548                                      distribution_version=\
2549                                            self.info.distribution_version.lower())
2550
2551                        build = query.find_build(builds, product_name, os_type,
2552                                                   self.info.architecture_type,
2553                                                                  full_version,
2554                            distribution_version=self.info.distribution_version.lower(),
2555                                  distribution_type=self.info.distribution_type.lower())
2556                    downloaded = self.download_binary_in_win(build.url, short_version)
2557                    if downloaded:
2558                        log.info('Successful download {0}.exe on {1} server'
2559                                             .format(short_version, self.ip))
2560                    else:
2561                        log.error('Download {0}.exe failed'.format(short_version))
2562                if not windows_msi:
2563                    dir_paths = ['/cygdrive/c/automation', '/cygdrive/c/tmp']
2564                    self.create_multiple_dir(dir_paths)
2565                    self.copy_files_local_to_remote('resources/windows/automation',
2566                                                      '/cygdrive/c/automation')
2567                    # modify bat file to run uninstall schedule task
2568                    #self.create_windows_capture_file(task, product, full_version)
2569                    capture_iss_file = self.modify_bat_file('/cygdrive/c/automation',
2570                                        bat_file, product, short_version, task)
2571                    self.stop_schedule_tasks()
2572
2573                """ Remove this workaround when bug MB-14504 is fixed """
2574                log.info("Kill any un/install process leftover in sherlock")
2575                log.info("Kill any cbq-engine.exe in sherlock")
2576                self.execute_command('taskkill /F /T /IM cbq-engine.exe')
2577                log.info('sleep for 5 seconds before running task '
2578                                    'schedule uninstall on {0}'.format(self.ip))
2579                """ End remove this workaround when bug MB-14504 is fixed """
2580
2581                """ Remove this workaround when bug MB-28775 is fixed """
2582                # log.info("Kill any eventing-consumer.exe in vulcan")
2583                # self.execute_command('taskkill /F /T /IM eventing*')
2584                """ End remove this workaround when bug MB-28775 is fixed """
2585
2586                self.stop_couchbase()
2587                time.sleep(5)
2588                # run schedule task uninstall couchbase server
2589                if windows_msi:
2590                    log.info("******** uninstall via msi method ***********")
2591                    output, error = \
2592                        self.execute_command("cd /cygdrive/c/tmp; msiexec /x %s /qn"\
2593                                             % build_name)
2594                    self.log_command_output(output, error)
2595                    var_dir = "/cygdrive/c/Program\ Files/Couchbase/Server/var"
2596                    self.execute_command("rm -rf %s" % var_dir)
2597                else:
2598                    output, error = \
2599                        self.execute_command("cmd /c schtasks /run /tn removeme")
2600                    self.log_command_output(output, error)
2601                deleted = self.wait_till_file_deleted(version_path, \
2602                                            VERSION_FILE, timeout_in_seconds=600)
2603                if not deleted:
2604                    if windows_msi:
2605                        log.info("******** repair via msi method ***********")
2606                        output, error = \
2607                            self.execute_command("cd /cygdrive/c/tmp; msiexec /fa %s /norestart"\
2608                                                 % build_name)
2609                        self.log_command_output(output, error)
2610                        log.info("******** uninstall via msi method ***********")
2611                        output, error = \
2612                            self.execute_command("cd /cygdrive/c/tmp; msiexec /x %s /qn" \
2613                                                 % build_name)
2614                        self.log_command_output(output, error)
2615                        var_dir = "/cygdrive/c/Program\ Files/Couchbase/"
2616                        self.execute_command("rm -rf %s" % var_dir)
2617                        deleted = self.wait_till_file_deleted(version_path, \
2618                                                              VERSION_FILE, timeout_in_seconds=300)
2619                        if not deleted:
2620                            log.error("Uninstall was failed at node {0}".format(self.ip))
2621                            sys.exit()
2622                    else:
2623                        log.error("Uninstall was failed at node {0}".format(self.ip))
2624                        sys.exit()
2625                if full_version[:3] != "2.5":
2626                    uninstall_process = full_version[:10]
2627                    if not windows_msi:
2628                        ended = self.wait_till_process_ended(uninstall_process)
2629                        if not ended:
2630                            sys.exit("****  Node %s failed to uninstall  ****"
2631                                                              % (self.ip))
2632                if full_version[:3] == "2.5":
2633                    self.sleep(20, "next step is to install")
2634                else:
2635                    self.sleep(10, "next step is to install")
2636                """ delete binary after uninstall """
2637                self.delete_file(WIN_TMP_PATH, build_name)
2638                """ the code below need to remove when bug MB-11328
2639                                                           is fixed in 3.0.1 """
2640                output, error = self.kill_erlang(os="windows")
2641                self.log_command_output(output, error)
2642                """ end remove code """
2643
2644
2645                """ the code below need to remove when bug MB-11985
2646                                                           is fixed in 3.0.1 """
2647                if full_version[:5] in COUCHBASE_VERSION_2 or \
2648                   full_version[:5] in COUCHBASE_FROM_VERSION_3:
2649                    log.info("due to bug MB-11985, we need to delete below registry")
2650                    output, error = self.execute_command("reg delete \
2651                            'HKLM\Software\Wow6432Node\Ericsson\Erlang\ErlSrv' /f ")
2652                    self.log_command_output(output, error)
2653                """ end remove code """
2654                if capture_iss_file and not windows_msi:
2655                    log.info("Delete {0} in windows automation directory" \
2656                                                          .format(capture_iss_file))
2657                    output, error = self.execute_command("rm -f \
2658                               /cygdrive/c/automation/{0}".format(capture_iss_file))
2659                    self.log_command_output(output, error)
2660                    log.info("Delete {0} in slave resources/windows/automation dir" \
2661                             .format(capture_iss_file))
2662                    os.system("rm -f resources/windows/automation/{0}" \
2663                                                          .format(capture_iss_file))
2664            else:
2665                log.info("*****No couchbase server on {0} server. Free to install" \
2666                                                                 .format(self.ip))
2667        elif type in LINUX_DISTRIBUTION_NAME:
2668            """ check if couchbase server installed by root """
2669            if self.nonroot:
2670                test = self.file_exists(LINUX_CB_PATH, VERSION_FILE)
2671                if self.file_exists(LINUX_CB_PATH, VERSION_FILE):
2672                    mesg = " ***** ERROR: ***** \n"\
2673                           " Couchbase Server was installed by root user. \n"\
2674                           " Use root user to uninstall it at %s \n"\
2675                           " This python process id: %d will be killed to stop the installation"\
2676                           % (self.ip, os.getpid())
2677                    self.stop_current_python_running(mesg)
2678                if self.file_exists(self.nr_home_path, NR_INSTALL_LOCATION_FILE):
2679                    output, error = self.execute_command("cat %s"
2680                                             % NR_INSTALL_LOCATION_FILE)
2681                    if output and output[0]:
2682                        log.info("Couchbase Server was installed in non default path %s"
2683                                                                            % output[0])
2684                        self.nr_home_path = output[0]
2685            # uninstallation command is different
2686            if type == "ubuntu":
2687                if self.nonroot:
2688                    """ check if old files from root install left in server """
2689                    if self.file_exists(LINUX_CB_PATH + "etc/couchdb/", "local.ini.debsave"):
2690                        print " ***** ERROR: ***** \n"\
2691                              "Couchbase Server files was left by root install at %s .\n"\
2692                              "Use root user to delete them all at server %s "\
2693                              " (rm -rf /opt/couchbase) to remove all couchbase folder.\n" \
2694                              % (LINUX_CB_PATH, self.ip)
2695                        sys.exit(1)
2696                    self.stop_server()
2697                else:
2698                    if sv in COUCHBASE_FROM_VERSION_4:
2699                        if self.is_enterprise(type):
2700                            if product is not None and product == "cbas":
2701                                uninstall_cmd = "dpkg -r {0};dpkg --purge {1};" \
2702                                        .format("couchbase-server-analytics", "couchbase-server-analytics")
2703                            else:
2704                                uninstall_cmd = "dpkg -r {0};dpkg --purge {1};" \
2705                                        .format("couchbase-server", "couchbase-server")
2706                        else:
2707                            uninstall_cmd = "dpkg -r {0};dpkg --purge {1};" \
2708                                         .format("couchbase-server-community",
2709                                                  "couchbase-server-community")
2710                    else:
2711                        uninstall_cmd = "dpkg -r {0};dpkg --purge {1};" \
2712                                     .format("couchbase-server", "couchbase-server")
2713                    output, error = self.execute_command(uninstall_cmd)
2714                    self.log_command_output(output, error)
2715            elif type in RPM_DIS_NAME:
2716                """ Sometimes, vm left with unfinish uninstall/install process.
2717                    We need to kill them before doing uninstall """
2718                if self.nonroot:
2719                    """ check if old files from root install left in server """
2720                    if self.file_exists(LINUX_CB_PATH + "etc/couchdb/", "local.ini.rpmsave"):
2721                        print "Couchbase Server files was left by root install at %s .\n"\
2722                               "Use root user to delete them all at server %s "\
2723                               " (rm -rf /opt/couchbase) to remove all couchbase folder.\n" \
2724                               % (LINUX_CB_PATH, self.ip)
2725                        sys.exit(1)
2726                    self.stop_server()
2727                else:
2728                    output, error = self.execute_command("killall -9 rpm")
2729                    self.log_command_output(output, error)
2730                    output, error = self.execute_command("rm -f /var/lib/rpm/.rpm.lock")
2731                    self.log_command_output(output, error)
2732                    if sv in COUCHBASE_FROM_VERSION_4:
2733                        if self.is_enterprise(type):
2734                            if product is not None and product == "cbas":
2735                                uninstall_cmd = 'rpm -e {0}'.format("couchbase-server-analytics")
2736                            else:
2737                                uninstall_cmd = 'rpm -e {0}'.format("couchbase-server")
2738                        else:
2739                            uninstall_cmd = 'rpm -e {0}' \
2740                                          .format("couchbase-server-community")
2741                    else:
2742                        uninstall_cmd = 'rpm -e {0}'.format("couchbase-server")
2743                    log.info('running rpm -e to remove couchbase-server')
2744                    output, error = self.execute_command(uninstall_cmd)
2745                    if output:
2746                        server_ip = "\n\n**** Uninstalling on server: {0} ****".format(self.ip)
2747                        output.insert(0, server_ip)
2748                    if error:
2749                        server_ip = "\n\n**** Uninstalling on server: {0} ****".format(self.ip)
2750                        error.insert(0, server_ip)
2751                    self.log_command_output(output, error)
2752                    output, error = self.execute_command("pkill -u couchbase")
2753                    self.log_command_output(output, error)
2754                    # This line is added for debugging purposes
2755                    output, error = self.execute_command("ps -ef | grep couchbase")
2756                    self.log_command_output(output, error)
2757            self.terminate_processes(self.info, terminate_process_list)
2758            if not self.nonroot:
2759                self.remove_folders(linux_folders)
2760                self.kill_memcached()
2761                output, error = self.execute_command("ipcrm")
2762                self.log_command_output(output, error)
2763        elif self.info.distribution_type.lower() == 'mac':
2764            self.stop_server(os='mac')
2765            """ close Safari browser before uninstall """
2766            self.terminate_process(self.info, "/Applications/Safari.app/Contents/MacOS/Safari")
2767            self.terminate_processes(self.info, terminate_process_list)
2768            output, error = self.execute_command("rm -rf /Applications/Couchbase\ Server.app")
2769            self.log_command_output(output, error)
2770            output, error = self.execute_command("rm -rf ~/Library/Application\ Support/Couchbase")
2771            self.log_command_output(output, error)
2772        if self.nonroot:
2773            if self.nr_home_path != "/home/%s/" % self.username:
2774                log.info("remove all non default install dir")
2775                output, error = self.execute_command("rm -rf %s"
2776                                                        % self.nr_home_path)
2777                self.log_command_output(output, error)
2778            else:
2779                log.info("Remove only Couchbase Server directories opt etc and usr ")
2780                output, error = self.execute_command("cd %s;rm -rf opt etc usr"
2781                                                           % self.nr_home_path)
2782                self.log_command_output(output, error)
2783                self.execute_command("cd %s;rm -rf couchbase-server-*"
2784                                                      % self.nr_home_path)
2785                output, error = self.execute_command("cd %s;ls -lh"
2786                                                      % self.nr_home_path)
2787                self.log_command_output(output, error)
2788            if "nr_install_dir" not in self.input.test_params:
2789                self.nr_home_path = "/home/%s/" % self.username
2790                output, error = self.execute_command(" :> %s"
2791                                             % NR_INSTALL_LOCATION_FILE)
2792                self.log_command_output(output, error)
2793            output, error = self.execute_command("ls -lh")
2794            self.log_command_output(output, error)
2795
2796    def couchbase_win_uninstall(self, product, version, os_name, query):
2797        log.info('*****couchbase_win_uninstall****')
2798        builds, changes = query.get_all_builds(version=version)
2799        bat_file = "uninstall.bat"
2800        task = "uninstall"
2801        deleted = False
2802
2803        self.extract_remote_info()
2804        ex_type = self.info.deliverable_type
2805        if self.info.architecture_type == "x86_64":
2806            os_type = "64"
2807        elif self.info.architecture_type == "x86":
2808            os_type = "32"
2809        if product == "cse":
2810            name = "couchbase-server-enterprise"
2811            version_path = "/cygdrive/c/Program Files/Couchbase/Server/"
2812
2813        exist = self.file_exists(version_path, VERSION_FILE)
2814        if exist:
2815            # call uninstall function to install couchbase server
2816            # Need to detect csse or cse when uninstall.
2817            log.info("Start uninstall cb server on this server")
2818            build_name, rm_version = self.find_build_version(version_path, VERSION_FILE)
2819            log.info('build needed to do auto uninstall {0}'.format(build_name))
2820            # find installed build in tmp directory to match with currently installed version
2821            build_name = build_name.rstrip() + ".exe"
2822            log.info('Check if {0} is in tmp directory'.format(build_name))
2823            exist = self.file_exists("/cygdrive/c/tmp/", build_name)
2824            if not exist:  # if not exist in tmp dir, start to download that version build
2825                build = query.find_build(builds, name, ex_type, self.info.architecture_type, rm_version)
2826                downloaded = self.download_binary_in_win(build.url, rm_version)
2827                if downloaded:
2828                    log.info('Successful download {0}.exe'.format(rm_version))
2829                else:
2830                    log.error('Download {0}.exe failed'.format(rm_version))
2831            # copy required files to automation directory
2832            dir_paths = ['/cygdrive/c/automation', '/cygdrive/c/tmp']
2833            self.create_multiple_dir(dir_paths)
2834            self.copy_files_local_to_remote('resources/windows/automation', '/cygdrive/c/automation')
2835            # modify bat file to run uninstall schedule task
2836            self.modify_bat_file('/cygdrive/c/automation', bat_file, product, rm_version, task)
2837            self.stop_couchbase()
2838
2839            """ the code below need to remove when bug MB-11328 is fixed in 3.0.1 """
2840            output, error = self.kill_erlang(os="windows")
2841            self.log_command_output(output, error)
2842            """ end remove code """
2843
2844            """ the code below need to removed when bug MB-28775 is fixed """
2845            output, error = self.kill_eventing_process(name="eventing-consumer")
2846            self.log_command_output(output, error)
2847            """ end remove code """
2848
2849            self.sleep(5, "before running task schedule uninstall")
2850            # run schedule task uninstall couchbase server
2851            output, error = self.execute_command("cmd /c schtasks /run /tn removeme")
2852            self.log_command_output(output, error)
2853            deleted = self.wait_till_file_deleted(version_path, VERSION_FILE, timeout_in_seconds=600)
2854            if not deleted:
2855                log.error("Uninstall was failed at node {0}".format(self.ip))
2856                sys.exit()
2857            ended = self.wait_till_process_ended(build_name[:10])
2858            if not ended:
2859                sys.exit("*****  Node %s failed to uninstall  *****" % (self.ip))
2860            self.sleep(10, "next step is to install")
2861            output, error = self.execute_command("rm -f \
2862                       /cygdrive/c/automation/*_{0}_uninstall.iss".format(self.ip))
2863            self.log_command_output(output, error)
2864            output, error = self.execute_command("cmd /c schtasks /Query /FO LIST /TN removeme /V")
2865            self.log_command_output(output, error)
2866            output, error = self.execute_command("rm -f /cygdrive/c/tmp/{0}".format(build_name))
2867            self.log_command_output(output, error)
2868
2869            """ the code below need to remove when bug MB-11328 is fixed in 3.0.1 """
2870            output, error = self.kill_erlang(os="windows")
2871            self.log_command_output(output, error)
2872            """ end remove code """
2873        else:
2874            log.info('No couchbase server on this server')
2875
2876    def membase_uninstall(self, save_upgrade_config=False):
2877        linux_folders = ["/var/opt/membase", "/opt/membase", "/etc/opt/membase",
2878                         "/var/membase/data/*", "/opt/membase/var/lib/membase/*"]
2879        terminate_process_list = ["beam", "memcached", "moxi", "vbucketmigrator",
2880                                  "couchdb", "epmd"]
2881        self.extract_remote_info()
2882        log.info(self.info.distribution_type)
2883        type = self.info.distribution_type.lower()
2884        if type == 'windows':
2885            product = "mb"
2886            query = BuildQuery()
2887            builds, changes = query.get_all_builds()
2888            os_type = "exe"
2889            task = "uninstall"
2890            bat_file = "uninstall.bat"
2891            deleted = False
2892            product_name = "membase-server-enterprise"
2893            version_path = "/cygdrive/c/Program Files/Membase/Server/"
2894
2895            exist = self.file_exists(version_path, VERSION_FILE)
2896            log.info("Is VERSION file existed? {0}".format(exist))
2897            if exist:
2898                log.info("VERSION file exists.  Start to uninstall")
2899                build_name, short_version, full_version = self.find_build_version(version_path, VERSION_FILE, product)
2900                if "1.8.0" in full_version or "1.8.1" in full_version:
2901                    product_name = "couchbase-server-enterprise"
2902                    product = "cb"
2903                log.info('Build name: {0}'.format(build_name))
2904                build_name = build_name.rstrip() + ".exe"
2905                log.info('Check if {0} is in tmp directory'.format(build_name))
2906                exist = self.file_exists("/cygdrive/c/tmp/", build_name)
2907                if not exist:  # if not exist in tmp dir, start to download that version build
2908                    build = query.find_build(builds, product_name, os_type, self.info.architecture_type, full_version)
2909                    downloaded = self.download_binary_in_win(build.url, short_version)
2910                    if downloaded:
2911                        log.info('Successful download {0}_{1}.exe'.format(product, short_version))
2912                    else:
2913                        log.error('Download {0}_{1}.exe failed'.format(product, short_version))
2914                dir_paths = ['/cygdrive/c/automation', '/cygdrive/c/tmp']
2915                self.create_multiple_dir(dir_paths)
2916                self.copy_files_local_to_remote('resources/windows/automation', '/cygdrive/c/automation')
2917                # modify bat file to run uninstall schedule task
2918                #self.create_windows_capture_file(task, product, full_version)
2919                self.modify_bat_file('/cygdrive/c/automation', bat_file, product, short_version, task)
2920                self.stop_schedule_tasks()
2921                self.sleep(5, "before running task schedule uninstall")
2922                # run schedule task uninstall Couchbase server
2923                output, error = self.execute_command("cmd /c schtasks /run /tn removeme")
2924                self.log_command_output(output, error)
2925                deleted = self.wait_till_file_deleted(version_path, VERSION_FILE, timeout_in_seconds=600)
2926                if not deleted:
2927                    log.error("Uninstall was failed at node {0}".format(self.ip))
2928                    sys.exit()
2929                ended = self.wait_till_process_ended(full_version[:10])
2930                if not ended:
2931                    sys.exit("****  Node %s failed to uninstall  ****" % (self.ip))
2932                self.sleep(10, "next step is to install")
2933                output, error = self.execute_command("rm -f \
2934                       /cygdrive/c/automation/*_{0}_uninstall.iss".format(self.ip))
2935                self.log_command_output(output, error)
2936                output, error = self.execute_command("cmd /c schtasks /Query /FO LIST /TN removeme /V")
2937                self.log_command_output(output, error)
2938            else:
2939                log.info("No membase server on this server.  Free to install")
2940        elif type in LINUX_DISTRIBUTION_NAME:
2941            # uninstallation command is different
2942            if type == "ubuntu":
2943                uninstall_cmd = 'dpkg -r {0};dpkg --purge {1};' \
2944                                 .format('membase-server', 'membase-server')
2945                output, error = self.execute_command(uninstall_cmd)
2946                self.log_command_output(output, error)
2947            elif type in RPM_DIS_NAME:
2948                """ Sometimes, vm left with unfinish uninstall/install process.
2949                    We need to kill them before doing uninstall """
2950                output, error = self.execute_command("killall -9 rpm")
2951                self.log_command_output(output, error)
2952                uninstall_cmd = 'rpm -e {0}'.format('membase-server')
2953                log.info('running rpm -e to remove membase-server')
2954                output, error = self.execute_command(uninstall_cmd)
2955                self.log_command_output(output, error)
2956            self.terminate_processes(self.info, terminate_process_list)
2957            if not save_upgrade_config:
2958                self.remove_folders(linux_folders)
2959
2960    def moxi_uninstall(self):
2961        terminate_process_list = ["moxi"]
2962        self.extract_remote_info()
2963        log.info(self.info.distribution_type)
2964        type = self.info.distribution_type.lower()
2965        if type == 'windows':
2966            log.error("Not implemented")
2967        elif type == "ubuntu":
2968            uninstall_cmd = "dpkg -r {0};dpkg --purge {1};".format("moxi-server", "moxi-server")
2969            output, error = self.execute_command(uninstall_cmd)
2970            self.log_command_output(output, error)
2971        elif type in LINUX_DISTRIBUTION_NAME:
2972            uninstall_cmd = 'rpm -e {0}'.format("moxi-server")
2973            log.info('running rpm -e to remove couchbase-server')
2974            output, error = self.execute_command(uninstall_cmd)
2975            self.log_command_output(output, error)
2976        self.terminate_processes(self.info, terminate_process_list)
2977
2978    def log_command_output(self, output, error, track_words=(), debug=True):
2979        # success means that there are no track_words in the output
2980        # and there are no errors at all, if track_words is not empty
2981        # if track_words=(), the result is not important, and we return True
2982        success = True
2983        for line in error:
2984            if debug:
2985                log.error(line)
2986            if track_words:
2987                if "Warning" in line and "hugepages" in line:
2988                    log.info("There is a warning about transparent_hugepage "
2989                             "may be in used when install cb server.\
2990                              So we will disable transparent_hugepage in this vm")
2991                    output, error = self.execute_command("echo never > \
2992                                        /sys/kernel/mm/transparent_hugepage/enabled")
2993                    self.log_command_output(output, error)
2994                    success = True
2995                elif "Warning" in line and "systemctl daemon-reload" in line:
2996                    log.info("Unit file of couchbase-server.service changed on disk,"
2997                             " we will run 'systemctl daemon-reload'")
2998                    output, error = self.execute_command("systemctl daemon-reload")
2999                    self.log_command_output(output, error)
3000                    success = True
3001                elif "Warning" in line and "RPMDB altered outside of yum" in line:
3002                    log.info("Warming: RPMDB altered outside of yum")
3003                    success = True
3004                elif "dirname" in line:
3005                    log.warning("Ignore dirname error message during couchbase "
3006                                "startup/stop/restart for CentOS 6.6 (MB-12536)")
3007                    success = True
3008                elif "Created symlink from /etc/systemd/system" in line:
3009                    log.info("This error is due to fix_failed_install.py script that only "
3010                             "happens in centos 7")
3011                    success = True
3012                elif "Created symlink /etc/systemd/system/multi-user.target.wants/couchbase-server.service" in line:
3013                    log.info(line)
3014                    log.info("This message comes only in debian8 and debian9 during installation. "
3015                             "This can be ignored.")
3016                    success = True
3017                else:
3018                    log.info("\nIf couchbase server is running with this error."
3019                              "\nGo to log_command_output to add error mesg to bypass it.")
3020                    success = False
3021        for line in output:
3022            if debug:
3023                log.info(line)
3024            if any(s.lower() in line.lower() for s in track_words):
3025                if "Warning" in line and "hugepages" in line:
3026                    log.info("There is a warning about transparent_hugepage may be in used when install cb server.\
3027                              So we will disable transparent_hugepage in this vm")
3028                    output, error = self.execute_command("echo never > /sys/kernel/mm/transparent_hugepage/enabled")
3029                    self.log_command_output(output, error, debug=debug)
3030                    success = True
3031                else:
3032                    success = False
3033                    log.error('something wrong happened on {0}!!! output:{1}, error:{2}, track_words:{3}'
3034                              .format(self.ip, output, error, track_words))
3035        return success
3036
3037    def execute_commands_inside(self, main_command,query, queries,bucket1,password,bucket2,source,subcommands=[], min_output_size=0,
3038                                end_msg='', timeout=250):
3039        self.extract_remote_info()
3040        filename = "/tmp/test2"
3041        iswin=False
3042
3043        if self.info.type.lower() == 'windows':
3044            iswin = True
3045            filename = "/cygdrive/c/tmp/test.txt"
3046
3047        filedata = ""
3048        if not(query==""):
3049            main_command = main_command + " -s=\"" + query+ '"'
3050        elif (self.remote and not(queries=="")):
3051            sftp = self._ssh_client.open_sftp()
3052            filein = sftp.open(filename, 'w')
3053            for query in queries:
3054                filein.write(query)
3055                filein.write('\n')
3056            fileout = sftp.open(filename,'r')
3057            filedata = fileout.read()
3058            #print filedata
3059            fileout.close()
3060        elif not(queries==""):
3061            f = open(filename, 'w')
3062            for query in queries:
3063                f.write(query)
3064                f.write('\n')
3065            f.close()
3066            fileout = open(filename,'r')
3067            filedata = fileout.read()
3068            fileout.close()
3069
3070        newdata = filedata.replace("bucketname",bucket2)
3071        newdata = newdata.replace("user",bucket1)
3072        newdata = newdata.replace("pass",password)
3073        newdata = newdata.replace("bucket1",bucket1)
3074
3075        newdata = newdata.replace("user1",bucket1)
3076        newdata = newdata.replace("pass1",password)
3077        newdata = newdata.replace("bucket2",bucket2)
3078        newdata = newdata.replace("user2",bucket2)
3079        newdata = newdata.replace("pass2",password)
3080
3081        if (self.remote and not(queries=="")) :
3082            f = sftp.open(filename,'w')
3083            f.write(newdata)
3084            f.close()
3085        elif not(queries==""):
3086            f = open(filename,'w')
3087            f.write(newdata)
3088            f.close()
3089        if not(queries==""):
3090            if (source):
3091                if iswin:
3092                    main_command = main_command + "  -s=\"\SOURCE " + 'c:\\\\tmp\\\\test.txt'
3093                else:
3094                    main_command = main_command + "  -s=\"\SOURCE " + filename+ '"'
3095            else:
3096                if iswin:
3097                    main_command = main_command + " -f=" + 'c:\\\\tmp\\\\test.txt'
3098                else:
3099                    main_command = main_command + " -f=" + filename
3100
3101        log.info("running command on {0}: {1}".format(self.ip, main_command))
3102        output=""
3103        if self.remote:
3104            (stdin, stdout, stderro) = self._ssh_client.exec_command(main_command)
3105            time.sleep(10)
3106            count = 0
3107            for line in stdout.readlines():
3108                if (count == 0) and line.lower().find("error") > 0:
3109                   output = "status:FAIL"
3110                   break
3111
3112                #if line.find("results") > 0 or line.find("status") > 0 or line.find("metrics") or line.find("elapsedTime")> 0 or  line.find("executionTime")> 0 or line.find("resultCount"):
3113                if (count > 0):
3114                    output+=line.strip()
3115                    output = output.strip()
3116                    if "Inputwasnotastatement" in output:
3117                        output = "status:FAIL"
3118                        break
3119                    if "timeout" in output:
3120                        output = "status:timeout"
3121                else:
3122                    count+=1
3123            stdin.close()
3124            stdout.close()
3125            stderro.close()
3126           # main_command = main_command + " < " + '/tmp/' + filename
3127           # stdin,stdout, ssh_stderr = ssh.exec_command(main_command)
3128           # stdin.close()
3129           # output = []
3130           # for line in stdout.read().splitlines():
3131           #   print(line)
3132           #   output = output.append(line)
3133           # f.close()
3134           # ssh.close()
3135
3136           #output = output + end_msg
3137
3138        else:
3139            p = Popen(main_command , shell=True, stdout=PIPE, stderr=PIPE)
3140            stdout, stderro = p.communicate()
3141            output = stdout
3142            print output
3143            time.sleep(1)
3144        # for cmd in subcommands:
3145        #       log.info("running command {0} inside {1} ({2})".format(
3146        #                                                 main_command, cmd, self.ip))
3147        #       stdin.channel.send("{0}\n".format(cmd))
3148        #       end_time = time.time() + float(timeout)
3149        #       while True:
3150        #           if time.time() > end_time:
3151        #               raise Exception("no output in {3} sec running command \
3152        #                                {0} inside {1} ({2})".format(main_command,
3153        #                                                             cmd, self.ip,
3154        #                                                             timeout))
3155        #           output = stdout.channel.recv(1024)
3156        #           if output.strip().endswith(end_msg) and len(output) >= min_output_size:
3157        #                   break
3158        #           time.sleep(2)
3159        #       log.info("{0}:'{1}' -> '{2}' output\n: {3}".format(self.ip, main_command, cmd, output))
3160        # stdin.close()
3161        # stdout.close()
3162        # stderro.close()
3163        if (self.remote and not(queries=="")) :
3164            sftp.remove(filename)
3165            sftp.close()
3166        elif not(queries==""):
3167            os.remove(filename)
3168
3169        output = re.sub('\s+', '', output)
3170        return (output)
3171
3172    def execute_command(self, command, info=None, debug=True, use_channel=False):
3173        if getattr(self, "info", None) is None and info is not None :
3174            self.info = info
3175        else:
3176            self.extract_remote_info()
3177
3178        if self.info.type.lower() == 'windows':
3179            self.use_sudo = False
3180
3181        if self.use_sudo:
3182            command = "sudo " + command
3183
3184        return self.execute_command_raw(command, debug=debug, use_channel=use_channel)
3185
3186    def execute_command_raw(self, command, debug=True, use_channel=False, timeout=600):
3187        if debug:
3188            log.info("running command.raw on {0}: {1}".format(self.ip, command))
3189        output = []
3190        error = []
3191        temp = ''
3192        if self.remote and self.use_sudo or use_channel:
3193            channel = self._ssh_client.get_transport().open_session()
3194            channel.get_pty()
3195            channel.settimeout(900)
3196            stdin = channel.makefile('wb')
3197            stdout = channel.makefile('rb')
3198            stderro = channel.makefile_stderr('rb')
3199            channel.exec_command(command)
3200            data = channel.recv(1024)
3201            while data:
3202                temp += data
3203                data = channel.recv(1024)
3204            channel.close()
3205            stdin.close()
3206        elif self.remote:
3207            stdin, stdout, stderro = self._ssh_client.exec_command(command, timeout=timeout)
3208            stdin.close()
3209
3210        if not self.remote:
3211            p = Popen(command , shell=True, stdout=PIPE, stderr=PIPE)
3212            output, error = p.communicate()
3213
3214        if self.remote:
3215            for line in stdout.read().splitlines():
3216                output.append(line)
3217            for line in stderro.read().splitlines():
3218                error.append(line)
3219            if temp:
3220                line = temp.splitlines()
3221                output.extend(line)
3222            stdout.close()
3223            stderro.close()
3224        if debug:
3225            log.info('command executed successfully')
3226        return output, error
3227
3228    def execute_non_sudo_command(self, command, info=None, debug=True, use_channel=False):
3229        info = info or self.extract_remote_info()
3230        self.info = info
3231
3232        return self.execute_command_raw(command, debug=debug, use_channel=use_channel)
3233
3234    def terminate_process(self, info=None, process_name='',force=False):
3235        self.extract_remote_info()
3236        if self.info.type.lower() == 'windows':
3237            o, r = self.execute_command("taskkill /F /T /IM {0}*"\
3238                                            .format(process_name), debug=False)
3239            self.log_command_output(o, r)
3240        else:
3241            if (force == True):
3242                o, r = self.execute_command("kill -9 "
3243                   "$(ps aux | grep '{0}' |  awk '{{print $2}}') "\
3244                               .format(process_name), debug=False)
3245                self.log_command_output(o, r)
3246            else:
3247                o, r = self.execute_command("kill "
3248                    "$(ps aux | grep '{0}' |  awk '{{print $2}}') "\
3249                                 .format(process_name), debug=False)
3250                self.log_command_output(o, r)
3251
3252    def disconnect(self):
3253        self._ssh_client.close()
3254
3255    def extract_remote_info(self):
3256        # initialize params
3257        os_distro = "linux"
3258        os_version = "default"
3259        is_linux_distro = True
3260        self.use_sudo = False
3261        is_mac = False
3262        arch = "local"
3263        ext = "local"
3264        # use ssh to extract remote machine info
3265        # use sftp to if certain types exists or not
3266        if getattr(self, "info", None) is not None and isinstance(self.info, RemoteMachineInfo):
3267            return self.info
3268        mac_check_cmd = "sw_vers | grep ProductVersion | awk '{ print $2 }'"
3269        if self.remote:
3270            stdin, stdout, stderro = self._ssh_client.exec_command(mac_check_cmd)
3271            stdin.close()
3272            ver, err = stdout.read(), stderro.read()
3273        else:
3274            p = Popen(mac_check_cmd , shell=True, stdout=PIPE, stderr=PIPE)
3275            ver, err = p.communicate()
3276        if not err and ver:
3277            os_distro = "Mac"
3278            os_version = ver
3279            is_linux_distro = True
3280            is_mac = True
3281            self.use_sudo = False
3282        elif self.remote:
3283            is_mac = False
3284            sftp = self._ssh_client.open_sftp()
3285            filenames = sftp.listdir('/etc/')
3286            os_distro = ""
3287            os_version = ""
3288            is_linux_distro = False
3289            for name in filenames:
3290                if name == 'issue':
3291                    # it's a linux_distro . let's downlaod this file
3292                    # format Ubuntu 10.04 LTS \n \l
3293                    filename = 'etc-issue-{0}'.format(uuid.uuid4())
3294                    sftp.get(localpath=filename, remotepath='/etc/issue')
3295                    file = open(filename)
3296                    etc_issue = ''
3297                    # let's only read the first line
3298                    for line in file.xreadlines():
3299                        # for SuSE that has blank first line
3300                        if line.rstrip('\n'):
3301                            etc_issue = line
3302                            break
3303                        # strip all extra characters
3304                    etc_issue = etc_issue.rstrip('\n').rstrip(' ').rstrip('\\l').rstrip(' ').rstrip('\\n').rstrip(' ')
3305                    if etc_issue.lower().find('ubuntu') != -1:
3306                        os_distro = 'Ubuntu'
3307                        os_version = etc_issue
3308                        tmp_str = etc_issue.split()
3309                        if tmp_str and tmp_str[1][:2].isdigit():
3310                            os_version = "Ubuntu %s" % tmp_str[1][:5]
3311                        is_linux_distro = True
3312                    elif etc_issue.lower().find('debian') != -1:
3313                        os_distro = 'Ubuntu'
3314                        os_version = etc_issue
3315                        is_linux_distro = True
3316                    elif etc_issue.lower().find('mint') != -1:
3317                        os_distro = 'Ubuntu'
3318                        os_version = etc_issue
3319                        is_linux_distro = True
3320                    elif etc_issue.lower().find('amazon linux ami') != -1:
3321                        os_distro = 'CentOS'
3322                        os_version = etc_issue
3323                        is_linux_distro = True
3324                    elif etc_issue.lower().find('centos') != -1:
3325                        os_distro = 'CentOS'
3326                        os_version = etc_issue
3327                        is_linux_distro = True
3328                    elif etc_issue.lower().find('red hat') != -1:
3329                        os_distro = 'Red Hat'
3330                        os_version = etc_issue
3331                        is_linux_distro = True
3332                    elif etc_issue.lower().find('opensuse') != -1:
3333                        os_distro = 'openSUSE'
3334                        os_version = etc_issue
3335                        is_linux_distro = True
3336                    elif etc_issue.lower().find('suse linux') != -1:
3337                        os_distro = 'SUSE'
3338                        os_version = etc_issue
3339                        tmp_str = etc_issue.split()
3340                        if tmp_str and tmp_str[6].isdigit():
3341                            os_version = "SUSE %s" % tmp_str[6]
3342                        is_linux_distro = True
3343                    elif etc_issue.lower().find('oracle linux') != -1:
3344                        os_distro = 'Oracle Linux'
3345                        os_version = etc_issue
3346                        is_linux_distro = True
3347                    else:
3348                        log.info("It could be other operating system."
3349                                 "  Go to check at other location")
3350                    file.close()
3351                    # now remove this file
3352                    os.remove(filename)
3353                    break
3354            else:
3355                os_distro = "linux"
3356                os_version = "default"
3357                is_linux_distro = True
3358                self.use_sudo = False
3359                is_mac = False
3360                arch = "local"
3361                ext = "local"
3362                filenames = []
3363            """ for Amazon Linux 2 only"""
3364            for name in filenames:
3365                if name == 'system-release' and os_distro == "":
3366                    # it's a amazon linux 2_distro . let's download this file
3367                    filename = 'amazon-linux2-release-{0}'.format(uuid.uuid4())
3368                    sftp.get(localpath=filename, remotepath='/etc/system-release')
3369                    file = open(filename)
3370                    etc_issue = ''
3371                    # let's only read the first line
3372                    for line in file.xreadlines():
3373                        # for SuSE that has blank first line
3374                        if line.rstrip('\n'):
3375                            etc_issue = line
3376                            break
3377                            # strip all extra characters
3378                    etc_issue = etc_issue.rstrip('\n').rstrip(' ').rstrip('\\l').rstrip(' ').rstrip('\\n').rstrip(' ')
3379                    os_distro = 'Amazon Linux 2'
3380                    os_version = etc_issue
3381                    is_linux_distro = True
3382                    file.close()
3383                    # now remove this file
3384                    os.remove(filename)
3385                    break
3386            """ for centos 7 or rhel8 """
3387            for name in filenames:
3388                if name == "redhat-release":
3389                    filename = 'redhat-release-{0}'.format(uuid.uuid4())
3390                    if self.remote:
3391                        sftp.get(localpath=filename, remotepath='/etc/redhat-release')
3392                    else:
3393                        p = Popen("cat /etc/redhat-release > {0}".format(filename) , shell=True, stdout=PIPE, stderr=PIPE)
3394                        var, err = p.communicate()
3395                    file = open(filename)
3396                    redhat_release = ''
3397                    for line in file.xreadlines():
3398                        redhat_release = line
3399                        break
3400                    redhat_release = redhat_release.rstrip('\n').rstrip('\\l').rstrip('\\n')
3401                    """ in ec2: Red Hat Enterprise Linux Server release 7.2 """
3402                    if redhat_release.lower().find('centos') != -1 \
3403                         or redhat_release.lower().find('linux server') != -1 \
3404                         or redhat_release.lower().find('red hat') != -1:
3405                        if redhat_release.lower().find('release 7') != -1:
3406                            os_distro = 'CentOS'
3407                            os_version = "CentOS 7"
3408                            is_linux_distro = True
3409                        elif redhat_release.lower().find('red hat enterprise') != -1:
3410                            if "8.0" in redhat_release.lower():
3411                                os_distro = "Red Hat"
3412                                os_version = "rhel8"
3413                                is_linux_distro = True
3414                    else:
3415                        log.error("Could not find OS name."
3416                                 "It could be unsupport OS")
3417                    file.close()
3418                    os.remove(filename)
3419                    break
3420
3421        if self.remote:
3422            if self.find_file("/cygdrive/c/Windows", "win.ini"):
3423                log.info("This is windows server!")
3424                is_linux_distro = False
3425        if not is_linux_distro:
3426            arch = ''
3427            os_version = 'unknown windows'
3428            win_info = self.find_windows_info()
3429            info = RemoteMachineInfo()
3430            info.type = win_info['os']
3431            info.windows_name = win_info['os_name']
3432            info.distribution_type = win_info['os']
3433            info.architecture_type = win_info['os_arch']
3434            info.ip = self.ip
3435            info.distribution_version = win_info['os']
3436            info.deliverable_type = 'exe'
3437            info.cpu = self.get_cpu_info(win_info)
3438            info.disk = self.get_disk_info(win_info)
3439            info.ram = self.get_ram_info(win_info)
3440            info.hostname = self.get_hostname(win_info)
3441            info.domain = self.get_domain(win_info)
3442            self.info = info
3443            return info
3444        else:
3445            # now run uname -m to get the architechtre type
3446            if self.remote:
3447                stdin, stdout, stderro = self._ssh_client.exec_command('uname -m')
3448                stdin.close()
3449                os_arch = ''
3450                text = stdout.read().splitlines()
3451            else:
3452                p = Popen('uname -m' , shell=True, stdout=PIPE, stderr=PIPE)
3453                text, err = p.communicate()
3454                os_arch = ''
3455            for line in text:
3456                os_arch += line
3457                # at this point we should know if its a linux or windows ditro
3458            ext = { 'Ubuntu' : "deb",
3459                   'CentOS'  : "rpm",
3460                   'Red Hat' : "rpm",
3461                   "Mac"     : "zip",
3462                   "Debian"  : "deb",
3463                   "openSUSE": "rpm",
3464                   "SUSE"    : "rpm",
3465                   "Oracle Linux": "rpm",
3466                    "Amazon Linux 2": "rpm"}.get(os_distro, '')
3467            arch = {'i686': 'x86',
3468                    'i386': 'x86'}.get(os_arch, os_arch)
3469
3470            info = RemoteMachineInfo()
3471            info.type = "Linux"
3472            info.distribution_type = os_distro
3473            info.architecture_type = arch
3474            info.ip = self.ip
3475            info.distribution_version = os_version
3476            info.deliverable_type = ext
3477            info.cpu = self.get_cpu_info(mac=is_mac)
3478            info.disk = self.get_disk_info(mac=is_mac)
3479            info.ram = self.get_ram_info(mac=is_mac)
3480            info.hostname = self.get_hostname()
3481            info.domain = self.get_domain()
3482            self.info = info
3483            return info
3484
3485    def get_extended_windows_info(self):
3486        info = {}
3487        win_info = self.extend_windows_info()
3488        info['ram'] = self.get_ram_info(win_info)
3489        info['disk'] = self.get_disk_info(win_info)
3490        info['cpu'] = self.get_cpu_info(win_info)
3491        info['hostname'] = self.get_hostname()
3492        return info
3493
3494    def get_hostname(self, win_info=None):
3495        if win_info:
3496            if 'Host Name' not in win_info:
3497                win_info = self.create_windows_info()
3498            o = win_info['Host Name']
3499        o, r = self.execute_command_raw('hostname', debug=False)
3500        if o:
3501            return o
3502
3503    def get_domain(self, win_info=None):
3504        if win_info:
3505            o, _ = self.execute_batch_command('ipconfig')
3506            suffix_dns_row = [row for row in o if row.find(" Connection-specific DNS Suffix") != -1 and \
3507                             len(row.split(':')[1]) > 1]
3508            if suffix_dns_row ==[]:
3509                #'   Connection-specific DNS Suffix  . : '
3510                ret=""
3511            else:
3512                ret = suffix_dns_row[0].split(':')[1].strip()
3513        else:
3514            ret = self.execute_command_raw('hostname -d', debug=False)
3515        return ret
3516
3517    def get_aws_public_hostname(self):
3518        #AWS supported url to retrieve metadata like public hostname of an instance from shell
3519        output, _ = self.execute_command("curl -s http://169.254.169.254/latest/meta-data/public-hostname")
3520        return output[0]
3521
3522    def get_full_hostname(self):
3523        info = self.extract_remote_info()
3524        if not info.domain:
3525            return None
3526        log.info("hostname of this {0} is {1}"
3527                 .