1import logger
2import unittest
3from TestInput import TestInputSingleton
4from remote.remote_util import RemoteMachineShellConnection, RemoteMachineHelper
5import re
6import time
7
8try:
9    import requests
10    from requests.exceptions import ConnectionError
11except ImportError as e:
12    print 'please install required modules:', e
13    raise
14
15
16class GatewayBaseTest(unittest.TestCase):
17    BUILDS = {
18        'http://latestbuilds.hq.couchbase.com/couchbase-sync-gateway': (
19            'release/1.1.0/{0}/couchbase-sync-gateway-community_{0}_{1}.{2}',
20            'release/1.1.1/{0}/couchbase-sync-gateway-community_{0}_{1}.{2}',
21            '1.1.0/{0}/couchbase-sync-gateway-community_{0}_{1}.{2}',
22            '0.0.0/{0}/couchbase-sync-gateway-community_{0}_{1}.{2}',
23            '0.0.1/{0}/couchbase-sync-gateway-community_{0}_{1}.{2}'
24        ),
25    }
26
27    def setUp(self):
28        super(GatewayBaseTest, self).setUp()
29        self.log = logger.Logger.get_logger()
30        self.input = TestInputSingleton.input
31        self.case_number = self.input.param("case_number", 0)
32        self.version = self.input.param("version_sg", "0.0.0-358")
33        self.extra_param = self.input.param("extra_param", "")
34        if isinstance(self.extra_param, str):
35            self.extra_param = self.extra_param.replace("$", "=")  # '=' is a delimiter in conf file
36        self.logsdir = self.input.param("logsdir", "/tmp/sync_gateway/logs")
37        self.datadir = self.input.param("datadir", "/tmp/sync_gateway")
38        self.configdir = self.input.param("configdir", "/tmp/sync_gateway")
39        self.configfile = self.input.param("configfile", "sync_gateway.json")
40        self.expected_error = self.input.param("expected_error", "")
41        self.servers = self.input.servers
42        self.master = self.servers[0]
43        self.folder_prefix = ""
44        self.installed_folder = '/opt/couchbase-sync-gateway/bin'
45        shell = RemoteMachineShellConnection(self.master)
46        type = shell.extract_remote_info().distribution_type
47        shell.disconnect()
48        if type.lower() == 'windows':
49            self.folder_prefix = "/cygdrive/c"
50            self.installed_folder = '/cygdrive/c/Program\ Files\ \(x86\)/Couchbase'
51            self.logsdir = self.folder_prefix + self.configdir
52            self.datadir = self.folder_prefix + self.configdir
53            self.configdir = self.folder_prefix + self.configdir
54
55    def find_package(self, shell):
56        for filename, url in self.get_expected_locations(shell):
57            try:
58                self.log.info('Trying to get "{0}"'.format(url))
59                status_code = requests.head(url).status_code
60            except ConnectionError:
61                pass
62            else:
63                if status_code == 200:
64                    self.log.info('Found "{0}"'.format(url))
65                    return filename, url
66        self.log.fatal('Target build not found')
67
68    def get_expected_locations(self, shell):
69        self.info = shell.extract_remote_info()
70        distribution_type = self.info.distribution_type.lower()
71        if self.info.type.lower() == 'linux':
72            if distribution_type == 'centos':
73                file_ext = 'rpm'
74            elif distribution_type == 'ubuntu':
75                file_ext = 'deb'
76            elif distribution_type == 'mac':
77                file_ext = 'tar.gz'
78        elif self.info.type.lower() == 'windows':
79            file_ext = 'exe'
80
81        for location, patterns in self.BUILDS.items():
82            for pattern in patterns:
83                url = '{0}/{1}'.format(location, pattern.format(self.version, self.info.architecture_type, file_ext))
84                filename = url.split('/')[-1]
85                yield filename, url
86
87    def install(self, shell):
88        self.kill_processes_gateway(shell)
89        self.uninstall_gateway(shell)
90        self.install_gateway(shell)
91        self.start_sync_gateways(shell)
92
93    def kill_processes_gateway(self, shell):
94        self.log.info('=== Killing Sync Gateway')
95        shell.terminate_process(process_name='sync_gateway')
96        shell.execute_command('killall sync_gateway')
97        shell.execute_command('pkill sync_gateway')  # centos 7
98
99    def uninstall_gateway(self, shell):
100        self.info = shell.extract_remote_info()
101        type = self.info.type.lower()
102        distribution_type = self.info.distribution_type.lower()
103        if self.info.type.lower() == 'linux':
104            if distribution_type == 'centos':
105                cmd = 'yes | yum remove couchbase-sync-gateway'
106            elif distribution_type == 'ubuntu':
107                cmd = 'dpkg -r couchbase-sync-gateway'
108            elif distribution_type == 'mac':
109                cmd = 'sudo launchctl unload /Library/LaunchDaemons/com.couchbase.mobile.sync_gateway.plist'
110            else:
111                self.log.info('uninstall_gateway is not supported on {0}, {1}'.format(type, distribution_type))
112            self.log.info('=== Un-installing Sync Gateway package on {0} - cmd: {1}'.format(self.master, cmd))
113            output, error = shell.execute_command(cmd)
114            shell.log_command_output(output, error)
115        elif self.info.type.lower() == 'windows':
116            cmd = "wmic product where name='Couchbase Sync Gateway' uninstall"
117            self.log.info('=== Un-installing Sync Gateway package on {0} - cmd: {1}'.format(self.master, cmd))
118            output, error = shell.execute_batch_command(cmd)
119            shell.log_command_output(output, error)
120
121    def install_gateway(self, shell):
122        filename, url = self.find_package(shell)
123        type = shell.extract_remote_info().type.lower()
124        distribution_type = self.info.distribution_type.lower()
125        if type == 'linux':
126            wget_str = 'wget'
127            if distribution_type == 'centos':
128                cmd = 'yes | rpm -i /tmp/{0}'.format(filename)
129            elif distribution_type == 'ubuntu':
130                cmd = 'yes | dpkg -i /tmp/{0}'.format(filename)
131            elif distribution_type == 'mac':
132                filename_tar = re.sub('\.gz$', '', filename)
133                cmd = 'cd /tmp; gunzip {0}; tar xopf {1}; cp -r couchbase-sync-gateway /opt' \
134                    .format(filename, filename_tar, filename_tar)
135                wget_str = '/usr/local/bin/wget'
136            else:
137                self.log.info('install_gateway is not supported on {0}, {1}'.format(type, distribution_type))
138            output, error = shell.execute_command_raw(
139                'cd /tmp; {0} -q -O {1} {2};ls -lh'.format(wget_str, filename, url))
140
141        elif type == 'windows':
142            cmd = "cd /cygdrive/c/tmp;cmd /c 'couchbase-sync-gateway.exe /S /v/qn'"  # couchbase-sync-gateway.exe /v"/l*v C:\install.log /qb
143            output, error = shell.execute_command(
144                "cd /cygdrive/c/tmp;cmd /c 'c:\\automation\\wget.exe --no-check-certificate -q"
145                " {0} -O {1}.exe';ls -lh;".format(url, "couchbase-sync-gateway"))
146        shell.log_command_output(output, error)
147        output, error = shell.execute_command_raw(cmd)
148        shell.log_command_output(output, error)
149        if type == 'windows':
150            exist = shell.file_exists('/cygdrive/c/Program Files (x86)/Couchbase', 'sync_gateway.exe')
151        else:
152            exist = shell.file_exists(self.installed_folder.replace("\\", ""), 'sync_gateway')
153        if exist:
154            return True
155        else:
156            self.log.error('Installed file not found - {0}/sync_gateway'.format(self.installed_folder))
157            return False
158
159    def start_sync_gateways(self, shell):
160        type = shell.extract_remote_info().type.lower()
161        self.log.info('=== Starting Sync Gateway instances')
162        shell.copy_files_local_to_remote('pytests/sg/resources', self.folder_prefix + '/tmp')
163        if type == 'windows':
164            output, error = shell.execute_command_raw('{0}/sync_gateway.exe'
165                                                      ' c:/tmp/gateway_config.json > {1}/tmp/gateway.log 2>&1 &'.
166                                                      format(self.installed_folder, self.folder_prefix))
167        else:
168            output, error = shell.execute_command_raw('nohup {0}/sync_gateway'
169                                                      ' /tmp/gateway_config.json >/tmp/gateway.log 2>&1 &'.
170                                                      format(self.installed_folder))
171        shell.log_command_output(output, error)
172
173    def start_simpleServe(self, shell):
174        self.log.info('=== Starting SimpleServe instances')
175        shell.copy_file_local_to_remote("pytests/sg/simpleServe.go",
176                                        "{0}/tmp/simpleServe.go".format(self.folder_prefix))
177        type = shell.extract_remote_info().type.lower()
178        if type == 'windows':
179            shell.terminate_process(process_name='simpleServe.exe')
180            output, error = shell.execute_command_raw('c:/Go/bin/go.exe run c:/tmp/simpleServe.go 8081'
181                                                      ' >{0}/tmp/simpleServe.txt 2>&1 &'.format(self.folder_prefix))
182        else:
183            shell.terminate_process(process_name='simpleServe')
184            shell.execute_command("kill $(ps aux | grep '8081' | awk '{print $2}')")
185            output, error = shell.execute_command_raw('go run {0}/tmp/simpleServe.go 8081'
186                                                      '  >{0}/tmp/simpleServe.txt 2>&1 &'.format(self.folder_prefix))
187        shell.log_command_output(output, error)
188
189    def add_user(self, shell, user_name):
190        self.log.info('=== Add user {0}'.format(user_name))
191        self.info = shell.extract_remote_info()
192        distribution_type = self.info.distribution_type.lower()
193        if self.info.type.lower() == 'linux':
194            if distribution_type == 'centos' or distribution_type == 'ubuntu':
195                if not self.check_user(shell, user_name):
196                    output, error = shell.execute_command_raw('useradd -m {0}'.format(user_name))
197                    shell.log_command_output(output, error)
198                return self.check_user(shell, user_name)
199            else:
200                self.log.info('add_user is not supported on Mac/Wind')
201                return False
202        else:
203            type = self.info.type.lower()
204            self.log.info('add_user is not supported on {0}, {1}'.format(type, distribution_type))
205            return False
206
207    def remove_user(self, shell, user_name):
208        self.log.info('=== Remove user {0}'.format(user_name))
209        self.info = shell.extract_remote_info()
210        type = self.info.type.lower()
211        distribution_type = self.info.distribution_type.lower()
212        if type == 'linux':
213            if distribution_type == 'centos' or distribution_type == 'ubuntu':
214                if self.check_user(shell, user_name):
215                    output, error = shell.execute_command_raw('userdel {0}'.format(user_name))
216                    shell.log_command_output(output, error)
217                return not self.check_user(shell, user_name)
218            else:
219                self.log.info('remove_user is only supported on centos and ubuntu')
220                return False
221        else:
222            self.log.info('remove_user is not supported on {0}, {1}'.format(type, distribution_type))
223            return False
224
225    def check_user(self, shell, user_name):
226        self.log.info('=== Check user {0} exists'.format(user_name))
227        self.info = shell.extract_remote_info()
228        distribution_type = self.info.distribution_type.lower()
229        if self.info.type.lower() == 'linux':
230            if distribution_type == 'centos' or distribution_type == 'ubuntu':
231                output, error = shell.execute_command_raw('cat /etc/passwd | grep {0} '.format(user_name))
232                shell.log_command_output(output, error)
233                if output and output[0].startswith('{0}:'.format(user_name)):
234                    self.log.info('User {0} exists'.format(user_name))
235                    return True
236                else:
237                    self.log.info('User {0} does not exists'.format(user_name))
238                    return False
239            else:
240                self.log.info('check_user is only supported on centos and ubuntu')
241                return False
242        else:
243            type = self.info.type.lower()
244            self.log.info('check_user is not supported on {0}, {1}', type, distribution_type)
245            return False
246
247    def is_sync_gateway_service_running(self, shell):
248        self.info = shell.extract_remote_info()
249        type = self.info.type.lower()
250        distribution_type = self.info.distribution_type.lower()
251        distribution_version = self.info.distribution_version
252        if self.info.type.lower() == 'linux':
253            if distribution_type == 'centos' and distribution_version == 'CentOS 7':
254                cmd = 'systemctl | grep sync_gateway'
255                service_str = 'sync_gateway.service                                                       ' \
256                              '               loaded active running'
257            elif distribution_type == 'ubuntu' or \
258                    (distribution_type == 'centos' and distribution_version != 'CentOS 7'):
259                cmd = 'initctl list | grep sync_gateway'
260                service_str = 'sync_gateway start/running, process '
261            elif distribution_type == 'mac':
262                cmd = 'launchctl  list | grep sync_gateway'
263                service_str = ''
264            else:
265                self.log.info('is_sync_gateway_service_running is not supported on {0}, {1}'
266                              .format(type, distribution_type))
267                return False
268        else:
269            self.log.info(
270                'is_sync_gateway_service_running is not supported on {0}, {1}'.format(type, distribution_type))
271            return False
272
273        self.log.info('=== Check whether Sync Gateway service is running on {0} - cmd: {1}'.format(self.master, cmd))
274        output, error = shell.execute_command_raw(cmd)
275        shell.log_command_output(output, error)
276        if output and output[0].startswith(service_str):
277            self.log.info('Sync Gateway service is running')
278            return True
279        else:
280            self.log.info('Sync Gateway service is NOT running')
281            return False
282
283    def start_gateway_service(self, shell):
284        self.info = shell.extract_remote_info()
285        type = self.info.type.lower()
286        distribution_type = self.info.distribution_type.lower()
287        distribution_version = self.info.distribution_version
288        if self.info.type.lower() == 'linux':
289            if distribution_type == 'centos':
290                if distribution_version == 'CentOS 7':
291                    cmd = 'systemctl start sync_gateway'
292                else:
293                    cmd = 'start sync_gateway'
294            elif distribution_type == 'ubuntu':
295                cmd = 'service sync_gateway start'
296            elif distribution_type == 'mac':
297                cmd = 'launchctl start com.couchbase.mobile.sync_gateway'
298            else:
299                self.log.info('start_gateway_service is not supported on {0}, {1}'.format(type, distribution_type))
300                return False
301        else:
302            self.log.info('start_gateway_service is not supported on {0}, {1}'.format(type, distribution_type))
303            return False
304
305        self.log.info('=== Starting gateway service')
306        output, error = shell.execute_command_raw(cmd)
307        return output, error
308
309    def stop_gateway_service(self, shell):
310        self.info = shell.extract_remote_info()
311        distribution_type = self.info.distribution_type.lower()
312        distribution_version = self.info.distribution_version
313        if self.info.type.lower() == 'linux':
314            if distribution_type == 'centos':
315                if (distribution_version == 'CentOS 7'):
316                    cmd = 'systemctl stop sync_gateway'
317                else:
318                    cmd = 'stop sync_gateway'
319            elif distribution_type == 'mac':
320                cmd = 'launchctl stop com.couchbase.mobile.sync_gateway'
321            else:
322                cmd = 'service sync_gateway stop'
323
324        self.log.info('=== Stopping gateway service')
325        output, error = shell.execute_command_raw(cmd)
326        return output, error
327
328    def is_sync_gateway_process_running(self, shell):
329        self.log.info('=== Check whether Sync Gateway process is running on {0}'.format(self.master))
330        obj = RemoteMachineHelper(shell).is_process_running('sync_gateway')
331        if obj and obj.pid:
332            self.log.info('Sync Gateway is running with pid of {0}'.format(obj.pid))
333            return True
334        else:
335            self.log.info('Sync Gateway is NOT running')
336            return False
337
338    def service_clean(self, shell):
339        self.log.info('=== Cleaning Sync Gateway service and remove files')
340        self.uninstall_gateway(shell)
341        shell.execute_command_raw(
342            'rm -rf /tmp/*.log /tmp/*.json /tmp/logs /tmp/data /tmp/sync_gateway/* '
343            '/dirNotExist /opt/couchbase-sync-gateway* /tmp/couchbase-sync-gateway '
344            '/tmp/couchbase-sync-gateway*tar')
345        # return true only if the service is not running
346        if self.is_sync_gateway_service_running(shell):
347            self.log.error('Fail to clean Sync Gateway service.   The service is still running')
348            return False
349        else:
350            return True
351
352    def service_install(self, shell):
353        self.install_gateway(shell)
354        self.run_sync_gateway_service_install(shell)
355
356    def run_sync_gateway_service_install(self, shell, options=''):
357        self.info = shell.extract_remote_info()
358        if self.servers[0].ssh_username == 'root':
359            cmd = 'cd /opt/couchbase-sync-gateway/service; . ./sync_gateway_service_install.sh '
360        else:
361            cmd = 'cd /opt/couchbase-sync-gateway/service;  echo {0} | sudo . ./sync_gateway_service_install.sh ' \
362                .format(self.servers[0].ssh_password)
363        output, error = shell.execute_command(cmd + options)
364        shell.log_command_output(output, error)
365        return output, error
366
367    def check_normal_error_output(self, shell, output, error):
368        self.info = shell.extract_remote_info()
369        type = self.info.type.lower()
370        distribution_type = self.info.distribution_type.lower()
371        distribution_version = self.info.distribution_version.lower()
372        if type == 'linux':
373            if distribution_type == 'centos' and distribution_version == 'centos 7':
374                if error[0].startswith('ln -s ') and 'sync_gateway.service' in error[0]:
375                    if output == []:
376                        return True
377                    else:
378                        self.log.error('check_normal_error_output: On CentOS 7, expect output to be null, but got {0}'
379                                       .format(output[0]))
380                        return False
381                else:
382                    self.log.error('check_normal_error_output: On CentOS 7, expect error: '
383                                   'ln -s \'/usr/lib/systemd/system/sync_gateway.service\'..., but got {0}'
384                                   .format(error[0]))
385                    return False
386            elif distribution_type == 'ubuntu' or \
387                    (distribution_type == 'centos' and distribution_version != 'centos 7'):
388                if error == []:
389                    if 'sync_gateway start/running' in output[0]:
390                        return True
391                    else:
392                        self.log.error('check_normal_error_output: On linux, expect output to be: '
393                                       '\'sync_gateway start/running\', but got {0}'.format(output[0]))
394                        return False
395                else:
396                    self.log.error('check_normal_error_output: On linux, expect error to be null, but got {0}'
397                                   .format(error[0]))
398                    return False
399            elif distribution_type == 'mac':
400                if error == []:
401                    expect_output = 'launchctl load /Library/LaunchDaemons/com.couchbase.mobile.sync_gateway.plist'
402                    if output[0] == expect_output:
403                        return True
404                    else:
405                        self.log.error('Expect output to be "{0}", but got {1}'.format(expect_output, output[0]))
406                        return False
407                else:
408                    self.log.error('Expect error to be null, but got {0}'.format(error[0]))
409                    return False
410            else:
411                self.log.info('check_normal_error_output is not supported on {0}, {1}'.format(type, distribution_type))
412                return False
413        else:
414            self.log.info('check_normal_error_output is not supported on {0}, {1}'.format(type, distribution_type))
415            return False
416
417    def check_job_already_running(self, shell, output, error):
418        self.info = shell.extract_remote_info()
419        type = self.info.type.lower()
420        distribution_type = self.info.distribution_type.lower()
421        distribution_version = self.info.distribution_version.lower()
422        if type == 'linux':
423            if (distribution_type == 'centos') and (distribution_version == 'centos 7'):
424                if error == []:
425                    if output == []:
426                        return True
427                    else:
428                        self.log.error('check_job_already_running: On CentOS 7, expect output to be null, but got {0}'
429                                       .format(output[0]))
430                        return False
431                else:
432                    self.log.error('check_job_already_running: On CentOS 7, expect error to be null, but got {0}'
433                                   .format(error[0]))
434                    return False
435            elif distribution_type == 'ubuntu' or distribution_type == 'mac' or \
436                    (distribution_type == 'centos' and distribution_version != 'centos 7'):
437                if 'is already running' in error[0]:
438                    if output == []:
439                        return True
440                    else:
441                        self.log.error('check_job_already_running: On linux, expect output to be null, but got {0}'
442                                       .format(output[0]))
443                        return False
444                else:
445                    self.log.error(
446                        'check_job_already_running: On linux, expect error to contain: \'is already running\' '
447                        ' but get {0}'.format(error[0]))
448                    return False
449            else:
450                self.log.info('check_job_already_running is not supported on {0}, {1}'.format(type, distribution_type))
451                return False
452        else:
453            self.log.info('check_job_already_running is not supported on {0}, {1}'.format(type, distribution_type))
454            return False
455
456    def check_status_in_gateway_log(self, shell):
457        logs = shell.read_remote_file('{0}/tmp/'.format(self.folder_prefix), 'gateway.log')[-5:]  # last 5 lines
458        self.log.info(logs)
459        status = re.search(".* got status (\w+)", logs[4])
460        if not status:
461            self.log.info('check_status_in_gateway_log failed, sync_gateway log has: {0}'.format(logs[4]))
462            return ''
463        else:
464            return status.group(1)
465
466    def check_message_in_gatewaylog(self, shell, expected_str):
467        if not expected_str:
468            return True
469        for i in range(3):
470            output, error = shell.execute_command_raw(
471                'grep \'{0}\' {1}/tmp/gateway.log'.format(expected_str, self.folder_prefix))
472            shell.log_command_output(output, error)
473            if not output or not output[0]:
474                if i < 2:
475                    time.sleep(1)
476                    continue
477                else:
478                    self.log.info('check_message_in_gatewaylog did not find expected error - {0}'.format(expected_str))
479                    output, error = shell.execute_command_raw('cat {0}/tmp/gateway.log'.format(self.folder_prefix))
480                    shell.log_command_output(output, error)
481                    return False
482            else:
483                return True
484