1952fee0dSpavel-paulau#!/usr/bin/env python
206a7d5d9Sfarshid
399d10dcbSSteve Yen# TODO: add installer support for membasez
414610b75Sfarshid
599d10dcbSSteve Yenimport getopt
6e81187faSsaigonimport copy, re
7fe259f4cSfarshidimport logging
8e0483781Ssaigonimport os
906a7d5d9Sfarshidimport sys
1006a7d5d9Sfarshidfrom threading import Thread
115ee062bbSKeith Battenfrom datetime import datetime
12f6f4c9ebSPavel.Paulauimport socket
13b9c46d85Sandreibaraimport Queue
1406a7d5d9Sfarshid
1525433975SKaran Kumarsys.path = [".", "lib"] + sys.path
16a607411eSfarshidimport testconstants
17a607411eSfarshidimport time
1806a7d5d9Sfarshidfrom builds.build_query import BuildQuery
19f607ec25Sfarshidimport logging.config
2006a7d5d9Sfarshidfrom membase.api.exception import ServerUnavailableException
2106a7d5d9Sfarshidfrom membase.api.rest_client import RestConnection, RestHelper
22b4777408SIrynaMironavafrom remote.remote_util import RemoteMachineShellConnection, RemoteUtilHelper
231f79a4e0SKeith Battenfrom membase.helper.cluster_helper import ClusterOperationHelper
24c1bf1505Ssaigonfrom testconstants import MV_LATESTBUILD_REPO
25a49a5579Ssaigonfrom testconstants import SHERLOCK_BUILD_REPO
266a023be1Ssaigonfrom testconstants import COUCHBASE_REPO
273c8a3212SAruna Piravifrom testconstants import CB_REPO, CB_DOWNLOAD_SERVER, CB_DOWNLOAD_SERVER_FQDN
28a13b0750Ssaigonfrom testconstants import COUCHBASE_VERSION_2
29c8a684a4Ssaigonfrom testconstants import COUCHBASE_VERSION_3, COUCHBASE_FROM_WATSON,\
30c8a684a4Ssaigon                          COUCHBASE_FROM_SPOCK
3162486499Ssaigonfrom testconstants import CB_VERSION_NAME, COUCHBASE_FROM_VERSION_4,\
32e81187faSsaigon                          CB_RELEASE_BUILDS, COUCHBASE_VERSIONS
33fce05e6bSRitesh Agarwalfrom testconstants import MIN_KV_QUOTA, INDEX_QUOTA, FTS_QUOTA, CBAS_QUOTA
3430b14716Sprasannafrom testconstants import LINUX_COUCHBASE_PORT_CONFIG_PATH, LINUX_COUCHBASE_OLD_CONFIG_PATH
3530b14716Sprasannafrom testconstants import WIN_COUCHBASE_PORT_CONFIG_PATH, WIN_COUCHBASE_OLD_CONFIG_PATH
3606a7d5d9Sfarshidimport TestInput
3706a7d5d9Sfarshid
3899d10dcbSSteve Yen
39f607ec25Sfarshidlogging.config.fileConfig("scripts.logging.conf")
40fe259f4cSfarshidlog = logging.getLogger()
41fe259f4cSfarshid
4299d10dcbSSteve Yendef usage(err=None):
4399d10dcbSSteve Yen    print """\
4499d10dcbSSteve YenSyntax: install.py [options]
4599d10dcbSSteve Yen
4699d10dcbSSteve YenOptions:
4799d10dcbSSteve Yen -p <key=val,...> Comma-separated key=value info.
4899d10dcbSSteve Yen -i <file>        Path to .ini file containing cluster information.
4999d10dcbSSteve Yen
5099d10dcbSSteve YenAvailable keys:
5130b14716Sprasanna product=cb|mb              Used to specify couchbase or membase.
5230b14716Sprasanna version=SHORT_VERSION      Example: "2.0.0r-71".
5330b14716Sprasanna parallel=false             Useful when you're installing a cluster.
5430b14716Sprasanna toy=                       Install a toy build
5530b14716Sprasanna init_nodes=False           Initialize nodes
5630b14716Sprasanna vbuckets=                  The number of vbuckets in the server installation.
5730b14716Sprasanna sync_threads=True          Sync or acync threads(+S or +A)
5830b14716Sprasanna erlang_threads=            Number of erlang threads (default=16:16 for +S type)
5930b14716Sprasanna upr=True                   Enable UPR replication
6030b14716Sprasanna xdcr_upr=                  Enable UPR for XDCR (temporary param until XDCR with UPR is stable), values: None | True | False
6130b14716Sprasanna fts_query_limit=1000000    Set a limit for the max results to be returned by fts for any query
6230b14716Sprasanna change_indexer_ports=false Sets indexer ports values to non-default ports
63be415535Sprasanna storage_mode=plasma        Sets indexer storage mode
647ef927e0SAruna Piravi enable_ipv6=False          Enable ipv6 mode in ns_server
6560c8290fSandreibara
6699d10dcbSSteve Yen
6799d10dcbSSteve YenExamples:
68329f692cSsaigon install.py -i /tmp/ubuntu.ini -p product=cb,version=2.2.0-792
69a89f6554Ssaigon install.py -i /tmp/ubuntu.ini -p product=cb,version=2.2.0-792,url=http://builds.hq.northscale.net/latestbuilds....
7074ac7dc1SKeith Batten install.py -i /tmp/ubuntu.ini -p product=mb,version=1.7.1r-38,parallel=true,toy=keith
71898ef744SSteve Yen install.py -i /tmp/ubuntu.ini -p product=mongo,version=2.0.2
726a023be1Ssaigon install.py -i /tmp/ubuntu.ini -p product=cb,version=0.0.0-704,toy=couchstore,parallel=true,vbuckets=1024
73329f692cSsaigon
74329f692cSsaigon # to run with build with require openssl version 1.0.0
75329f692cSsaigon install.py -i /tmp/ubuntu.ini -p product=cb,version=2.2.0-792,openssl=1
76f6ad5ef6Ssaigon
77f6ad5ef6Ssaigon # to install latest release of couchbase server via repo (apt-get and yum)
78f6ad5ef6Ssaigon  install.py -i /tmp/ubuntu.ini -p product=cb,linux_repo=true
79f6ad5ef6Ssaigon
80e0d52abdSsaigon # to install non-root non default path, add nr_install_dir params
81e0d52abdSsaigon   install.py -i /tmp/ubuntu.ini -p product=cb,version=5.0.0-1900,nr_install_dir=testnow1
82e0d52abdSsaigon
8399d10dcbSSteve Yen"""
8499d10dcbSSteve Yen    sys.exit(err)
8599d10dcbSSteve Yen
8699d10dcbSSteve Yen
8706a7d5d9Sfarshidproduct = "membase-server(ms),couchbase-single-server(css),couchbase-server(cs),zynga(z)"
8806a7d5d9Sfarshid
8906a7d5d9Sfarshiderrors = {"UNREACHABLE": "",
9006a7d5d9Sfarshid          "UNINSTALL-FAILED": "unable to uninstall the product",
9106a7d5d9Sfarshid          "INSTALL-FAILED": "unable to install",
9206a7d5d9Sfarshid          "BUILD-NOT-FOUND": "unable to find build",
9306a7d5d9Sfarshid          "INVALID-PARAMS": "invalid params given"}
9406a7d5d9Sfarshid
9514610b75Sfarshid
9614610b75Sfarshiddef installer_factory(params):
97898ef744SSteve Yen    if params.get("product", None) is None:
98898ef744SSteve Yen        sys.exit("ERROR: don't know what product you want installed")
99898ef744SSteve Yen
10014610b75Sfarshid    mb_alias = ["membase", "membase-server", "mbs", "mb"]
101fce05e6bSRitesh Agarwal    cb_alias = ["couchbase", "couchbase-server", "cb", "cbas"]
102fd0dbaeaSSubhashni Balakrishnan    sdk_alias = ["python-sdk", "pysdk"]
103f3c23bacSSubhashni Balakrishnan    es_alias = ["elasticsearch"]
104e081454fSKeith Batten    css_alias = ["couchbase-single", "couchbase-single-server", "css"]
105898ef744SSteve Yen    mongo_alias = ["mongo"]
1062724a9f1SIrynaMironava    moxi_alias = ["moxi", "moxi-server"]
107898ef744SSteve Yen
10814610b75Sfarshid    if params["product"] in mb_alias:
10984c3ae37Sandreibara        return MembaseServerInstaller()
11014610b75Sfarshid    elif params["product"] in cb_alias:
11184c3ae37Sandreibara        return CouchbaseServerInstaller()
112898ef744SSteve Yen    elif params["product"] in mongo_alias:
113898ef744SSteve Yen        return MongoInstaller()
1142724a9f1SIrynaMironava    elif params["product"] in moxi_alias:
1152724a9f1SIrynaMironava        return MoxiInstaller()
116fd0dbaeaSSubhashni Balakrishnan    elif params["product"] in sdk_alias:
117fd0dbaeaSSubhashni Balakrishnan        return SDKInstaller()
118f3c23bacSSubhashni Balakrishnan    elif params["product"] in es_alias:
119f3c23bacSSubhashni Balakrishnan        return ESInstaller()
120898ef744SSteve Yen
121898ef744SSteve Yen    sys.exit("ERROR: don't know about product " + params["product"])
12206a7d5d9Sfarshid
12306a7d5d9Sfarshid
12414610b75Sfarshidclass Installer(object):
12514610b75Sfarshid
12606a7d5d9Sfarshid    def install(self, params):
12706a7d5d9Sfarshid        pass
12806a7d5d9Sfarshid
12906a7d5d9Sfarshid    def initialize(self, params):
13006a7d5d9Sfarshid        pass
13106a7d5d9Sfarshid
13206a7d5d9Sfarshid    def uninstall(self, params):
13314610b75Sfarshid        remote_client = RemoteMachineShellConnection(params["server"])
134e4e8ec92Ssaigon        #remote_client.membase_uninstall()
135a6dbe12bSEric
1364af7601eSsaigon        self.msi = 'msi' in params and params['msi'].lower() == 'true'
137e63eb5b6SRitesh Agarwal        remote_client.couchbase_uninstall(windows_msi=self.msi,product=params['product'])
1389390b028Sandreibara        remote_client.disconnect()
13906a7d5d9Sfarshid
140f2ad897eSsaigon
14106a7d5d9Sfarshid    def build_url(self, params):
142a607411eSfarshid        _errors = []
14306a7d5d9Sfarshid        version = ''
14406a7d5d9Sfarshid        server = ''
145329f692cSsaigon        openssl = ''
14606a7d5d9Sfarshid        names = []
14798c070fcSsaigon        url = ''
148a89f6554Ssaigon        direct_build_url = None
14906a7d5d9Sfarshid
150a9d19da9SAndrei Baranouski        # replace "v" with version
151a9d19da9SAndrei Baranouski        # replace p with product
15206a7d5d9Sfarshid        tmp = {}
15306a7d5d9Sfarshid        for k in params:
15406a7d5d9Sfarshid            value = params[k]
15506a7d5d9Sfarshid            if k == "v":
15606a7d5d9Sfarshid                tmp["version"] = value
15706a7d5d9Sfarshid            elif k == "p":
15806a7d5d9Sfarshid                tmp["version"] = value
15906a7d5d9Sfarshid            else:
16006a7d5d9Sfarshid                tmp[k] = value
16106a7d5d9Sfarshid        params = tmp
162c17d791dSAndrei Baranouski
16306a7d5d9Sfarshid        ok = True
164c4be36b0SSangharsh Agarwal        if not "version" in params and len(params["version"]) < 5:
165a607411eSfarshid            _errors.append(errors["INVALID-PARAMS"])
16606a7d5d9Sfarshid            ok = False
16706a7d5d9Sfarshid        else:
16806a7d5d9Sfarshid            version = params["version"]
16906a7d5d9Sfarshid
17006a7d5d9Sfarshid        if ok:
17106a7d5d9Sfarshid            if not "product" in params:
172a607411eSfarshid                _errors.append(errors["INVALID-PARAMS"])
17306a7d5d9Sfarshid                ok = False
17406a7d5d9Sfarshid        if ok:
17506a7d5d9Sfarshid            if not "server" in params:
176a607411eSfarshid                _errors.append(errors["INVALID-PARAMS"])
17706a7d5d9Sfarshid                ok = False
17806a7d5d9Sfarshid            else:
17906a7d5d9Sfarshid                server = params["server"]
18006a7d5d9Sfarshid
18174ac7dc1SKeith Batten        if ok:
18274ac7dc1SKeith Batten            if "toy" in params:
18374ac7dc1SKeith Batten                toy = params["toy"]
18474ac7dc1SKeith Batten            else:
18574ac7dc1SKeith Batten                toy = ""
18674ac7dc1SKeith Batten
187329f692cSsaigon        if ok:
188329f692cSsaigon            if "openssl" in params:
189329f692cSsaigon                openssl = params["openssl"]
190c17d791dSAndrei Baranouski
191a89f6554Ssaigon        if ok:
192fa99bf5cSsaigon            if "url" in params and params["url"] != ""\
193fa99bf5cSsaigon               and isinstance(params["url"], str):
194a89f6554Ssaigon                direct_build_url = params["url"]
1959209801eSsaigon        if ok:
1969209801eSsaigon            if "linux_repo" in params and params["linux_repo"].lower() == "true":
1979209801eSsaigon                linux_repo = True
1989209801eSsaigon            else:
1999209801eSsaigon                linux_repo = False
2004af7601eSsaigon        if ok:
2014af7601eSsaigon            if "msi" in params and params["msi"].lower() == "true":
2024af7601eSsaigon                msi = True
2034af7601eSsaigon            else:
2044af7601eSsaigon                msi = False
20506a7d5d9Sfarshid        if ok:
20606a7d5d9Sfarshid            mb_alias = ["membase", "membase-server", "mbs", "mb"]
20706a7d5d9Sfarshid            cb_alias = ["couchbase", "couchbase-server", "cb"]
208e081454fSKeith Batten            css_alias = ["couchbase-single", "couchbase-single-server", "css"]
2092724a9f1SIrynaMironava            moxi_alias = ["moxi", "moxi-server"]
210fce05e6bSRitesh Agarwal            cbas_alias = ["cbas", "server-analytics"]
211be415535Sprasanna
212fce05e6bSRitesh Agarwal            if params["product"] in cbas_alias:
2133412ca98SRitesh Agarwal                names = ['couchbase-server-analytics', 'server-analytics']
214fce05e6bSRitesh Agarwal            elif params["product"] in mb_alias:
215fce05e6bSRitesh Agarwal                names = ['membase-server-enterprise', 'membase-server-community']
21606a7d5d9Sfarshid            elif params["product"] in cb_alias:
217fce05e6bSRitesh Agarwal                if "type" in params and params["type"].lower() in "couchbase-server-community":
218c17d791dSAndrei Baranouski                    names = ['couchbase-server-community']
219fce05e6bSRitesh Agarwal                elif "type" in params and params["type"].lower() in "couchbase-server-enterprise":
220c17d791dSAndrei Baranouski                    names = ['couchbase-server-enterprise']
221c17d791dSAndrei Baranouski                else:
222fce05e6bSRitesh Agarwal                    names = ['couchbase-server-enterprise', 'couchbase-server-community']
223e081454fSKeith Batten            elif params["product"] in css_alias:
224fce05e6bSRitesh Agarwal                names = ['couchbase-single-server-enterprise', 'couchbase-single-server-community']
2252724a9f1SIrynaMironava            elif params["product"] in moxi_alias:
2262724a9f1SIrynaMironava                names = ['moxi-server']
22706a7d5d9Sfarshid            else:
22806a7d5d9Sfarshid                ok = False
229a607411eSfarshid                _errors.append(errors["INVALID-PARAMS"])
230329f692cSsaigon            if "1" in openssl:
231fce05e6bSRitesh Agarwal                names = ['couchbase-server-enterprise_centos6', 'couchbase-server-community_centos6', \
232fce05e6bSRitesh Agarwal                         'couchbase-server-enterprise_ubuntu_1204', 'couchbase-server-community_ubuntu_1204']
233c1fd6b19SKetaki            if "toy" in params:
2346a023be1Ssaigon                names = ['couchbase-server-enterprise']
235329f692cSsaigon
236a9d19da9SAndrei Baranouski        remote_client = RemoteMachineShellConnection(server)
237a9d19da9SAndrei Baranouski        info = remote_client.extract_remote_info()
2383aaa1a27Ssaigon        print "\n*** OS version of this server %s is %s ***" % (remote_client.ip,
2393aaa1a27Ssaigon                                                       info.distribution_version)
2403aaa1a27Ssaigon        if info.distribution_version.lower() == "suse 12":
2413aaa1a27Ssaigon            if version[:5] not in COUCHBASE_FROM_SPOCK:
2423aaa1a27Ssaigon                mesg = "%s does not support cb version %s \n" % \
2433aaa1a27Ssaigon                         (info.distribution_version, version[:5])
2443aaa1a27Ssaigon                remote_client.stop_current_python_running(mesg)
245c8a684a4Ssaigon        if info.type.lower() == "windows":
246c8a684a4Ssaigon            if "-" in version:
247c8a684a4Ssaigon                msi_build = version.split("-")
248b981a4f4Ssaigon                """
249b981a4f4Ssaigon                    In spock from build 2924 and later release, we only support
250b981a4f4Ssaigon                    msi installation method on windows
251b981a4f4Ssaigon                """
252ff2070e9Ssaigon                if "2k8" in info.windows_name:
253ff2070e9Ssaigon                    info.windows_name = 2008
254ff2070e9Ssaigon
255b981a4f4Ssaigon                if msi_build[0] in COUCHBASE_FROM_SPOCK:
256c8a684a4Ssaigon                    info.deliverable_type = "msi"
257ff2070e9Ssaigon                elif "5" > msi_build[0] and info.windows_name == 2016:
258887bf294Ssaigon                    log.info("\n========\n"
259887bf294Ssaigon                        "         Build version %s does not support on\n"
260887bf294Ssaigon                        "         Windows Server 2016\n"
261887bf294Ssaigon                        "========"  % msi_build[0])
262887bf294Ssaigon                    os.system("ps aux | grep python | grep %d " % os.getpid())
263887bf294Ssaigon                    time.sleep(5)
264887bf294Ssaigon                    os.system('kill %d' % os.getpid())
265c8a684a4Ssaigon            else:
266c8a684a4Ssaigon                print "Incorrect version format"
267c8a684a4Ssaigon                sys.exit()
268a9d19da9SAndrei Baranouski        remote_client.disconnect()
2699209801eSsaigon        if ok and not linux_repo:
270c4be36b0SSangharsh Agarwal            timeout = 300
2710e70fabcSIryna            if "timeout" in params:
2720e70fabcSIryna                timeout = int(params["timeout"])
273cadf4efeSIrynaMironava            releases_version = ["1.6.5.4", "1.7.0", "1.7.1", "1.7.1.1", "1.8.0"]
274e9551aa0Ssaigon            cb_releases_version = ["1.8.1", "2.0.0", "2.0.1", "2.1.0", "2.1.1", "2.2.0",
2756864e996Ssaigon                                    "2.5.0", "2.5.1", "2.5.2", "3.0.0", "3.0.1", "3.0.2",
276e5a88a7fSEric                                    "3.0.3", "3.1.0", "3.1.1", "3.1.2", "3.1.3", "3.1.5","3.1.6",
2771aeea310Ssaigon                                    "4.0.0", "4.0.1", "4.1.0", "4.1.1", "4.1.2", "4.5.0"]
278a49a5579Ssaigon            build_repo = MV_LATESTBUILD_REPO
2796a023be1Ssaigon            if toy is not "":
280a13b0750Ssaigon                build_repo = CB_REPO
281fce05e6bSRitesh Agarwal            elif "server-analytics" in names:
282fce05e6bSRitesh Agarwal                build_repo = CB_REPO.replace("couchbase-server", "server-analytics") + CB_VERSION_NAME[version[:3]] + "/"
283deee09eaSsaigon            elif "moxi-server" in names and version[:5] != "2.5.2":
284deee09eaSsaigon                print "version   ", version
285deee09eaSsaigon                """
286deee09eaSsaigon                moxi repo:
287deee09eaSsaigon                   http://172.23.120.24/builds/latestbuilds/moxi/4.6.0/101/moxi-server..
288deee09eaSsaigon                """
289deee09eaSsaigon                build_repo = CB_REPO.replace("couchbase-server", "moxi") + version[:5] + "/"
290a13b0750Ssaigon            elif version[:5] not in COUCHBASE_VERSION_2 and \
291a13b0750Ssaigon                 version[:5] not in COUCHBASE_VERSION_3:
292a13b0750Ssaigon                if version[:3] in CB_VERSION_NAME:
293a13b0750Ssaigon                    build_repo = CB_REPO + CB_VERSION_NAME[version[:3]] + "/"
294a13b0750Ssaigon                else:
295a13b0750Ssaigon                    sys.exit("version is not support yet")
2963c8a3212SAruna Piravi            if 'enable_ipv6' in params and params['enable_ipv6']:
2973c8a3212SAruna Piravi                build_repo = build_repo.replace(CB_DOWNLOAD_SERVER,
2983c8a3212SAruna Piravi                                                CB_DOWNLOAD_SERVER_FQDN)
29906a7d5d9Sfarshid            for name in names:
300f586f28fSsaigon                if version[:5] in releases_version:
30162486499Ssaigon                    build = BuildQuery().find_membase_release_build(
30262486499Ssaigon                                             deliverable_type=info.deliverable_type,
30362486499Ssaigon                                             os_architecture=info.architecture_type,
30462486499Ssaigon                                             build_version=version,
30562486499Ssaigon                                             product='membase-server-enterprise')
3061e56f36bSsaigon                elif len(version) > 6 and version[6:].replace("-rel", "") == \
3071e56f36bSsaigon                                                    CB_RELEASE_BUILDS[version[:5]]:
308972639abSsaigon                    build = BuildQuery().find_couchbase_release_build(
3097086f250Ssaigon                                            deliverable_type=info.deliverable_type,
3107086f250Ssaigon                                            os_architecture=info.architecture_type,
3117086f250Ssaigon                                            build_version=version,
3127086f250Ssaigon                                            product=name,
3136d5639efSsaigon                                            os_version = info.distribution_version,
3146d5639efSsaigon                                            direct_build_url=direct_build_url)
315c13b8f2aSsaigon                else:
31662486499Ssaigon                    builds, changes = BuildQuery().get_all_builds(version=version,
31762486499Ssaigon                                      timeout=timeout,
31862486499Ssaigon                                      direct_build_url=direct_build_url,
31962486499Ssaigon                                      deliverable_type=info.deliverable_type,
32062486499Ssaigon                                      architecture_type=info.architecture_type,
32162486499Ssaigon                                      edition_type=name,
32262486499Ssaigon                                      repo=build_repo, toy=toy,
32362486499Ssaigon                                      distribution_version=info.distribution_version.lower(),
324766fb9e7Ssaigon                                      distribution_type=info.distribution_type.lower())
32562486499Ssaigon                    build = BuildQuery().find_build(builds, name, info.deliverable_type,
32662486499Ssaigon                                               info.architecture_type, version, toy=toy,
32762486499Ssaigon                                     openssl=openssl, direct_build_url=direct_build_url,
32862486499Ssaigon                                 distribution_version=info.distribution_version.lower(),
32962486499Ssaigon                                       distribution_type=info.distribution_type.lower())
330898ef744SSteve Yen
3317fa87c49Sfarshid                if build:
33206a7d5d9Sfarshid                    if 'amazon' in params:
33376f684b9Ssaigon                        type = info.type.lower()
33476f684b9Ssaigon                        if type == 'windows' and version in releases_version:
33576f684b9Ssaigon                            build.url = build.url.replace("http://builds.hq.northscale.net",
33676f684b9Ssaigon                                                          "https://s3.amazonaws.com/packages.couchbase")
33776f684b9Ssaigon                            build.url = build.url.replace("enterprise", "community")
33876f684b9Ssaigon                            build.name = build.name.replace("enterprise", "community")
33976f684b9Ssaigon                        else:
34000243638Ssaigon                            """ since url in S3 insert version into it, we need to put version
34100243638Ssaigon                                in like ..latestbuilds/3.0.0/... """
34200243638Ssaigon                            cb_version = version[:5]
34300243638Ssaigon                            build.url = build.url.replace("http://builds.hq.northscale.net/latestbuilds",
34400243638Ssaigon                                        "http://packages.northscale.com/latestbuilds/{0}".format(cb_version))
34500243638Ssaigon                            """ test enterprise version """
34600243638Ssaigon                            #build.url = build.url.replace("enterprise", "community")
34700243638Ssaigon                            #build.name = build.name.replace("enterprise", "community")
348b1c9c13dSsaigon                    """ check if URL is live """
349b1c9c13dSsaigon                    remote_client = RemoteMachineShellConnection(server)
350b1c9c13dSsaigon                    if remote_client.is_url_live(build.url):
351b1c9c13dSsaigon                        return build
352b1c9c13dSsaigon                    else:
353b1c9c13dSsaigon                        sys.exit("ERROR: URL is not good. Check URL again")
354a607411eSfarshid            _errors.append(errors["BUILD-NOT-FOUND"])
3559209801eSsaigon        if not linux_repo:
3569209801eSsaigon            msg = "unable to find a build for product {0} version {1} for package_type {2}"
3579209801eSsaigon            raise Exception(msg.format(names, version, info.deliverable_type))
35806a7d5d9Sfarshid
359f6f4c9ebSPavel.Paulau    def is_socket_active(self, host, port, timeout=300):
360f6f4c9ebSPavel.Paulau        """ Check if remote socket is open and active
361f6f4c9ebSPavel.Paulau
362f6f4c9ebSPavel.Paulau        Keyword arguments:
363f6f4c9ebSPavel.Paulau        host -- remote address
364f6f4c9ebSPavel.Paulau        port -- remote port
365f6f4c9ebSPavel.Paulau        timeout -- check timeout (in seconds)
366f6f4c9ebSPavel.Paulau
367f6f4c9ebSPavel.Paulau        Returns:
368f6f4c9ebSPavel.Paulau        True -- socket is active
369f6f4c9ebSPavel.Paulau        False -- otherwise
370f6f4c9ebSPavel.Paulau
371f6f4c9ebSPavel.Paulau        """
372f6f4c9ebSPavel.Paulau        start_time = time.time()
373f6f4c9ebSPavel.Paulau
374f6f4c9ebSPavel.Paulau        sckt = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
375f6f4c9ebSPavel.Paulau
376f6f4c9ebSPavel.Paulau        while time.time() - start_time < timeout:
377f6f4c9ebSPavel.Paulau            try:
378f6f4c9ebSPavel.Paulau                sckt.connect((host, port))
379f6f4c9ebSPavel.Paulau                sckt.shutdown(2)
380f6f4c9ebSPavel.Paulau                sckt.close()
381f6f4c9ebSPavel.Paulau                return True
382f6f4c9ebSPavel.Paulau            except:
383f6f4c9ebSPavel.Paulau                time.sleep(10)
384f6f4c9ebSPavel.Paulau
385f6f4c9ebSPavel.Paulau        return False
386e3087ca6SKaran Kumar
38706a7d5d9Sfarshidclass MembaseServerInstaller(Installer):
38806a7d5d9Sfarshid    def __init__(self):
38906a7d5d9Sfarshid        Installer.__init__(self)
39006a7d5d9Sfarshid
39106a7d5d9Sfarshid    def initialize(self, params):
39206a7d5d9Sfarshid        start_time = time.time()
39306a7d5d9Sfarshid        cluster_initialized = False
39406a7d5d9Sfarshid        server = params["server"]
39515f3a652SAndrei Baranouski        while time.time() < (start_time + (5 * 60)):
39606a7d5d9Sfarshid            rest = RestConnection(server)
39706a7d5d9Sfarshid            try:
39832ec96c2SKaran Kumar                if server.data_path:
39932ec96c2SKaran Kumar                    remote_client = RemoteMachineShellConnection(server)
4004a524cd4Skkumar                    remote_client.execute_command('rm -rf {0}/*'.format(server.data_path))
4014a524cd4Skkumar                    # Make sure that data_path is writable by membase user
40232ec96c2SKaran Kumar                    remote_client.execute_command("chown -R membase.membase {0}".format(server.data_path))
4039390b028Sandreibara                    remote_client.disconnect()
40432ec96c2SKaran Kumar                    rest.set_data_path(data_path=server.data_path)
40506a7d5d9Sfarshid                rest.init_cluster(username=server.rest_username, password=server.rest_password)
406a607411eSfarshid                rest.init_cluster_memoryQuota(memoryQuota=rest.get_nodes_self().mcdMemoryReserved)
40706a7d5d9Sfarshid                cluster_initialized = True
40806a7d5d9Sfarshid                break
40906a7d5d9Sfarshid            except ServerUnavailableException:
41006a7d5d9Sfarshid                log.error("error happened while initializing the cluster @ {0}".format(server.ip))
41106a7d5d9Sfarshid            log.info('sleep for 5 seconds before trying again ...')
41206a7d5d9Sfarshid            time.sleep(5)
41306a7d5d9Sfarshid        if not cluster_initialized:
41406a7d5d9Sfarshid            raise Exception("unable to initialize membase node")
41506a7d5d9Sfarshid
416b9c46d85Sandreibara    def install(self, params, queue=None):
417b9c46d85Sandreibara        try:
418b9c46d85Sandreibara            build = self.build_url(params)
419b9c46d85Sandreibara        except Exception, e:
420b9c46d85Sandreibara            if queue:
421b9c46d85Sandreibara                queue.put(False)
422b9c46d85Sandreibara            raise e
42306a7d5d9Sfarshid        remote_client = RemoteMachineShellConnection(params["server"])
42468d9bc3dSsaigon        info = remote_client.extract_remote_info()
42568d9bc3dSsaigon        type = info.type.lower()
426edc8139aSkkumar        server = params["server"]
427aa58812aSChisheng Hong        if "vbuckets" in params:
428aa58812aSChisheng Hong            vbuckets = int(params["vbuckets"][0])
429aa58812aSChisheng Hong        else:
430aa58812aSChisheng Hong            vbuckets = None
43152632f18Ssaigon        if "swappiness" in params:
43252632f18Ssaigon            swappiness = int(params["swappiness"])
433abe63c9eSsaigon        else:
4349a779cd8Sarunapiravi            swappiness = 0
43552632f18Ssaigon
436329f692cSsaigon        if "openssl" in params:
437329f692cSsaigon            openssl = params["openssl"]
438016ee3a2Ssaigon        else:
439016ee3a2Ssaigon            openssl = ""
440329f692cSsaigon
44168d9bc3dSsaigon        if type == "windows":
44268d9bc3dSsaigon            build = self.build_url(params)
443641112f5Ssaigon            remote_client.download_binary_in_win(build.url, params["version"])
444a4657190Ssaigon            success = remote_client.install_server_win(build, params["version"], \
445a4657190Ssaigon                                                       vbuckets=vbuckets)
44668d9bc3dSsaigon        else:
44768d9bc3dSsaigon            downloaded = remote_client.download_build(build)
44868d9bc3dSsaigon            if not downloaded:
449e97b0123Ssaigon                log.error('server {1} unable to download binaries : {0}' \
450e97b0123Ssaigon                          .format(build.url, params["server"].ip))
4519a0422afSandreibara                return False
452edc8139aSkkumar            path = server.data_path or '/tmp'
453329f692cSsaigon            success &= remote_client.install_server(build, path=path, vbuckets=vbuckets, \
45479250bc7SAndrei Baranouski                                                    swappiness=swappiness, openssl=openssl)
45568d9bc3dSsaigon            ready = RestHelper(RestConnection(params["server"])).is_ns_server_running(60)
45668d9bc3dSsaigon            if not ready:
45768d9bc3dSsaigon                log.error("membase-server did not start...")
45877266c30Ssaigon            log.info('wait 5 seconds for Membase server to start')
45968d9bc3dSsaigon            time.sleep(5)
4609390b028Sandreibara        remote_client.disconnect()
461b9c46d85Sandreibara        if queue:
462b9c46d85Sandreibara            queue.put(success)
463b9c46d85Sandreibara        return success
46406a7d5d9Sfarshid
465e3087ca6SKaran Kumar
46606a7d5d9Sfarshidclass CouchbaseServerInstaller(Installer):
46706a7d5d9Sfarshid    def __init__(self):
46806a7d5d9Sfarshid        Installer.__init__(self)
46906a7d5d9Sfarshid
47006a7d5d9Sfarshid    def initialize(self, params):
47139b7b731Ssaigon        #log.info('*****CouchbaseServerInstaller initialize the application ****')
47206a7d5d9Sfarshid        start_time = time.time()
47306a7d5d9Sfarshid        cluster_initialized = False
47406a7d5d9Sfarshid        server = params["server"]
47593eb8ff8Skkumar        remote_client = RemoteMachineShellConnection(params["server"])
476db6e1c26Ssaigon        success = True
477db6e1c26Ssaigon        success &= remote_client.is_couchbase_installed()
478db6e1c26Ssaigon        if not success:
479db6e1c26Ssaigon            mesg = "\n\nServer {0} failed to install".format(params["server"].ip)
480db6e1c26Ssaigon            sys.exit(mesg)
48115f3a652SAndrei Baranouski        while time.time() < start_time + 5 * 60:
48206a7d5d9Sfarshid            try:
483f3914971Sronniedada                rest = RestConnection(server)
484f3914971Sronniedada
485ce5a5b38SPavel.Paulau                # Optionally change node name and restart server
486ce5a5b38SPavel.Paulau                if params.get('use_domain_names', 0):
487b4777408SIrynaMironava                    RemoteUtilHelper.use_hostname_for_server_settings(server)
488506ad72aSPavel.Paulau
4897ef927e0SAruna Piravi                if params.get('enable_ipv6', 0):
490fe268802SAruna Piravi                    status, content = RestConnection(server).rename_node(
491017f6c20SAruna Piravi                        hostname=server.ip.replace('[', '').replace(']', ''))
492fe268802SAruna Piravi                    if status:
493fe268802SAruna Piravi                        log.info("Node {0} renamed to {1}".format(server.ip,
494fe268802SAruna Piravi                                                                  server.ip.replace('[', '').
495fe268802SAruna Piravi                                                                  replace(']', '')))
496fe268802SAruna Piravi                    else:
497fe268802SAruna Piravi                        log.error("Error renaming node {0} to {1}: {2}".
498fe268802SAruna Piravi                                  format(server.ip,
499fe268802SAruna Piravi                                         server.ip.replace('[', '').replace(']', ''),
500fe268802SAruna Piravi                                         content))
5017ef927e0SAruna Piravi
5026b81dc3fSandreibara                # Make sure that data_path and index_path are writable by couchbase user
5036b81dc3fSandreibara                for path in set(filter(None, [server.data_path, server.index_path])):
50493eb8ff8Skkumar                    time.sleep(3)
505506ad72aSPavel.Paulau
5066b81dc3fSandreibara                    for cmd in ("rm -rf {0}/*".format(path),
5076b81dc3fSandreibara                                "chown -R couchbase:couchbase {0}".format(path)):
508506ad72aSPavel.Paulau                        remote_client.execute_command(cmd)
5096b81dc3fSandreibara                rest.set_data_path(data_path=server.data_path,
51016e90a76SPavel.Paulau                                       index_path=server.index_path)
5116b81dc3fSandreibara                time.sleep(3)
512506ad72aSPavel.Paulau
513b8100915SPavel.Paulau                # Initialize cluster
514b5c63c23SParag Agarwal                if "init_nodes" in params:
515b5c63c23SParag Agarwal                    init_nodes = params["init_nodes"]
516b5c63c23SParag Agarwal                else:
517b5c63c23SParag Agarwal                    init_nodes = "True"
51880157716Sprasanna                if (isinstance(init_nodes, bool) and init_nodes) or \
51980157716Sprasanna                        (isinstance(init_nodes, str) and init_nodes.lower() == "true"):
5207cb3b4beSsaigon                    if not server.services:
5217cb3b4beSsaigon                        set_services = ["kv"]
5227cb3b4beSsaigon                    elif server.services:
5237cb3b4beSsaigon                        set_services = server.services.split(',')
5247cb3b4beSsaigon
5257cb3b4beSsaigon                    kv_quota = 0
5267cb3b4beSsaigon                    while kv_quota == 0:
5277cb3b4beSsaigon                        time.sleep(1)
5280a5618a4Sritamcouchbase                        kv_quota = int(rest.get_nodes_self().mcdMemoryReserved)
5290a5618a4Sritamcouchbase                    info = rest.get_nodes_self()
5300a5618a4Sritamcouchbase                    cb_version = info.version[:5]
531fce05e6bSRitesh Agarwal                    kv_quota = int(info.mcdMemoryReserved * 2/3)
532fce05e6bSRitesh Agarwal                    """ for fts, we need to grep quota from ns_server
533fce05e6bSRitesh Agarwal                                but need to make it works even RAM of vm is
534fce05e6bSRitesh Agarwal                                smaller than 2 GB """
535be415535Sprasanna
5360a5618a4Sritamcouchbase                    if cb_version in COUCHBASE_FROM_VERSION_4:
537fce05e6bSRitesh Agarwal                        if "index" in set_services:
538fce05e6bSRitesh Agarwal                            log.info("quota for index service will be %s MB" % (INDEX_QUOTA))
539fce05e6bSRitesh Agarwal                            kv_quota -= INDEX_QUOTA
540f2ad897eSsaigon                            log.info("set index quota to node %s " % server.ip)
541fbcb51c4SRitesh Agarwal                            rest.set_service_memoryQuota(service='indexMemoryQuota', memoryQuota=INDEX_QUOTA)
542fce05e6bSRitesh Agarwal                        if "fts" in set_services:
543fce05e6bSRitesh Agarwal                            log.info("quota for fts service will be %s MB" % (FTS_QUOTA))
544fce05e6bSRitesh Agarwal                            kv_quota -= FTS_QUOTA
545fce05e6bSRitesh Agarwal                            log.info("set both index and fts quota at node %s "% server.ip)
546fbcb51c4SRitesh Agarwal                            rest.set_service_memoryQuota(service='ftsMemoryQuota', memoryQuota=FTS_QUOTA)
547fce05e6bSRitesh Agarwal                        if "cbas" in set_services:
548fce05e6bSRitesh Agarwal                            log.info("quota for cbas service will be %s MB" % (CBAS_QUOTA))
549fce05e6bSRitesh Agarwal                            kv_quota -= CBAS_QUOTA
550fbcb51c4SRitesh Agarwal                            rest.set_service_memoryQuota(service = "cbasMemoryQuota", memoryQuota=CBAS_QUOTA)
551fce05e6bSRitesh Agarwal                        if kv_quota < MIN_KV_QUOTA:
5520a5618a4Sritamcouchbase                                raise Exception("KV RAM needs to be more than %s MB"
5530a5618a4Sritamcouchbase                                        " at node  %s"  % (MIN_KV_QUOTA, server.ip))
554e4069263Ssaigon                    """ set kv quota smaller than 1 MB so that it will satify
555e4069263Ssaigon                        the condition smaller than allow quota """
556e4069263Ssaigon                    kv_quota -= 1
5570a5618a4Sritamcouchbase                    log.info("quota for kv: %s MB" % kv_quota)
5587cb3b4beSsaigon                    rest.init_cluster_memoryQuota(server.rest_username, \
5597cb3b4beSsaigon                                                       server.rest_password, \
5607cb3b4beSsaigon                                                                     kv_quota)
5610a5618a4Sritamcouchbase                    if params["version"][:5] in COUCHBASE_FROM_VERSION_4:
5620a5618a4Sritamcouchbase                        rest.init_node_services(username=server.rest_username,
563af987c62SIrynaMironava                                                password=server.rest_password,
5647cb3b4beSsaigon                                                        services=set_services)
5656a6582bdSsaigon                    if "index" in set_services:
566be415535Sprasanna                        if "storage_mode" in params:
567be415535Sprasanna                            storageMode = params["storage_mode"]
568be415535Sprasanna                        else:
569be415535Sprasanna                            storageMode = "plasma"
570be415535Sprasanna                        rest.set_indexer_storage_mode(storageMode=storageMode)
5710a1108bbSIrynaMironava                    rest.init_cluster(username=server.rest_username,
5727cb3b4beSsaigon                                         password=server.rest_password)
573506ad72aSPavel.Paulau
574506ad72aSPavel.Paulau                # Optionally disable consistency check
575506ad72aSPavel.Paulau                if params.get('disable_consistency', 0):
576506ad72aSPavel.Paulau                    rest.set_couchdb_option(section='couchdb',
577506ad72aSPavel.Paulau                                            option='consistency_check_ratio',
578506ad72aSPavel.Paulau                                            value='0.0')
579506ad72aSPavel.Paulau
5800cbf53f4Sronniedada                # memcached env variable
5810cbf53f4Sronniedada                mem_req_tap_env = params.get('MEMCACHED_REQS_TAP_EVENT', 0)
5820cbf53f4Sronniedada                if mem_req_tap_env:
5830cbf53f4Sronniedada                    remote_client.set_environment_variable('MEMCACHED_REQS_TAP_EVENT',
5840cbf53f4Sronniedada                                                           mem_req_tap_env)
5850a5618a4Sritamcouchbase                """ set cbauth environment variables from Watson version
5860a5618a4Sritamcouchbase                    it is checked version inside method """
5870a5618a4Sritamcouchbase                remote_client.set_cbauth_env(server)
58852df04ceSsaigon                remote_client.check_man_page()
58900403f66Ssaigon                """ add unzip command on server if it is not available """
59000403f66Ssaigon                remote_client.check_cmd("unzip")
591898cd79aSsaigon                remote_client.is_ntp_installed()
5929390b028Sandreibara                remote_client.disconnect()
593a9d19da9SAndrei Baranouski                # TODO: Make it work with windows
594b2cd510cSKaran Kumar                if "erlang_threads" in params:
59560c8290fSandreibara                    num_threads = params.get('erlang_threads', testconstants.NUM_ERLANG_THREADS)
596b2cd510cSKaran Kumar                    # Stop couchbase-server
597b2cd510cSKaran Kumar                    ClusterOperationHelper.stop_cluster([server])
59860c8290fSandreibara                    if "sync_threads" in params or ':' in num_threads:
59960c8290fSandreibara                        sync_threads = params.get('sync_threads', True)
60060c8290fSandreibara                    else:
60160c8290fSandreibara                        sync_threads = False
60260c8290fSandreibara                    # Change type of threads(sync/async) and num erlang threads
60360c8290fSandreibara                    ClusterOperationHelper.change_erlang_threads_values([server], sync_threads, num_threads)
604b2cd510cSKaran Kumar                    # Start couchbase-server
605b2cd510cSKaran Kumar                    ClusterOperationHelper.start_cluster([server])
6060abbae49SKaran Kumar                if "erlang_gc_level" in params:
6070abbae49SKaran Kumar                    erlang_gc_level = params.get('erlang_gc_level', None)
6080abbae49SKaran Kumar                    if erlang_gc_level is None:
6090abbae49SKaran Kumar                        # Don't change the value
6100abbae49SKaran Kumar                        break
6110abbae49SKaran Kumar                    # Stop couchbase-server
6120abbae49SKaran Kumar                    ClusterOperationHelper.stop_cluster([server])
6130abbae49SKaran Kumar                    # Change num erlang threads
6140abbae49SKaran Kumar                    ClusterOperationHelper.change_erlang_gc([server], erlang_gc_level)
6150abbae49SKaran Kumar                    # Start couchbase-server
6160abbae49SKaran Kumar                    ClusterOperationHelper.start_cluster([server])
61706a7d5d9Sfarshid                cluster_initialized = True
61806a7d5d9Sfarshid                break
61906a7d5d9Sfarshid            except ServerUnavailableException:
62006a7d5d9Sfarshid                log.error("error happened while initializing the cluster @ {0}".format(server.ip))
62106a7d5d9Sfarshid            log.info('sleep for 5 seconds before trying again ...')
62206a7d5d9Sfarshid            time.sleep(5)
62306a7d5d9Sfarshid        if not cluster_initialized:
624a6b8bb2dSandreibara            sys.exit("unable to initialize couchbase node")
62506a7d5d9Sfarshid
626b9c46d85Sandreibara    def install(self, params, queue=None):
627a6dbe12bSEric
628a6dbe12bSEric        log.info('********CouchbaseServerInstaller:install')
629a6dbe12bSEric
6304af7601eSsaigon        self.msi = 'msi' in params and params['msi'].lower() == 'true'
631a8cb468fSAruna Piravi        start_server = True
632b9c46d85Sandreibara        try:
633c0adb809Ssaigon            if "linux_repo" not in params:
634c0adb809Ssaigon                build = self.build_url(params)
635