1952fee0dSpavel-paulau#!/usr/bin/env python
206a7d5d9Sfarshid
399d10dcbSSteve Yen# TODO: add installer support for membasez
414610b75Sfarshid
599d10dcbSSteve Yenimport getopt
6e81187faSsaigonimport copy, re
7fe259f4cSfarshidimport logging
8e0483781Ssaigonimport os
906a7d5d9Sfarshidimport sys
1006a7d5d9Sfarshidfrom threading import Thread
115ee062bbSKeith Battenfrom datetime import datetime
12f6f4c9ebSPavel.Paulauimport socket
13b9c46d85Sandreibaraimport Queue
1406a7d5d9Sfarshid
1525433975SKaran Kumarsys.path = [".", "lib"] + sys.path
16a607411eSfarshidimport testconstants
17a607411eSfarshidimport time
1806a7d5d9Sfarshidfrom builds.build_query import BuildQuery
19f607ec25Sfarshidimport logging.config
2006a7d5d9Sfarshidfrom membase.api.exception import ServerUnavailableException
2106a7d5d9Sfarshidfrom membase.api.rest_client import RestConnection, RestHelper
22b4777408SIrynaMironavafrom remote.remote_util import RemoteMachineShellConnection, RemoteUtilHelper
231f79a4e0SKeith Battenfrom membase.helper.cluster_helper import ClusterOperationHelper
24c1bf1505Ssaigonfrom testconstants import MV_LATESTBUILD_REPO
25a49a5579Ssaigonfrom testconstants import SHERLOCK_BUILD_REPO
266a023be1Ssaigonfrom testconstants import COUCHBASE_REPO
273c8a3212SAruna Piravifrom testconstants import CB_REPO, CB_DOWNLOAD_SERVER, CB_DOWNLOAD_SERVER_FQDN
28a13b0750Ssaigonfrom testconstants import COUCHBASE_VERSION_2
29c8a684a4Ssaigonfrom testconstants import COUCHBASE_VERSION_3, COUCHBASE_FROM_WATSON,\
30c8a684a4Ssaigon                          COUCHBASE_FROM_SPOCK
3162486499Ssaigonfrom testconstants import CB_VERSION_NAME, COUCHBASE_FROM_VERSION_4,\
32e81187faSsaigon                          CB_RELEASE_BUILDS, COUCHBASE_VERSIONS
3333445f7aSsaigonfrom testconstants import MIN_KV_QUOTA, INDEX_QUOTA, FTS_QUOTA, CBAS_QUOTA, CLUSTER_QUOTA_RATIO
3430b14716Sprasannafrom testconstants import LINUX_COUCHBASE_PORT_CONFIG_PATH, LINUX_COUCHBASE_OLD_CONFIG_PATH
3530b14716Sprasannafrom testconstants import WIN_COUCHBASE_PORT_CONFIG_PATH, WIN_COUCHBASE_OLD_CONFIG_PATH
3606a7d5d9Sfarshidimport TestInput
3706a7d5d9Sfarshid
3899d10dcbSSteve Yen
39f607ec25Sfarshidlogging.config.fileConfig("scripts.logging.conf")
40fe259f4cSfarshidlog = logging.getLogger()
41fe259f4cSfarshid
4299d10dcbSSteve Yendef usage(err=None):
4399d10dcbSSteve Yen    print """\
4499d10dcbSSteve YenSyntax: install.py [options]
4599d10dcbSSteve Yen
4699d10dcbSSteve YenOptions:
4799d10dcbSSteve Yen -p <key=val,...> Comma-separated key=value info.
4899d10dcbSSteve Yen -i <file>        Path to .ini file containing cluster information.
4999d10dcbSSteve Yen
5099d10dcbSSteve YenAvailable keys:
5130b14716Sprasanna product=cb|mb              Used to specify couchbase or membase.
5230b14716Sprasanna version=SHORT_VERSION      Example: "2.0.0r-71".
5330b14716Sprasanna parallel=false             Useful when you're installing a cluster.
5430b14716Sprasanna toy=                       Install a toy build
5530b14716Sprasanna init_nodes=False           Initialize nodes
5630b14716Sprasanna vbuckets=                  The number of vbuckets in the server installation.
5730b14716Sprasanna sync_threads=True          Sync or acync threads(+S or +A)
5830b14716Sprasanna erlang_threads=            Number of erlang threads (default=16:16 for +S type)
5930b14716Sprasanna upr=True                   Enable UPR replication
6030b14716Sprasanna xdcr_upr=                  Enable UPR for XDCR (temporary param until XDCR with UPR is stable), values: None | True | False
6130b14716Sprasanna fts_query_limit=1000000    Set a limit for the max results to be returned by fts for any query
6207df1264SAruna Piravi cbft_env_options           Additional fts environment variables
6330b14716Sprasanna change_indexer_ports=false Sets indexer ports values to non-default ports
64be415535Sprasanna storage_mode=plasma        Sets indexer storage mode
657ef927e0SAruna Piravi enable_ipv6=False          Enable ipv6 mode in ns_server
6660c8290fSandreibara
6799d10dcbSSteve Yen
6899d10dcbSSteve YenExamples:
69329f692cSsaigon install.py -i /tmp/ubuntu.ini -p product=cb,version=2.2.0-792
70a89f6554Ssaigon install.py -i /tmp/ubuntu.ini -p product=cb,version=2.2.0-792,url=http://builds.hq.northscale.net/latestbuilds....
7174ac7dc1SKeith Batten install.py -i /tmp/ubuntu.ini -p product=mb,version=1.7.1r-38,parallel=true,toy=keith
72898ef744SSteve Yen install.py -i /tmp/ubuntu.ini -p product=mongo,version=2.0.2
736a023be1Ssaigon install.py -i /tmp/ubuntu.ini -p product=cb,version=0.0.0-704,toy=couchstore,parallel=true,vbuckets=1024
74329f692cSsaigon
75329f692cSsaigon # to run with build with require openssl version 1.0.0
76329f692cSsaigon install.py -i /tmp/ubuntu.ini -p product=cb,version=2.2.0-792,openssl=1
77f6ad5ef6Ssaigon
78f6ad5ef6Ssaigon # to install latest release of couchbase server via repo (apt-get and yum)
79f6ad5ef6Ssaigon  install.py -i /tmp/ubuntu.ini -p product=cb,linux_repo=true
80f6ad5ef6Ssaigon
81e0d52abdSsaigon # to install non-root non default path, add nr_install_dir params
82e0d52abdSsaigon   install.py -i /tmp/ubuntu.ini -p product=cb,version=5.0.0-1900,nr_install_dir=testnow1
83e0d52abdSsaigon
8499d10dcbSSteve Yen"""
8599d10dcbSSteve Yen    sys.exit(err)
8699d10dcbSSteve Yen
8799d10dcbSSteve Yen
8806a7d5d9Sfarshidproduct = "membase-server(ms),couchbase-single-server(css),couchbase-server(cs),zynga(z)"
8906a7d5d9Sfarshid
9006a7d5d9Sfarshiderrors = {"UNREACHABLE": "",
9106a7d5d9Sfarshid          "UNINSTALL-FAILED": "unable to uninstall the product",
9206a7d5d9Sfarshid          "INSTALL-FAILED": "unable to install",
9306a7d5d9Sfarshid          "BUILD-NOT-FOUND": "unable to find build",
9406a7d5d9Sfarshid          "INVALID-PARAMS": "invalid params given"}
9506a7d5d9Sfarshid
9614610b75Sfarshid
9714610b75Sfarshiddef installer_factory(params):
98898ef744SSteve Yen    if params.get("product", None) is None:
99898ef744SSteve Yen        sys.exit("ERROR: don't know what product you want installed")
100898ef744SSteve Yen
10114610b75Sfarshid    mb_alias = ["membase", "membase-server", "mbs", "mb"]
102fce05e6bSRitesh Agarwal    cb_alias = ["couchbase", "couchbase-server", "cb", "cbas"]
103fd0dbaeaSSubhashni Balakrishnan    sdk_alias = ["python-sdk", "pysdk"]
104f3c23bacSSubhashni Balakrishnan    es_alias = ["elasticsearch"]
105e081454fSKeith Batten    css_alias = ["couchbase-single", "couchbase-single-server", "css"]
106898ef744SSteve Yen    mongo_alias = ["mongo"]
1072724a9f1SIrynaMironava    moxi_alias = ["moxi", "moxi-server"]
108898ef744SSteve Yen
10914610b75Sfarshid    if params["product"] in mb_alias:
11084c3ae37Sandreibara        return MembaseServerInstaller()
11114610b75Sfarshid    elif params["product"] in cb_alias:
11284c3ae37Sandreibara        return CouchbaseServerInstaller()
113898ef744SSteve Yen    elif params["product"] in mongo_alias:
114898ef744SSteve Yen        return MongoInstaller()
1152724a9f1SIrynaMironava    elif params["product"] in moxi_alias:
1162724a9f1SIrynaMironava        return MoxiInstaller()
117fd0dbaeaSSubhashni Balakrishnan    elif params["product"] in sdk_alias:
118fd0dbaeaSSubhashni Balakrishnan        return SDKInstaller()
119f3c23bacSSubhashni Balakrishnan    elif params["product"] in es_alias:
120f3c23bacSSubhashni Balakrishnan        return ESInstaller()
121898ef744SSteve Yen
122898ef744SSteve Yen    sys.exit("ERROR: don't know about product " + params["product"])
12306a7d5d9Sfarshid
12406a7d5d9Sfarshid
12514610b75Sfarshidclass Installer(object):
12614610b75Sfarshid
12706a7d5d9Sfarshid    def install(self, params):
12806a7d5d9Sfarshid        pass
12906a7d5d9Sfarshid
13006a7d5d9Sfarshid    def initialize(self, params):
13106a7d5d9Sfarshid        pass
13206a7d5d9Sfarshid
13306a7d5d9Sfarshid    def uninstall(self, params):
13414610b75Sfarshid        remote_client = RemoteMachineShellConnection(params["server"])
135e4e8ec92Ssaigon        #remote_client.membase_uninstall()
136a6dbe12bSEric
1374af7601eSsaigon        self.msi = 'msi' in params and params['msi'].lower() == 'true'
138e63eb5b6SRitesh Agarwal        remote_client.couchbase_uninstall(windows_msi=self.msi,product=params['product'])
1399390b028Sandreibara        remote_client.disconnect()
14006a7d5d9Sfarshid
141f2ad897eSsaigon
14206a7d5d9Sfarshid    def build_url(self, params):
143a607411eSfarshid        _errors = []
14406a7d5d9Sfarshid        version = ''
14506a7d5d9Sfarshid        server = ''
146329f692cSsaigon        openssl = ''
14706a7d5d9Sfarshid        names = []
14898c070fcSsaigon        url = ''
149a89f6554Ssaigon        direct_build_url = None
15006a7d5d9Sfarshid
151a9d19da9SAndrei Baranouski        # replace "v" with version
152a9d19da9SAndrei Baranouski        # replace p with product
15306a7d5d9Sfarshid        tmp = {}
15406a7d5d9Sfarshid        for k in params:
15506a7d5d9Sfarshid            value = params[k]
15606a7d5d9Sfarshid            if k == "v":
15706a7d5d9Sfarshid                tmp["version"] = value
15806a7d5d9Sfarshid            elif k == "p":
15906a7d5d9Sfarshid                tmp["version"] = value
16006a7d5d9Sfarshid            else:
16106a7d5d9Sfarshid                tmp[k] = value
16206a7d5d9Sfarshid        params = tmp
163c17d791dSAndrei Baranouski
16406a7d5d9Sfarshid        ok = True
165c4be36b0SSangharsh Agarwal        if not "version" in params and len(params["version"]) < 5:
166a607411eSfarshid            _errors.append(errors["INVALID-PARAMS"])
16706a7d5d9Sfarshid            ok = False
16806a7d5d9Sfarshid        else:
16906a7d5d9Sfarshid            version = params["version"]
17006a7d5d9Sfarshid
17106a7d5d9Sfarshid        if ok:
17206a7d5d9Sfarshid            if not "product" in params:
173a607411eSfarshid                _errors.append(errors["INVALID-PARAMS"])
17406a7d5d9Sfarshid                ok = False
17506a7d5d9Sfarshid        if ok:
17606a7d5d9Sfarshid            if not "server" in params:
177a607411eSfarshid                _errors.append(errors["INVALID-PARAMS"])
17806a7d5d9Sfarshid                ok = False
17906a7d5d9Sfarshid            else:
18006a7d5d9Sfarshid                server = params["server"]
18106a7d5d9Sfarshid
18274ac7dc1SKeith Batten        if ok:
18374ac7dc1SKeith Batten            if "toy" in params:
18474ac7dc1SKeith Batten                toy = params["toy"]
18574ac7dc1SKeith Batten            else:
18674ac7dc1SKeith Batten                toy = ""
18774ac7dc1SKeith Batten
188329f692cSsaigon        if ok:
189329f692cSsaigon            if "openssl" in params:
190329f692cSsaigon                openssl = params["openssl"]
191c17d791dSAndrei Baranouski
192a89f6554Ssaigon        if ok:
193fa99bf5cSsaigon            if "url" in params and params["url"] != ""\
194fa99bf5cSsaigon               and isinstance(params["url"], str):
195a89f6554Ssaigon                direct_build_url = params["url"]
1969209801eSsaigon        if ok:
1979209801eSsaigon            if "linux_repo" in params and params["linux_repo"].lower() == "true":
1989209801eSsaigon                linux_repo = True
1999209801eSsaigon            else:
2009209801eSsaigon                linux_repo = False
2014af7601eSsaigon        if ok:
2024af7601eSsaigon            if "msi" in params and params["msi"].lower() == "true":
2034af7601eSsaigon                msi = True
2044af7601eSsaigon            else:
2054af7601eSsaigon                msi = False
20606a7d5d9Sfarshid        if ok:
20706a7d5d9Sfarshid            mb_alias = ["membase", "membase-server", "mbs", "mb"]
20806a7d5d9Sfarshid            cb_alias = ["couchbase", "couchbase-server", "cb"]
209e081454fSKeith Batten            css_alias = ["couchbase-single", "couchbase-single-server", "css"]
2102724a9f1SIrynaMironava            moxi_alias = ["moxi", "moxi-server"]
211fce05e6bSRitesh Agarwal            cbas_alias = ["cbas", "server-analytics"]
212be415535Sprasanna
213fce05e6bSRitesh Agarwal            if params["product"] in cbas_alias:
2143412ca98SRitesh Agarwal                names = ['couchbase-server-analytics', 'server-analytics']
215fce05e6bSRitesh Agarwal            elif params["product"] in mb_alias:
216fce05e6bSRitesh Agarwal                names = ['membase-server-enterprise', 'membase-server-community']
21706a7d5d9Sfarshid            elif params["product"] in cb_alias:
218fce05e6bSRitesh Agarwal                if "type" in params and params["type"].lower() in "couchbase-server-community":
219c17d791dSAndrei Baranouski                    names = ['couchbase-server-community']
220fce05e6bSRitesh Agarwal                elif "type" in params and params["type"].lower() in "couchbase-server-enterprise":
221c17d791dSAndrei Baranouski                    names = ['couchbase-server-enterprise']
222c17d791dSAndrei Baranouski                else:
223fce05e6bSRitesh Agarwal                    names = ['couchbase-server-enterprise', 'couchbase-server-community']
224e081454fSKeith Batten            elif params["product"] in css_alias:
225fce05e6bSRitesh Agarwal                names = ['couchbase-single-server-enterprise', 'couchbase-single-server-community']
2262724a9f1SIrynaMironava            elif params["product"] in moxi_alias:
2272724a9f1SIrynaMironava                names = ['moxi-server']
22806a7d5d9Sfarshid            else:
22906a7d5d9Sfarshid                ok = False
230a607411eSfarshid                _errors.append(errors["INVALID-PARAMS"])
231329f692cSsaigon            if "1" in openssl:
232fce05e6bSRitesh Agarwal                names = ['couchbase-server-enterprise_centos6', 'couchbase-server-community_centos6', \
233fce05e6bSRitesh Agarwal                         'couchbase-server-enterprise_ubuntu_1204', 'couchbase-server-community_ubuntu_1204']
234c1fd6b19SKetaki            if "toy" in params:
2356a023be1Ssaigon                names = ['couchbase-server-enterprise']
236329f692cSsaigon
237a9d19da9SAndrei Baranouski        remote_client = RemoteMachineShellConnection(server)
238a9d19da9SAndrei Baranouski        info = remote_client.extract_remote_info()
2393aaa1a27Ssaigon        print "\n*** OS version of this server %s is %s ***" % (remote_client.ip,
2403aaa1a27Ssaigon                                                       info.distribution_version)
2413aaa1a27Ssaigon        if info.distribution_version.lower() == "suse 12":
2423aaa1a27Ssaigon            if version[:5] not in COUCHBASE_FROM_SPOCK:
2433aaa1a27Ssaigon                mesg = "%s does not support cb version %s \n" % \
2443aaa1a27Ssaigon                         (info.distribution_version, version[:5])
2453aaa1a27Ssaigon                remote_client.stop_current_python_running(mesg)
246c8a684a4Ssaigon        if info.type.lower() == "windows":
247c8a684a4Ssaigon            if "-" in version:
248c8a684a4Ssaigon                msi_build = version.split("-")
249b981a4f4Ssaigon                """
250b981a4f4Ssaigon                    In spock from build 2924 and later release, we only support
251b981a4f4Ssaigon                    msi installation method on windows
252b981a4f4Ssaigon                """
253ff2070e9Ssaigon                if "2k8" in info.windows_name:
254ff2070e9Ssaigon                    info.windows_name = 2008
255b981a4f4Ssaigon                if msi_build[0] in COUCHBASE_FROM_SPOCK:
256c8a684a4Ssaigon                    info.deliverable_type = "msi"
257ff2070e9Ssaigon                elif "5" > msi_build[0] and info.windows_name == 2016:
258887bf294Ssaigon                    log.info("\n========\n"
259887bf294Ssaigon                        "         Build version %s does not support on\n"
260887bf294Ssaigon                        "         Windows Server 2016\n"
261887bf294Ssaigon                        "========"  % msi_build[0])
262887bf294Ssaigon                    os.system("ps aux | grep python | grep %d " % os.getpid())
263887bf294Ssaigon                    time.sleep(5)
264887bf294Ssaigon                    os.system('kill %d' % os.getpid())
265c8a684a4Ssaigon            else:
266c8a684a4Ssaigon                print "Incorrect version format"
267c8a684a4Ssaigon                sys.exit()
268a9d19da9SAndrei Baranouski        remote_client.disconnect()
2699209801eSsaigon        if ok and not linux_repo:
270c4be36b0SSangharsh Agarwal            timeout = 300
2710e70fabcSIryna            if "timeout" in params:
2720e70fabcSIryna                timeout = int(params["timeout"])
273cadf4efeSIrynaMironava            releases_version = ["1.6.5.4", "1.7.0", "1.7.1", "1.7.1.1", "1.8.0"]
274e9551aa0Ssaigon            cb_releases_version = ["1.8.1", "2.0.0", "2.0.1", "2.1.0", "2.1.1", "2.2.0",
2756864e996Ssaigon                                    "2.5.0", "2.5.1", "2.5.2", "3.0.0", "3.0.1", "3.0.2",
276e5a88a7fSEric                                    "3.0.3", "3.1.0", "3.1.1", "3.1.2", "3.1.3", "3.1.5","3.1.6",
2771aeea310Ssaigon                                    "4.0.0", "4.0.1", "4.1.0", "4.1.1", "4.1.2", "4.5.0"]
278a49a5579Ssaigon            build_repo = MV_LATESTBUILD_REPO
2796a023be1Ssaigon            if toy is not "":
280a13b0750Ssaigon                build_repo = CB_REPO
281fce05e6bSRitesh Agarwal            elif "server-analytics" in names:
282fce05e6bSRitesh Agarwal                build_repo = CB_REPO.replace("couchbase-server", "server-analytics") + CB_VERSION_NAME[version[:3]] + "/"
283deee09eaSsaigon            elif "moxi-server" in names and version[:5] != "2.5.2":
284deee09eaSsaigon                """
285deee09eaSsaigon                moxi repo:
286deee09eaSsaigon                   http://172.23.120.24/builds/latestbuilds/moxi/4.6.0/101/moxi-server..
287deee09eaSsaigon                """
288deee09eaSsaigon                build_repo = CB_REPO.replace("couchbase-server", "moxi") + version[:5] + "/"
289a13b0750Ssaigon            elif version[:5] not in COUCHBASE_VERSION_2 and \
290a13b0750Ssaigon                 version[:5] not in COUCHBASE_VERSION_3:
291a13b0750Ssaigon                if version[:3] in CB_VERSION_NAME:
292a13b0750Ssaigon                    build_repo = CB_REPO + CB_VERSION_NAME[version[:3]] + "/"
293a13b0750Ssaigon                else:
294a13b0750Ssaigon                    sys.exit("version is not support yet")
2953c8a3212SAruna Piravi            if 'enable_ipv6' in params and params['enable_ipv6']:
2963c8a3212SAruna Piravi                build_repo = build_repo.replace(CB_DOWNLOAD_SERVER,
2973c8a3212SAruna Piravi                                                CB_DOWNLOAD_SERVER_FQDN)
29806a7d5d9Sfarshid            for name in names:
299f586f28fSsaigon                if version[:5] in releases_version:
30062486499Ssaigon                    build = BuildQuery().find_membase_release_build(
30162486499Ssaigon                                             deliverable_type=info.deliverable_type,
30262486499Ssaigon                                             os_architecture=info.architecture_type,
30362486499Ssaigon                                             build_version=version,
30462486499Ssaigon                                             product='membase-server-enterprise')
3051e56f36bSsaigon                elif len(version) > 6 and version[6:].replace("-rel", "") == \
3061e56f36bSsaigon                                                    CB_RELEASE_BUILDS[version[:5]]:
307972639abSsaigon                    build = BuildQuery().find_couchbase_release_build(
3087086f250Ssaigon                                            deliverable_type=info.deliverable_type,
3097086f250Ssaigon                                            os_architecture=info.architecture_type,
3107086f250Ssaigon                                            build_version=version,
3117086f250Ssaigon                                            product=name,
3126d5639efSsaigon                                            os_version = info.distribution_version,
3136d5639efSsaigon                                            direct_build_url=direct_build_url)
314c13b8f2aSsaigon                else:
31562486499Ssaigon                    builds, changes = BuildQuery().get_all_builds(version=version,
31662486499Ssaigon                                      timeout=timeout,
31762486499Ssaigon                                      direct_build_url=direct_build_url,
31862486499Ssaigon                                      deliverable_type=info.deliverable_type,
31962486499Ssaigon                                      architecture_type=info.architecture_type,
32062486499Ssaigon                                      edition_type=name,
32162486499Ssaigon                                      repo=build_repo, toy=toy,
32262486499Ssaigon                                      distribution_version=info.distribution_version.lower(),
323766fb9e7Ssaigon                                      distribution_type=info.distribution_type.lower())
32462486499Ssaigon                    build = BuildQuery().find_build(builds, name, info.deliverable_type,
32562486499Ssaigon                                               info.architecture_type, version, toy=toy,
32662486499Ssaigon                                     openssl=openssl, direct_build_url=direct_build_url,
32762486499Ssaigon                                 distribution_version=info.distribution_version.lower(),
32862486499Ssaigon                                       distribution_type=info.distribution_type.lower())
329898ef744SSteve Yen
3307fa87c49Sfarshid                if build:
33106a7d5d9Sfarshid                    if 'amazon' in params:
33276f684b9Ssaigon                        type = info.type.lower()
33376f684b9Ssaigon                        if type == 'windows' and version in releases_version:
33476f684b9Ssaigon                            build.url = build.url.replace("http://builds.hq.northscale.net",
33576f684b9Ssaigon                                                          "https://s3.amazonaws.com/packages.couchbase")
33676f684b9Ssaigon                            build.url = build.url.replace("enterprise", "community")
33776f684b9Ssaigon                            build.name = build.name.replace("enterprise", "community")
33876f684b9Ssaigon                        else:
33900243638Ssaigon                            """ since url in S3 insert version into it, we need to put version
34000243638Ssaigon                                in like ..latestbuilds/3.0.0/... """
34100243638Ssaigon                            cb_version = version[:5]
34200243638Ssaigon                            build.url = build.url.replace("http://builds.hq.northscale.net/latestbuilds",
34300243638Ssaigon                                        "http://packages.northscale.com/latestbuilds/{0}".format(cb_version))
34400243638Ssaigon                            """ test enterprise version """
34500243638Ssaigon                            #build.url = build.url.replace("enterprise", "community")
34600243638Ssaigon                            #build.name = build.name.replace("enterprise", "community")
347b1c9c13dSsaigon                    """ check if URL is live """
3485aec3126Ssaigon                    url_valid = False
349b1c9c13dSsaigon                    remote_client = RemoteMachineShellConnection(server)
3505aec3126Ssaigon                    url_valid = remote_client.is_url_live(build.url)
3515aec3126Ssaigon                    remote_client.disconnect()
3525aec3126Ssaigon                    if url_valid:
353b1c9c13dSsaigon                        return build
354b1c9c13dSsaigon                    else:
355b1c9c13dSsaigon                        sys.exit("ERROR: URL is not good. Check URL again")
356a607411eSfarshid            _errors.append(errors["BUILD-NOT-FOUND"])
3579209801eSsaigon        if not linux_repo:
3589209801eSsaigon            msg = "unable to find a build for product {0} version {1} for package_type {2}"
3599209801eSsaigon            raise Exception(msg.format(names, version, info.deliverable_type))
36006a7d5d9Sfarshid
361f6f4c9ebSPavel.Paulau    def is_socket_active(self, host, port, timeout=300):
362f6f4c9ebSPavel.Paulau        """ Check if remote socket is open and active
363f6f4c9ebSPavel.Paulau
364f6f4c9ebSPavel.Paulau        Keyword arguments:
365f6f4c9ebSPavel.Paulau        host -- remote address
366f6f4c9ebSPavel.Paulau        port -- remote port
367f6f4c9ebSPavel.Paulau        timeout -- check timeout (in seconds)
368f6f4c9ebSPavel.Paulau
369f6f4c9ebSPavel.Paulau        Returns:
370f6f4c9ebSPavel.Paulau        True -- socket is active
371f6f4c9ebSPavel.Paulau        False -- otherwise
372f6f4c9ebSPavel.Paulau
373f6f4c9ebSPavel.Paulau        """
374f6f4c9ebSPavel.Paulau        start_time = time.time()
375f6f4c9ebSPavel.Paulau
376f6f4c9ebSPavel.Paulau        sckt = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
377f6f4c9ebSPavel.Paulau
378f6f4c9ebSPavel.Paulau        while time.time() - start_time < timeout:
379f6f4c9ebSPavel.Paulau            try:
380f6f4c9ebSPavel.Paulau                sckt.connect((host, port))
381f6f4c9ebSPavel.Paulau                sckt.shutdown(2)
382f6f4c9ebSPavel.Paulau                sckt.close()
383f6f4c9ebSPavel.Paulau                return True
384f6f4c9ebSPavel.Paulau            except:
385f6f4c9ebSPavel.Paulau                time.sleep(10)
386f6f4c9ebSPavel.Paulau
387f6f4c9ebSPavel.Paulau        return False
388e3087ca6SKaran Kumar
38906a7d5d9Sfarshidclass MembaseServerInstaller(Installer):
39006a7d5d9Sfarshid    def __init__(self):
39106a7d5d9Sfarshid        Installer.__init__(self)
39206a7d5d9Sfarshid
39306a7d5d9Sfarshid    def initialize(self, params):
39406a7d5d9Sfarshid        start_time = time.time()
39506a7d5d9Sfarshid        cluster_initialized = False
39606a7d5d9Sfarshid        server = params["server"]
39715f3a652SAndrei Baranouski        while time.time() < (start_time + (5 * 60)):
39806a7d5d9Sfarshid            rest = RestConnection(server)
39906a7d5d9Sfarshid            try:
40032ec96c2SKaran Kumar                if server.data_path:
40132ec96c2SKaran Kumar                    remote_client = RemoteMachineShellConnection(server)
4024a524cd4Skkumar                    remote_client.execute_command('rm -rf {0}/*'.format(server.data_path))
4034a524cd4Skkumar                    # Make sure that data_path is writable by membase user
40432ec96c2SKaran Kumar                    remote_client.execute_command("chown -R membase.membase {0}".format(server.data_path))
4059390b028Sandreibara                    remote_client.disconnect()
40632ec96c2SKaran Kumar                    rest.set_data_path(data_path=server.data_path)
40706a7d5d9Sfarshid                rest.init_cluster(username=server.rest_username, password=server.rest_password)
408a607411eSfarshid                rest.init_cluster_memoryQuota(memoryQuota=rest.get_nodes_self().mcdMemoryReserved)
40906a7d5d9Sfarshid                cluster_initialized = True
41006a7d5d9Sfarshid                break
41106a7d5d9Sfarshid            except ServerUnavailableException:
41206a7d5d9Sfarshid                log.error("error happened while initializing the cluster @ {0}".format(server.ip))
41306a7d5d9Sfarshid            log.info('sleep for 5 seconds before trying again ...')
41406a7d5d9Sfarshid            time.sleep(5)
41506a7d5d9Sfarshid        if not cluster_initialized:
41606a7d5d9Sfarshid            raise Exception("unable to initialize membase node")
41706a7d5d9Sfarshid
418b9c46d85Sandreibara    def install(self, params, queue=None):
419b9c46d85Sandreibara        try:
420b9c46d85Sandreibara            build = self.build_url(params)
421b9c46d85Sandreibara        except Exception, e:
422b9c46d85Sandreibara            if queue:
423b9c46d85Sandreibara                queue.put(False)
424b9c46d85Sandreibara            raise e
42506a7d5d9Sfarshid        remote_client = RemoteMachineShellConnection(params["server"])
42668d9bc3dSsaigon        info = remote_client.extract_remote_info()
42768d9bc3dSsaigon        type = info.type.lower()
428edc8139aSkkumar        server = params["server"]
429aa58812aSChisheng Hong        if "vbuckets" in params:
430aa58812aSChisheng Hong            vbuckets = int(params["vbuckets"][0])
431aa58812aSChisheng Hong        else:
432aa58812aSChisheng Hong            vbuckets = None
43352632f18Ssaigon        if "swappiness" in params:
43452632f18Ssaigon            swappiness = int(params["swappiness"])
435abe63c9eSsaigon        else:
4369a779cd8Sarunapiravi            swappiness = 0
43752632f18Ssaigon
438329f692cSsaigon        if "openssl" in params:
439329f692cSsaigon            openssl = params["openssl"]
440016ee3a2Ssaigon        else:
441016ee3a2Ssaigon            openssl = ""
442329f692cSsaigon
44368d9bc3dSsaigon        if type == "windows":
44468d9bc3dSsaigon            build = self.build_url(params)
445641112f5Ssaigon            remote_client.download_binary_in_win(build.url, params["version"])
446a4657190Ssaigon            success = remote_client.install_server_win(build, params["version"], \
447a4657190Ssaigon                                                       vbuckets=vbuckets)
44868d9bc3dSsaigon        else:
44968d9bc3dSsaigon            downloaded = remote_client.download_build(build)
45068d9bc3dSsaigon            if not downloaded:
451e97b0123Ssaigon                log.error('server {1} unable to download binaries : {0}' \
452e97b0123Ssaigon                          .format(build.url, params["server"].ip))
4539a0422afSandreibara                return False
454edc8139aSkkumar            path = server.data_path or '/tmp'
455329f692cSsaigon            success &= remote_client.install_server(build, path=path, vbuckets=vbuckets, \
45679250bc7SAndrei Baranouski                                                    swappiness=swappiness, openssl=openssl)
45768d9bc3dSsaigon            ready = RestHelper(RestConnection(params["server"])).is_ns_server_running(60)
45868d9bc3dSsaigon            if not ready:
45968d9bc3dSsaigon                log.error("membase-server did not start...")
46077266c30Ssaigon            log.info('wait 5 seconds for Membase server to start')
46168d9bc3dSsaigon            time.sleep(5)
4629390b028Sandreibara        remote_client.disconnect()
463b9c46d85Sandreibara        if queue:
464b9c46d85Sandreibara            queue.put(success)
465b9c46d85Sandreibara        return success
46606a7d5d9Sfarshid
467e3087ca6SKaran Kumar
46806a7d5d9Sfarshidclass CouchbaseServerInstaller(Installer):
46906a7d5d9Sfarshid    def __init__(self):
47006a7d5d9Sfarshid        Installer.__init__(self)
47106a7d5d9Sfarshid
47206a7d5d9Sfarshid    def initialize(self, params):
47339b7b731Ssaigon        #log.info('*****CouchbaseServerInstaller initialize the application ****')
47406a7d5d9Sfarshid        start_time = time.time()
47506a7d5d9Sfarshid        cluster_initialized = False
47606a7d5d9Sfarshid        server = params["server"]
47793eb8ff8Skkumar        remote_client = RemoteMachineShellConnection(params["server"])
478db6e1c26Ssaigon        success = True
479db6e1c26Ssaigon        success &= remote_client.is_couchbase_installed()
480db6e1c26Ssaigon        if not success:
481db6e1c26Ssaigon            mesg = "\n\nServer {0} failed to install".format(params["server"].ip)
482db6e1c26Ssaigon            sys.exit(mesg)
48315f3a652SAndrei Baranouski        while time.time() < start_time + 5 * 60:
48406a7d5d9Sfarshid            try:
485f3914971Sronniedada                rest = RestConnection(server)
486f3914971Sronniedada
487ce5a5b38SPavel.Paulau                # Optionally change node name and restart server
488ce5a5b38SPavel.Paulau                if params.get('use_domain_names', 0):
489b4777408SIrynaMironava                    RemoteUtilHelper.use_hostname_for_server_settings(server)
490506ad72aSPavel.Paulau
4917ef927e0SAruna Piravi                if params.get('enable_ipv6', 0):
492fe268802SAruna Piravi                    status, content = RestConnection(server).rename_node(
493017f6c20SAruna Piravi                        hostname=server.ip.replace('[', '').replace(']', ''))
494fe268802SAruna Piravi                    if status:
495fe268802SAruna Piravi                        log.info("Node {0} renamed to {1}".format(server.ip,
496fe268802SAruna Piravi                                                                  server.ip.replace('[', '').
497fe268802SAruna Piravi                                                                  replace(']', '')))
498fe268802SAruna Piravi                    else:
499fe268802SAruna Piravi                        log.error("Error renaming node {0} to {1}: {2}".
500fe268802SAruna Piravi                                  format(server.ip,
501fe268802SAruna Piravi                                         server.ip.replace('[', '').replace(']', ''),
502fe268802SAruna Piravi                                         content))
5037ef927e0SAruna Piravi
5046b81dc3fSandreibara                # Make sure that data_path and index_path are writable by couchbase user
5056b81dc3fSandreibara                for path in set(filter(None, [server.data_path, server.index_path])):
50693eb8ff8Skkumar                    time.sleep(3)
507506ad72aSPavel.Paulau
5086b81dc3fSandreibara                    for cmd in ("rm -rf {0}/*".format(path),
5096b81dc3fSandreibara                                "chown -R couchbase:couchbase {0}".format(path)):
510506ad72aSPavel.Paulau                        remote_client.execute_command(cmd)
5116b81dc3fSandreibara                rest.set_data_path(data_path=server.data_path,
51216e90a76SPavel.Paulau                                       index_path=server.index_path)
5136b81dc3fSandreibara                time.sleep(3)
514506ad72aSPavel.Paulau
515b8100915SPavel.Paulau                # Initialize cluster
516b5c63c23SParag Agarwal                if "init_nodes" in params:
517b5c63c23SParag Agarwal                    init_nodes = params["init_nodes"]
518b5c63c23SParag Agarwal                else:
519b5c63c23SParag Agarwal                    init_nodes = "True"
52080157716Sprasanna                if (isinstance(init_nodes, bool) and init_nodes) or \
52180157716Sprasanna                        (isinstance(init_nodes, str) and init_nodes.lower() == "true"):
5227cb3b4beSsaigon                    if not server.services:
5237cb3b4beSsaigon                        set_services = ["kv"]
5247cb3b4beSsaigon                    elif server.services:
5257cb3b4beSsaigon                        set_services = server.services.split(',')
5267cb3b4beSsaigon
5277cb3b4beSsaigon                    kv_quota = 0
5287cb3b4beSsaigon                    while kv_quota == 0:
5297cb3b4beSsaigon                        time.sleep(1)
5300a5618a4Sritamcouchbase                        kv_quota = int(rest.get_nodes_self().mcdMemoryReserved)
5310a5618a4Sritamcouchbase                    info = rest.get_nodes_self()
5320a5618a4Sritamcouchbase                    cb_version = info.version[:5]
53333445f7aSsaigon                    kv_quota = int(info.mcdMemoryReserved * CLUSTER_QUOTA_RATIO)
534fce05e6bSRitesh Agarwal                    """ for fts, we need to grep quota from ns_server
535fce05e6bSRitesh Agarwal                                but need to make it works even RAM of vm is
536fce05e6bSRitesh Agarwal                                smaller than 2 GB """
537be415535Sprasanna
5380a5618a4Sritamcouchbase                    if cb_version in COUCHBASE_FROM_VERSION_4:
539fce05e6bSRitesh Agarwal                        if "index" in set_services:
540fce05e6bSRitesh Agarwal                            log.info("quota for index service will be %s MB" % (INDEX_QUOTA))
541fce05e6bSRitesh Agarwal                            kv_quota -= INDEX_QUOTA
542f2ad897eSsaigon                            log.info("set index quota to node %s " % server.ip)
543fbcb51c4SRitesh Agarwal                            rest.set_service_memoryQuota(service='indexMemoryQuota', memoryQuota=INDEX_QUOTA)
544fce05e6bSRitesh Agarwal                        if "fts" in set_services:
545fce05e6bSRitesh Agarwal                            log.info("quota for fts service will be %s MB" % (FTS_QUOTA))
546fce05e6bSRitesh Agarwal                            kv_quota -= FTS_QUOTA
547fce05e6bSRitesh Agarwal                            log.info("set both index and fts quota at node %s "% server.ip)
548fbcb51c4SRitesh Agarwal                            rest.set_service_memoryQuota(service='ftsMemoryQuota', memoryQuota=FTS_QUOTA)
549fce05e6bSRitesh Agarwal                        if "cbas" in set_services:
550fce05e6bSRitesh Agarwal                            log.info("quota for cbas service will be %s MB" % (CBAS_QUOTA))
551fce05e6bSRitesh Agarwal                            kv_quota -= CBAS_QUOTA
552fbcb51c4SRitesh Agarwal                            rest.set_service_memoryQuota(service = "cbasMemoryQuota", memoryQuota=CBAS_QUOTA)
553fce05e6bSRitesh Agarwal                        if kv_quota < MIN_KV_QUOTA:
5540a5618a4Sritamcouchbase                                raise Exception("KV RAM needs to be more than %s MB"
5550a5618a4Sritamcouchbase                                        " at node  %s"  % (MIN_KV_QUOTA, server.ip))
556e4069263Ssaigon                    """ set kv quota smaller than 1 MB so that it will satify
557e4069263Ssaigon                        the condition smaller than allow quota """
558e4069263Ssaigon                    kv_quota -= 1
5590a5618a4Sritamcouchbase                    log.info("quota for kv: %s MB" % kv_quota)
5607cb3b4beSsaigon                    rest.init_cluster_memoryQuota(server.rest_username, \
5617cb3b4beSsaigon                                                       server.rest_password, \
5627cb3b4beSsaigon                                                                     kv_quota)
5630a5618a4Sritamcouchbase                    if params["version"][:5] in COUCHBASE_FROM_VERSION_4:
5640a5618a4Sritamcouchbase                        rest.init_node_services(username=server.rest_username,
565af987c62SIrynaMironava                                                password=server.rest_password,
5667cb3b4beSsaigon                                                        services=set_services)
5676a6582bdSsaigon                    if "index" in set_services:
568be415535Sprasanna                        if "storage_mode" in params:
569be415535Sprasanna                            storageMode = params["storage_mode"]
570be415535Sprasanna                        else:
571be415535Sprasanna                            storageMode = "plasma"
572be415535Sprasanna                        rest.set_indexer_storage_mode(storageMode=storageMode)
5730a1108bbSIrynaMironava                    rest.init_cluster(username=server.rest_username,
5747cb3b4beSsaigon                                         password=server.rest_password)
575506ad72aSPavel.Paulau
576506ad72aSPavel.Paulau                # Optionally disable consistency check
577506ad72aSPavel.Paulau                if params.get('disable_consistency', 0):
578506ad72aSPavel.Paulau                    rest.set_couchdb_option(section='couchdb',
579506ad72aSPavel.Paulau                                            option='consistency_check_ratio',
580506ad72aSPavel.Paulau                                            value='0.0')
581506ad72aSPavel.Paulau
5820cbf53f4Sronniedada                # memcached env variable
5830cbf53f4Sronniedada                mem_req_tap_env = params.get('MEMCACHED_REQS_TAP_EVENT', 0)
5840cbf53f4Sronniedada                if mem_req_tap_env:
5850cbf53f4Sronniedada                    remote_client.set_environment_variable('MEMCACHED_REQS_TAP_EVENT',
5860cbf53f4Sronniedada                                                           mem_req_tap_env)
5870a5618a4Sritamcouchbase                """ set cbauth environment variables from Watson version
5880a5618a4Sritamcouchbase                    it is checked version inside method """
5890a5618a4Sritamcouchbase                remote_client.set_cbauth_env(server)
59052df04ceSsaigon                remote_client.check_man_page()
59100403f66Ssaigon                """ add unzip command on server if it is not available """
59200403f66Ssaigon                remote_client.check_cmd("unzip")
593898cd79aSsaigon                remote_client.is_ntp_installed()
5949390b028Sandreibara                remote_client.disconnect()
595a9d19da9SAndrei Baranouski                # TODO: Make it work with windows
596b2cd510cSKaran Kumar                if "erlang_threads" in params:
59760c8290fSandreibara                    num_threads = params.get('erlang_threads', testconstants.NUM_ERLANG_THREADS)
598b2cd510cSKaran Kumar                    # Stop couchbase-server
599b2cd510cSKaran Kumar                    ClusterOperationHelper.stop_cluster([server])
60060c8290fSandreibara                    if "sync_threads" in params or ':' in num_threads:
60160c8290fSandreibara                        sync_threads = params.get('sync_threads', True)
60260c8290fSandreibara                    else:
60360c8290fSandreibara                        sync_threads = False
60460c8290fSandreibara                    # Change type of threads(sync/async) and num erlang threads
60560c8290fSandreibara                    ClusterOperationHelper.change_erlang_threads_values([server], sync_threads, num_threads)
606b2cd510cSKaran Kumar                    # Start couchbase-server
607b2cd510cSKaran Kumar                    ClusterOperationHelper.start_cluster([server])
6080abbae49SKaran Kumar                if "erlang_gc_level" in params:
6090abbae49SKaran Kumar                    erlang_gc_level = params.get('erlang_gc_level', None)
6100abbae49SKaran Kumar                    if erlang_gc_level is None:
6110abbae49SKaran Kumar                        # Don't change the value
6120abbae49SKaran Kumar                        break
6130abbae49SKaran Kumar                    # Stop couchbase-server
6140abbae49SKaran Kumar                    ClusterOperationHelper.stop_cluster([server])
6150abbae49SKaran Kumar                    # Change num erlang threads
6160abbae49SKaran Kumar                    ClusterOperationHelper.change_erlang_gc([server], erlang_gc_level)
6170abbae49SKaran Kumar                    # Start couchbase-server
6180abbae49SKaran Kumar                    ClusterOperationHelper.start_cluster([server])
61906a7d5d9Sfarshid                cluster_initialized = True
62006a7d5d9Sfarshid                break
62106a7d5d9Sfarshid            except ServerUnavailableException:
62206a7d5d9Sfarshid                log.error("error happened while initializing the cluster @ {0}".format(server.ip))
62306a7d5d9Sfarshid            log.info('sleep for 5 seconds before trying again ...')
62406a7d5d9Sfarshid            time.sleep(5)
62506a7d5d9Sfarshid        if not cluster_initialized:
626a6b8bb2dSandreibara            sys.exit("unable to initialize couchbase node")
62706a7d5d9Sfarshid
628b9c46d85Sandreibara    def install(self, params, queue=None):
629a6dbe12bSEric
630a6dbe12bSEric        log.info('********CouchbaseServerInstaller:install')
631a6dbe12bSEric
6324af7601eSsaigon        self.msi = 'msi' in params and params['msi'].lower() == 'true'
633a8cb468fSAruna Piravi        start_server = True
634b9c46d85Sandreibara        try:
635