1# fab --version
2# fab -h
3
4# command line switches
5#
6# fab -i <pem-file> -f 2ifab.py -u ubuntu -H <host> <command>
7#
8
9# to list all commands,
10#
11# $ fab -f 2ifab.py -l
12#
13
14# help for a specific command
15#
16# $ fab -f 2ifab.py -d <command>
17
18# to run a command in remote machine
19#
20# $ fab -H system1,system2,system3 -- uname -a
21#
22
23# to setup remote machine directories, packages, golang, clone 2i repo.
24#
25# $ fab -i <pem> -u <user> -H <host> -f 2ifab.py setup
26#
27
28# to cleanup remote machine,
29#
30# $ fab -i <pem> -u <user> -H <host> -f 2ifab.py cleanall
31#
32
33# to uninstall and install couchbase,
34#
35# fab -i <pem> -f 2ifab.py cb_uninstall cb_install:url=<url-link>
36#
37
38# to stop, start or restart couchbase service,
39#
40# fab -i <pem> -u <user> -H <host> -f 2ifab.py cb_service:do=stop
41# fab -i <pem> -u <user> -H <host> -f 2ifab.py cb_service:do=start
42# fab -i <pem> -u <user> -H <host> -f 2ifab.py cb_service:do=restart
43
44# to initialize a node and start a cluster,
45#
46# fab -i <pem> -f 2ifab.py cluster_init:services="data;index"
47#
48# to initialize a node and add to a cluster,
49#
50# fab -i <pem> -f 2ifab.py server_add:services="data;index",cluster=<node>
51#
52
53# to create one or more buckets,
54#
55# $ fab -i <pem> -f 2ifab.py create_buckets:buckets="default;users"
56#
57
58# to load documents,
59#
60# fab -i <pem> -f 2ifab.py loadgen:procs=32,count=625000,par=2
61#
62
63# some shell commands,
64#
65# $ adduser {user} --disabled-password --gecos ""
66# $ addgroup {group}
67# $ echo "%{group} ALL=(ALL) ALL" >> # /etc/sudoers
68# $ adduser {user} {group}
69# $ echo "{user}:{password}" | chpasswd
70# $ cbcollect-info <filename.zip>"
71# $ curl -v --upload-file logs232.tar https://s3.amazonaws.com/bugdb/jira/MB-16033/logs232.tar
72#
73
74from __future__ import with_statement
75from __future__ import print_function
76import fabric
77import fabric.utils
78from fabric.api import *
79from fabric.contrib.console import confirm
80import os
81import os.path
82import time
83
84pkgs = [
85    "git", "mercurial", "libsasl2-2", "sasl2-bin", "gcc", "cmake", "make",
86    "libsnappy-dev", "g++", "protobuf-compiler", "sysstat", "graphviz",
87    "atop", "htop", "iotop"
88]
89
90env.use_ssh_config = True
91
92(user2i, passw2i) = "Administrator", "asdasd"
93ramsize2i = 8192
94pkgdir = "/opt/pkgs"
95installdir = "/opt/couchbase"
96goproj, godeps = "/opt/goproj", "/opt/godeps"
97gopath = ":".join([goproj, godeps])
98goroot = "/usr/local/go"
99shpath = goroot + "/bin" + ":$PATH"
100
101fabric.state.output["running"] = False
102fabric.state.output["stdout"] = False
103
104#---- node tasks
105
106ulimit_conf = """#<domain> <type> <item> <value>
107* soft nofile {flimit}
108* hard nofile {flimit}
109"""
110@task
111def ulimit(flimit=20000):
112    """you might want to use `fab -f 2ifab.py -- <command>`"""
113    txt = ulimit_conf.format(flimit=flimit)
114    cmd = 'echo "{txt}" >> /etc/security/limits.conf'.format(txt=txt)
115    trycmd(cmd.format(ulimit_conf=ulimit_conf), op="sudo")
116
117@task
118@parallel
119def new_user(user, passw, group) :
120    """create a new user and group"""
121    env.warn_only = True
122    trycmd('adduser {user} --disabled-password --gecos ""'.format(user=user),
123           op="sudo")
124    trycmd('adduser {user} {group}'.format(user=user,group=group), op="sudo")
125    trycmd('echo "{user}:{passw}" | chpasswd'.format(user=user,passw=passw),
126            op="sudo")
127    env.warn_only = False
128
129govers = {
130    "133": "https://storage.googleapis.com/golang/go1.3.3.linux-amd64.tar.gz",
131    "143": "https://storage.googleapis.com/golang/go1.4.3.linux-amd64.tar.gz",
132    "142": "https://storage.googleapis.com/golang/go1.4.2.linux-amd64.tar.gz",
133    "151": "https://storage.googleapis.com/golang/go1.5.1.linux-amd64.tar.gz",
134    "162": "https://storage.googleapis.com/golang/go1.6.2.linux-amd64.tar.gz",
135}
136@task
137@parallel
138def setup(targetos="deb", gover="162"):
139    """setup target nodes in the cluster
140    - install os level packages
141    - create couchbase user
142    - create and setup /opt/{pkgs,couchbase,godeps,goproj}
143    - install golang specified version
144    - install github.com/couchbase/indexing repository and all its deps
145    """
146    packages = " ".join(pkgs)
147    if targetos == "deb" :
148        trycmd("apt-get install %s --assume-yes" % packages, op="sudo")
149    if targetos == "centos" :
150        trycmd("yum install %s -y" % packages, op="sudo")
151
152    for d in [pkgdir, goproj, godeps] :
153       trycmd("rm -rf %s" % d, op="sudo")
154       trycmd("mkdir -p %s" % d, op="sudo")
155       trycmd("chown %s:%s %s" % (env.user, env.user, d), op="sudo")
156
157    # install golang
158    trycmd("rm -rf %s" % goroot, op="sudo") # first un-install
159    with cd(pkgdir):
160        link = govers[gover]
161        targz = link.split("/")[-1]
162        trycmd("wget %s" % link)
163        trycmd("tar -C /usr/local -xzf %s" % targz, op="sudo")
164    with shell_env(PATH=shpath, GOPATH=gopath, GOROOT=goroot):
165         trycmd("go version", v=True)
166
167    # clone 2i repository and all its dependencies
168    with shell_env(PATH=shpath, GOPATH=gopath, GOROOT=goroot) :
169        trycmd("go get -u -d github.com/couchbase/indexing/...")
170
171    # set up protobuf
172    path = os.sep.join([goproj, "src", "github.com", "golang", "protobuf"])
173    with cd(path), shell_env(PATH=shpath, GOPATH=gopath, GOROOT=goroot) :
174        trycmd("go get -u -d ./...")
175        trycmd("go install ./...")
176
177@task
178@parallel
179def cleanall():
180    """cleanup /opts/{pkgs,couchbase,goproj,godeps}"""
181    for d in [pkgdir, goproj, godeps] :
182        trycmd("rm -rf %s" %d)
183    trycmd("rm -f /tmp/patch*")
184    trycmd("rm -rf %s" % goroot, op="sudo")
185
186@task
187@parallel
188def reboot():
189    trycmd("shutdown -r now", op="sudo")
190
191@task
192@parallel
193def killall(programs="indexer;projector"):
194    [ trycmd("killall {program}".format(program=program), op="sudo")
195      for program in programs.split(";") ]
196
197@task
198@parallel
199def fix_dpkg():
200    """fix dpkg in case of broken ssh connection"""
201    trycmd("dpkg --configure -a", op="sudo")
202    trycmd("apt-get update", op="sudo")
203
204
205#---- coucbase node tasks
206
207@task
208@parallel
209def cb_install(tar="", debs=""):
210    """download the tar file from `tar` and install"""
211    pp = pp_for_host(env.host)
212
213    if tar != "" :
214        installfile = tar.split("/")[-1]
215        commands = [
216            ["rm -f couchbase-server* installer*", {}],
217            ["wget %s" % tar, {}],
218            ["tar xvf %s" % installfile, {}],
219            ["dpkg -i couchbase-server_*", {"op":"sudo"}],
220        ]
221        with cd(pkgdir) :
222            all(map(lambda x: trycmd(x[0], **x[1]), commands))
223
224    elif debs != "" :
225        debs = debs.replace(";",",").split(",")
226        for deb in debs :
227            commands = [
228                ["rm -f couchbase-server* installer*", {}],
229                ["wget %s" % deb, {}],
230                ["dpkg -i couchbase-server*", {"op":"sudo"}],
231            ]
232            with cd(pkgdir) :
233                all(map(lambda x: trycmd(x[0], **x[1]), commands))
234
235    else :
236        pp("error please provide a tar or deb files")
237        return
238
239@task
240@parallel
241def cb_uninstall():
242    """uninstall couchbase server"""
243    env.warn_only = True
244    trycmd("/etc/init.d/couchbase-server stop", op="sudo")
245    for d in [installdir] :
246        trycmd("rm -rf %s" %d, op="sudo")
247    trycmd("dpkg -r couchbase-server", op="sudo")
248    trycmd("dpkg --purge couchbase-server", op="sudo")
249    env.warn_only = False
250
251@task
252@parallel
253def cb_service(do="restart"):
254    """start/stop/restart couchbase server"""
255    trycmd("/etc/init.d/couchbase-server %s" % do, op="sudo")
256
257fmt_cluster_init = "\
258./couchbase-cli cluster-init \
259--cluster=%s --cluster-username=%s --cluster-password=%s \
260--cluster-ramsize=%s -d --services=%s"
261@task
262def cluster_init(services="",ramsize=8192):
263    """initialize couchbase cluster and rebalance them,
264    EG: services="data;index;query" """
265    pp = pp_for_host(env.host)
266    if services == "" :
267        print("please provide the services to start for this node")
268        return
269    services = services.replace(";",",")
270    with cd("/opt/couchbase/bin"):
271        # cluster-init
272        params = (env.host, user2i, passw2i, ramsize, services)
273        cmd = fmt_cluster_init % params
274        trycmd(cmd, op="run")
275
276fmt_server_add = "\
277./couchbase-cli server-add --cluster=%s --user=%s --password=%s \
278--server-add=%s --server-add-username=%s --server-add-password=%s \
279--services='%s'"
280fmt_rebalance_in = "\
281./couchbase-cli rebalance --cluster=%s --user=%s --password=%s \
282--server-add=%s --server-add-username=%s --server-add-password=%s \
283--services='%s'"
284@task
285def server_add(services="", cluster=""):
286    """add node to server"""
287
288    pp = pp_for_host(env.host)
289    if services == "" :
290        print("please provide the services to start for this node")
291        return
292    if cluster == "" :
293        print("please provide the cluster address to this node")
294        return
295
296    services = services.replace(";",",")
297    with cd("/opt/couchbase/bin"):
298        # server-add
299        cmd = fmt_rebalance_in % (cluster,  user2i, passw2i, env.host,
300              user2i, passw2i, services)
301        trycmd(cmd, op="run")
302
303fmt_failover = "\
304./couchbase-cli failover --cluster=%s --user=%s --password=%s \
305--server-failover=%s --force"
306@task
307def failover(cluster=""):
308    pp = pp_for_host(env.host)
309    if cluster == "" :
310        print("please provide the cluster address to this node")
311        return
312
313    with cd("/opt/couchbase/bin"):
314        # server-add
315        cmd = fmt_failover % (cluster,  user2i, passw2i, env.host)
316        trycmd(cmd, op="run")
317
318fmt_rebalanceout = "\
319./couchbase-cli rebalance --cluster=%s --user=%s --password=%s \
320--server-remove=%s"
321@task
322def rebalance_out(cluster=""):
323    pp = pp_for_host(env.host)
324    if cluster == "" :
325        print("please provide the cluster address to this node")
326        return
327
328    with cd("/opt/couchbase/bin"):
329        # server-add
330        cmd = fmt_rebalanceout % (cluster,  user2i, passw2i, env.host)
331        trycmd(cmd, op="run")
332
333fmt_create_bucket = "\
334./couchbase-cli bucket-create \
335--cluster=%s --user=%s --password=%s \
336--bucket=%s \
337--bucket-password="" \
338--bucket-ramsize=%s \
339--bucket-replica=1 \
340--bucket-type=couchbase \
341--enable-flush=1 \
342--wait"
343@task
344def create_buckets(buckets="", ramsize="2048"):
345    """create one or more buckets (input received as csv of buckets)"""
346    if buckets == "" :
347        print("please provided comma-separated list of buckets to create")
348
349    for bucket in buckets.split(";") :
350        with cd("/opt/couchbase/bin"):
351            cmd = fmt_create_bucket % (env.host, user2i, passw2i, bucket, ramsize)
352            trycmd(cmd, op="run")
353
354fmt_bucket_flush = "\
355./couchbase-cli bucket-flush \
356--cluster=%s:%s --user=%s --password=%s \
357--bucket=%s \
358--force"
359@task
360def bucket_flush(buckets="",port="8091"):
361    """flush one or more buckets"""
362    if buckets == "" :
363        print("please provided comma-separated list of buckets to create")
364
365    for bucket in buckets.split(";") :
366        with cd("/opt/couchbase/bin"):
367            cmd = fmt_bucket_flush % (env.host, port, user2i, passw2i, bucket)
368            trycmd(cmd, op="run")
369
370
371fmt_loadgen = "\
372go build; GOMAXPROCS=%s ./loadgen -auth %s:%s -count %s -par %s -worker %s \
373-ratio %s -buckets %s -bagdir %s -prods %s -randkey=%s -prefix %s %s"
374@task
375@parallel
376def loadgen(
377        cluster="localhost:9000", procs=32, count=100000, par=16,
378        buckets="default", worker="monster", prods="projects.prod", randkey="True",
379        prefix="", ratio="0;0;0") :
380    """genetate load over couchbase buckets"""
381    repopath = os.sep.join(["src", "github.com", "couchbase", "indexing"])
382    pathldgn = os.sep.join([goproj, repopath, "secondary", "tools", "loadgen"])
383
384    path_monster = os.sep.join(["src", "github.com", "prataprc", "monster"])
385    bagdir = os.sep.join([goproj, path_monster, "bags"])
386    prodpath = os.sep.join([goproj, path_monster, "prods"])
387
388    buckets = buckets.replace(";",",")
389    ratio = ratio.replace(";", ",")
390    prodfiles = [ os.sep.join([prodpath, prod]) for prod in prods.split(";") ]
391    prodfiles = ",".join(list(prodfiles))
392    if prefix == "" :
393        prefix = env.host
394    with shell_env(PATH=shpath, GOPATH=gopath, GOROOT=goroot), cd(pathldgn) :
395        params = (
396            procs, user2i, passw2i, count, par, worker, ratio, buckets, bagdir,
397            prodfiles, randkey.lower(), prefix, cluster)
398        trycmd(fmt_loadgen % params, op="run")
399
400fmt_log2i = """\
401gunzip {comp}.log.*; cat `ls -1 {comp}.log* | sort -t. -nk3 -r` > \
402{comp}.full.log;"""
403@task
404@parallel
405def log2i(target, comps=""):
406    target = os.sep.join([target, env.host])
407    trycmd("mkdir -p {target}".format(target=target), op="local")
408    logpath = os.sep.join([installdir, "var", "lib", "couchbase", "logs"])
409    with cd(logpath), lcd(target):
410        for comp in comps.replace(";", ",").split(";") :
411            fulllog = comp + ".full.log"
412            trycmd(fmt_log2i.format(comp=comp), op="sudo")
413            trycmd("chown {user}:{group} {f}".format(user=env.user, group=env.user, f=fulllog), op="sudo")
414            targetfile = os.sep.join([logpath, fulllog])
415            trycmd("gzip {targetfile}".format(targetfile=targetfile), op="sudo")
416            cmd = "scp -i {keyfile} {user}@{host}:{targetfile}.gz .".format(
417                    keyfile=env.key_filename[0], user=env.user, host=env.host,
418                    targetfile=targetfile)
419            trycmd(cmd, op="local")
420            trycmd("gunzip %s.gz" % os.path.basename(targetfile), op="local")
421
422@task
423@parallel
424def getprofiles(target, comps=""):
425    target = os.sep.join([target, env.host])
426    trycmd("mkdir -p {target}".format(target=target), op="local")
427    urlmprof = {
428        "projector": "http://%s:9999/debug/pprof/heap",
429        "indexer": "http://%s:9102/debug/pprof/heap",
430    }
431    urlpprof = {
432        "projector": "http://%s:9999/debug/pprof/profile",
433        "indexer": "http://%s:9102/debug/pprof/profile",
434    }
435    for comp in comps.replace(";", ",").split(";") :
436        ex = os.sep.join([installdir, "bin", comp])
437        with lcd(target), shell_env(PATH=shpath, GOPATH=gopath, GOROOT=goroot) :
438            # get memory profile information
439            url = urlmprof[comp] % env.host
440            margs = [
441                ("-inuse_space", ex, url, "/tmp/%s.mprofi.svg"%comp),
442                ("-alloc_space", ex, url, "/tmp/%s.mprofa.svg"%comp),
443                ("-inuse_objects", ex, url, "/tmp/%s.mprofio.svg"%comp),
444                ("-alloc_objects", ex, url, "/tmp/%s.mprofao.svg"%comp),
445            ]
446            for arg in margs :
447                cmd = "go tool pprof %s -svg %s %s > %s" % arg
448                trycmd(cmd)
449                targetfile = arg[-1]
450                cmd = "scp -i {keyfile} {user}@{host}:{targetfile} .".format(
451                        keyfile=env.key_filename[0], user=env.user,
452                        host=env.host, targetfile=targetfile)
453                trycmd(cmd, op="local")
454            # get cpu profile information
455            url = urlpprof[comp] % env.host
456            parg = ( ex, url, "/tmp/%s.pprof.svg"%comp)
457            cmd = "go tool pprof -svg %s %s > %s" % parg
458            trycmd(cmd)
459            targetfile = parg[-1]
460            cmd = "scp -i {keyfile} {user}@{host}:{targetfile} .".format(
461                    keyfile=env.key_filename[0], user=env.user,
462                    host=env.host, targetfile=targetfile)
463            trycmd(cmd, op="local")
464
465fmt_indexperf = "\
466go build; ./cbindexperf -configfile {f} -cluster {cluster} -auth {user}:{passw}"
467@task
468@parallel
469def indexperf(cluster, configfile):
470    """indexer performance tool"""
471    repopath = os.sep.join(["src", "github.com", "couchbase", "indexing"])
472    pathcbp = os.sep.join([goproj, repopath, "secondary", "cmd", "cbindexperf"])
473    targetfile = "/" + os.sep.join(["tmp", os.path.basename(configfile)])
474    trycmd("rm -rf {f}".format(f=targetfile))
475    put(configfile, targetfile)
476    with shell_env(PATH=shpath, GOPATH=gopath, GOROOT=goroot), cd(pathcbp) :
477        trycmd(fmt_indexperf.format(f=targetfile,cluster=cluster,user=user2i,passw=passw2i),
478                v=True)
479
480#---- patching and building target
481
482@task
483@parallel
484def pull2i(branch="unstable"):
485    """pull latest 2i-branch"""
486    repo2i =os.sep.join([goproj,"src","github.com","couchbase","indexing"])
487    with cd(repo2i) :
488        trycmd("git checkout .")
489        trycmd("git clean -f -d")
490        trycmd("git checkout {branch}".format(branch=branch))
491        trycmd("git pull --rebase origin {branch}".format(branch=branch))
492
493@task
494@parallel
495def indexing_unstable():
496    """switch to github.com/couchbase/indexing:unstable branch on all nodes"""
497    path =os.sep.join([gopath,"src","github.com","couchbase","indexing"])
498    with cd(path) :
499        trycmd("git checkout .")
500        trycmd("git clean -f -d")
501        trycmd("git checkout unstable")
502        trycmd("git pull --rebase origin unstable")
503
504@task
505@parallel
506def patch_target(R1="",abort=False):
507    """patch the target node
508    - if R1 is provided, `format-patch` to apply revisions from R1 to target.
509      else, `diff` will be used to apply the uncommited patch to target.
510    - if abort, then any incomplete patch on the target will be aborted.
511    """
512    pp = pp_for_host(env.host)
513    path =os.sep.join([gopath,"src","github.com","couchbase","indexing"])
514
515    with cd(path):
516        if abort :
517            trycmd("git am --abort", v=True)
518            return
519
520    if R1 == "" :
521        patchfile = "/tmp/patch-%s-%s.diff" % (env.host, str(time.time()))
522        pp("patchfile: %s" % patchfile)
523        cmd = "git diff > %s" % patchfile
524    else :
525        patchfile = "/tmp/patch-%s-%s.am" % (env.host, str(time.time()))
526        pp("patchfile: %s" % patchfile)
527        cmd = "git format-patch -k %s..HEAD --stdout > %s" % (R1, patchfile)
528    trycmd(cmd, op="local")
529
530    put(patchfile, patchfile)
531
532    with cd(path):
533        trycmd("git checkout .")
534        trycmd("git clean -f -d")
535        if R1 == "" :
536            trycmd("git apply %s" % patchfile)
537        else :
538            trycmd("git am -3 -k < %s || git am --abort" % patchfile)
539
540@task
541@parallel
542def rebuild_forestdb():
543    """rebuild and install forestdb to remote's source path"""
544    path = os.sep.join([pkgdir, "forestdb"])
545    with cd(pkgdir), shell_env(GOPATH=gopath, GOROOT=goroot) :
546        trycmd("rm -rf %s" % path)
547        trycmd("git clone https://github.com/couchbase/forestdb.git")
548
549    with cd(path), shell_env(GOPATH=gopath, GOROOT=goroot) :
550        trycmd("mkdir -p build")
551        trycmd("cd build; cmake ..; cd ..")
552        trycmd("cd build; make; cd ..")
553        trycmd("cd build; make install; cd ..", op="sudo")
554        target = os.sep.join([installdir, "lib"])
555        trycmd("cp build/libforestdb.so %s" % target, op="sudo")
556        trycmd("ldconfig", op="sudo")
557
558@task
559@parallel
560def rebuild_indexing(R1=""):
561    """patch indexing and rebuild projector and indexer"""
562    if R1 : patch_target(R1=R1)
563    patch_target()
564
565    path = os.sep.join([gopath,"src","github.com","couchbase","indexing"])
566    target = os.sep.join([installdir, "bin"])
567    with cd(path), shell_env(PATH=binpath2i,GOPATH=gopath, GOROOT=goroot):
568        trycmd("cd secondary; ./build.sh; cd ..", v=True)
569        trycmd("mv secondary/cmd/projector/projector %s" % target, v=True)
570        trycmd("mv secondary/cmd/indexer/indexer %s" % target, v=True)
571
572
573@task
574@parallel
575def gitcmd(path="", cmd=""):
576    """run a git command on all nodes"""
577    if cmd == "" :
578        return
579    if path == "" :
580        path = os.sep.join([gopath,"src","github.com","couchbase","indexing"])
581    with cd(path), shell_env(GOPATH=gopath, GOROOT=goroot) :
582        trycmd(cmd, v=True)
583
584#---- local functions
585
586def pp_for_host(host_string) :
587    def fn(*args, **kwargs) :
588        msg = "[%s] " % host_string
589        msg += " ".join(map(str, args))
590        msg += "\n".join(map(lambda k, v: "   %s: %s" % (k, v), kwargs.items()))
591        if msg.lower().find("error") > 0 :
592            fabric.utils.error(msg)
593        else :
594            print(msg)
595    return fn
596
597def trycmd(cmd, op="run", v=False):
598    pp = pp_for_host(env.host)
599    if op == "sudo" :
600        out = sudo(cmd)
601    elif op == "run" :
602        out = run(cmd)
603    elif op == "local" :
604        out = local(cmd)
605
606    if out.failed :
607        pp("cmd failed: %s" % cmd)
608        pp(out)
609        return out.failed
610    elif v :
611        pp(cmd, ":\n", out)
612    else :
613        pp(cmd, ": ok")
614    return out.succeeded
615
616