xref: /3.0.3-GA/couchbase-cli/cbbackupwrapper (revision 5dbd02ab)
1#!/usr/bin/env python
2# -*-python-*-
3
4import base64
5import optparse
6import os
7import re
8import simplejson as json
9import socket
10import subprocess
11import sys
12import urllib2
13
14"""Written by Daniel Owen owend@couchbase.com on 27 June 2014
15Version 1.4    Last updated 10 July 2014
16
17The current implementation of cbbackup that comes with Couchbase Server 2.5.1
18uses only one thead per node.  Therefore when using cbbackup with the single-node
19parameter we are limited to one thread - this impacts performance.
20
21This script provides a wrapper to invoke multiple cbbackup processes.
22It automatically detects which buckets and vbuckets are
23on the node.  It allow the user to specify how many vbuckets to backup in a single
24cbbackup process and then invokes the necessary number of processes.
25An example invocation is as follows:
26
27python cbbackupwrapper.py http://127.0.0.1:8091 ../backup/ --single-node -n 4 \
28-u Administrator -p myPassword --path /opt/couchbbase/bin/  -v
29
30This will backup all the buckets on node 127.0.0.1 into ../backup
31It will backup 4 vbuckets per cbbackup process
32Access to the cluster is authenticated using username=Administrator and
33password=myPassword.and cbbackup will be found in /opt/couchbase/bin
34
35Run python cbbackupwrapper -h for more information.
36
37See the cbrestorewrapper.py script for restoring backups made with this script."""
38
39bucketList = []
40vbucketList = []
41processes = {}
42
43def argumentParsing():
44    usage = "usage: %prog CLUSTER BACKUPDIR OPTIONS"
45    parser = optparse.OptionParser(usage)
46
47    parser.add_option('-b', '--bucket-source', default='',
48                        help='Specify the bucket to backup.  Defaults to all buckets')
49    parser.add_option('--single-node', action='store_true',
50                        default=False, help='use a single server node from the source only')
51    parser.add_option('-u', '--username', default='Administrator',
52                        help='REST username for source cluster or server node. Default is Administrator')
53    parser.add_option('-p', '--password', default='PASSWORD',
54                        help='REST password for source cluster or server node. Defaults to PASSWORD')
55    parser.add_option('-v', '--verbose', action='store_true',
56                        default=False, help='Enable verbose messaging')
57    parser.add_option('--path', default='.',
58                        help='Specify the path to cbbackup. Defaults to current directory')
59    parser.add_option('--port', default='11210',
60                        help='Specify the bucket port.  Defaults to 11210')
61    parser.add_option('-n', '--number', default='100',
62                        help='Specify the number of vbuckets per process. Defaults to 100')
63    parser.add_option('-x', '--extra', default=None,
64                        help="""Provide extra, uncommon config parameters;
65                        comma-separated key=val(,key=val)* pairs""")
66    try:
67        import pump_bfd2
68        parser.add_option("-m", "--mode",
69                        action="store", type="string", default="diff",
70                        help="backup mode: full, diff or accu [default:%default]")
71    except ImportError:
72        parser.add_option("-m", "--mode",
73                        action="store", type="string", default="full",
74                        help="backup mode: full")
75    options, rest = parser.parse_args()
76    if len(rest) != 2:
77        parser.print_help()
78        sys.exit("\nError: please provide both cluster IP and backup directory path.")
79
80    return options, rest[0], rest[1]
81
82def findAllVbucketsForBucket(node, bucket, path, port, restport, username, password, single_node):
83    localvbucketlist = []
84    request = urllib2.Request(
85        'http://' + node + ':' + restport + '/pools/default/buckets/' + bucket)
86    base64string = base64.encodestring(
87        '%s:%s' % (username, password)).replace('\n', '')
88    request.add_header('Authorization', 'Basic %s' % base64string)
89    try:
90        response = urllib2.urlopen(request)
91    except:
92        print('Authorization failed.  Please check username and password.')
93        exit(1)
94    data = json.loads(response.read())
95    vbucketserverdata = data['vBucketServerMap']
96    vbucketdata = vbucketserverdata['vBucketMap']
97    serverlist = vbucketserverdata['serverList']
98    # all possibles names / ipaddress for the node
99    aliases = []
100    # check to see if node was given as ip addess
101    matchObj = re.match(r'^\d+.\d+.\d+.\d+$', node, re.I)
102    if matchObj:
103        # node was entered as its IP address
104        nodeip = node
105        aliases.append(nodeip)
106        try:
107            (node, other_names, other_ips) = socket.gethostbyaddr(nodeip)
108            aliases.append(node)
109            aliases + other_names
110        except:
111            print("WARN: Could not find name for nodeip")
112    else:
113        aliases.append(node)
114        nodeip = socket.gethostbyname(node)
115        aliases.append(nodeip)
116
117    aliases = [alias + ":" + port for alias in aliases]
118
119    if args.verbose:
120        print("aliases list is ")
121        for x in aliases:
122            print(str(x))
123        print("server list is")
124        for x in serverlist:
125            print(str(x))
126
127    # find out the index in the serverlist for this node
128    serverindex = -1
129    for i in range(len(serverlist)):
130        for nodewithport in aliases:
131            if nodewithport == serverlist[i]:
132                serverindex = i
133    if serverindex == -1:
134        print serverindex
135        print 'Could not find node:port in server list.'
136        exit(1)
137
138    if single_node:
139        # iterate through all vbuckets and see which are active on this node
140        for i in range(len(vbucketdata)):
141            if vbucketdata[i][0] == serverindex:
142                vbucket = i
143                localvbucketlist.append(vbucket)
144    else:
145        # Just iterate through all vbuckets
146        for i in range(len(vbucketdata)):
147            vbucket = i
148            localvbucketlist.append(vbucket)
149
150    return localvbucketlist
151
152
153# Get the buckets that exist on the cluster
154def getBuckets(node, rest_port, username, password):
155    request = urllib2.Request(
156        'http://' + node + ':' + rest_port + '/pools/default/buckets')
157    base64string = base64.encodestring(
158        '%s:%s' % (username, password)).replace('\n', '')
159    request.add_header('Authorization', 'Basic %s' % base64string)
160    try:
161        response = urllib2.urlopen(request)
162    except:
163        print('Authorization failed.  Please check username and password.')
164        exit(1)
165    bucketsOnCluster = []
166    data = json.loads(response.read())
167    for item in data:
168        bucket = item['name']
169        bucketsOnCluster.append(bucket)
170    return bucketsOnCluster
171
172
173if __name__ == '__main__':
174    # Parse the arguments given.
175    args, cluster, backupDir = argumentParsing()
176
177    # Remove any white-spaces from start and end of strings
178    backupDir = backupDir.strip()
179    path = args.path.strip()
180
181    # Check to see if root backup directory exists
182    if not os.path.isdir(backupDir):
183        try:
184            os.makedirs(backupDir)
185        except:
186            exit("Cannot create backup root directory:%s" % backupDir)
187
188    # Check to see if path is correct
189    if not os.path.isdir(path):
190        print 'The path to cbbackup does not exist'
191        print 'Please run with a different path'
192        exit(1)
193    if not os.path.isfile(os.path.join(path, 'cbbackup')):
194        print 'cbbackup could not be found in ' + path
195        exit(1)
196
197    # Check to see if log directory exists if not create it
198    dir = os.path.join(backupDir, 'logs')
199    try:
200        os.stat(dir)
201    except:
202        try:
203            os.mkdir(dir)
204        except:
205            print('Error trying to create directory ' + dir)
206            exit(1)
207
208    # Separate out node and REST port
209    matchObj = re.match(r'^http://(.*):(\d+)$', cluster, re.I)
210    if matchObj:
211        node = matchObj.group(1)
212        rest = matchObj.group(2)
213    else:
214        print("Please enter the source as http://hostname:port")
215        print("For example http://localhost:8091 or http://127.0.0.1:8091")
216        exit(1)
217
218    # Check to see if backing-up all buckets or just a specified bucket
219    if args.bucket_source == '':
220        bucketList = getBuckets(
221            node, rest, args.username, args.password)
222    else:
223        # Check that the bucket exists
224        for item in getBuckets(node, rest, args.username, args.password):
225            if item == args.bucket_source:
226                bucketList.append(args.bucket_source)
227
228        if len(bucketList) == 0:
229            print 'Bucket ' + args.bucket_source + ' does not exist'
230            print 'Please enter a different bucket'
231            exit(1)
232
233    # For each bucket
234    for item in bucketList:
235        perbucketvbucketlist = findAllVbucketsForBucket(
236            node, item, path, args.port, rest, args.username, args.password, args.single_node)
237        for item in perbucketvbucketlist:
238            if item not in vbucketList:
239                vbucketList.append(item)
240
241    # Handle the case when path has spaces
242    # i.e. /Applications/Couchbase Server.app/Contents/...
243    if os.name == 'nt':
244        path = re.sub(r' ', '^ ', path)
245    else:
246        path = re.sub(r' ', '\ ', path)
247
248    # If a bucket was specfified then set-up the string to pass to cbbackup.
249    specific_bucket = ''
250    if len(bucketList) == 1:
251        specific_bucket = ' -b ' + bucketList[0]
252
253    extra_options = ''
254    if args.extra:
255        extra_options = ' -x ' + args.extra
256
257    mode_options = ''
258    if args.mode:
259        mode_options = ' -m ' + args.mode
260
261    # Group the number of vbuckets per process
262    for i in range(0, len(vbucketList), int(args.number)):
263        chunk = vbucketList[i:i + int(args.number)]
264        vbucketsname = str(chunk[0]) + '-' + str(chunk[-1])
265        command_line = os.path.join(path, 'cbbackup') + ' -v -t 1 --vbucket-list=' + ''.join(str(chunk).split()) + ' http://' \
266            + node + ':' + rest + ' ' + os.path.join(backupDir, vbucketsname) + ' -u ' + args.username \
267            + ' -p ' + args.password + extra_options + mode_options + specific_bucket + ' 2>' + \
268            os.path.join(backupDir, 'logs', vbucketsname) + '.err'
269        if args.verbose:
270            print command_line
271        p = subprocess.Popen(command_line, shell=True)
272        processes[p] = vbucketsname
273
274    # Did we backup anything?
275    if len(processes) == 0:
276        print 'Did not backup anything'
277        print 'Please check that you have the buckets on ' + args.node
278        exit(1)
279    else:
280        print 'Waiting for the backup to complete...'
281        successCount = 0
282        for p in processes:
283            p.wait()
284            if p.returncode == 1:
285                print 'Error with backup - look in ' + os.path.join(backupDir, 'logs', processes[p]) + '.err for details'
286            else:
287                successCount += 1
288
289        if successCount == len(processes):
290            print 'SUCCESSFULLY COMPLETED!'
291        else:
292            print 'ERROR!'
293            exit(1)
294