xref: /6.0.3/ns_server/cluster_run (revision cf0a8817)
1#!/usr/bin/env python
2import os, os.path, subprocess, sys, signal, atexit, socket, getopt, select, shlex
3import platform, fnmatch
4
5base_direct_port = 12000
6base_api_port = 9000
7base_couch_port = 9500
8base_projector_port = 10000
9base_xdcr_port = 13000
10base_indexer_port = 9100
11base_fts_port = 9200
12base_eventing_port = 9300
13
14LOGLEVELS = ["debug", "info", "warn", "error", "critical"]
15
16def read_configuration():
17    with open("build/cluster_run.configuration") as f:
18        def fn(line):
19            k, v = line.strip().split('=')
20            return k, shlex.split(v)[0]
21
22        return dict(fn(line) for line in f.readlines())
23
24config = read_configuration()
25PREFIX = config['prefix']
26
27def setup_path():
28    def ebin_search(path):
29        dirs = os.walk(path)
30        ebins = []
31
32        for d, _, _ in dirs:
33            if os.path.basename(d) == "ebin":
34                ebins.append(d)
35
36        return ebins
37
38    path = ebin_search(".")
39    couchpath = ebin_search("{0}/lib/couchdb/erlang/lib".format(PREFIX))
40    couch_plugins = ebin_search("{0}/lib/couchdb/plugins".format(PREFIX))
41
42    if len(couchpath) == 0:
43       sys.exit("Couch libs wasn't found.\nCan't handle it")
44
45    return couchpath + path + couch_plugins
46
47def mk_node_couch_config(i):
48    try:
49        os.mkdir("couch")
50    except os.error:
51        pass
52
53    with open("couch/n_{0}_conf.ini".format(i), "w") as f:
54        f.write("[httpd]\n")
55        f.write("port={0}\n".format(base_couch_port + i))
56        f.write("[couchdb]\n")
57        f.write("database_dir={0}/data/n_{1}/data\n".format(os.getcwd(), i))
58        f.write("view_index_dir={0}/data/n_{1}/data\n".format(os.getcwd(), i))
59        f.write("max_dbs_open=10000\n")
60        f.write("[upr]\n")
61        f.write("port={0}\n".format(base_direct_port + i * 2))
62        f.write("[dcp]\n")
63        f.write("port={0}\n".format(base_direct_port + i * 2))
64
65
66def couch_configs(i):
67    mk_node_couch_config(i)
68    return ["{0}/etc/couchdb/default.ini".format(PREFIX),
69            "{0}/etc/couchdb/default.d/capi.ini".format(PREFIX),
70            "{0}/etc/couchdb/default.d/geocouch.ini".format(PREFIX),
71            "couch/n_{0}_conf.ini".format(i)]
72
73def os_specific(args, params):
74    """Add os-specific junk to the cluster startup."""
75    if platform.system() == 'Windows':
76        args += ["dont_suppress_stderr_logger", "false"]
77    else:
78        args += ["dont_suppress_stderr_logger", "true"]
79    if platform.system() == 'Darwin':
80        import resource
81        ## OS X has a pretty tiny default fd limit.  Let's increase it (if it hasn't already been).
82        (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
83        if soft < 2048:
84            resource.setrlimit(resource.RLIMIT_NOFILE, (2048, 2048))
85        params['env'] = {"ERL_MAX_PORTS": "2048"}
86        params['env'].update(os.environ)
87
88ebin_path = None
89cluster_extra_args = None
90cluster_args_prefix = None
91
92def is_ipv6_setup():
93    return os.getenv("IPV6", "false") == "true"
94
95def prepare_start_cluster(extra_args, args_prefix):
96    global cluster_args_prefix
97    global cluster_extra_args
98    global ebin_path
99
100    ebin_path = setup_path()
101    cluster_extra_args = extra_args
102    cluster_args_prefix = args_prefix
103
104def quote_string_for_erl(s):
105    return '"' + s.replace("\\", "\\\\").replace("\"", "\\\"") + '"'
106
107def erlang_args_for_node(i):
108    logdir = "logs/n_{0}".format(i)
109
110    args = cluster_args_prefix + ["erl", "+MMmcs" "30",
111                                  "+A", "16", "+sbtu",
112                                  "+P", "327680", "-pa"] + ebin_path
113    args += [
114        "-setcookie", "nocookie",
115        "-kernel", "inet_dist_listen_min", "21100",
116        "inet_dist_listen_max", "21199",
117        "error_logger", "false",
118        "-sasl", "sasl_error_logger", "false",
119        "-couch_ini"] + couch_configs(i)
120
121    datadir = os.path.abspath('data/n_{0}'.format(i))
122    tempdir = os.path.abspath('tmp/')
123    nodefile = os.path.join(datadir, "nodefile")
124    babysitternodefile = os.path.join(datadir, "couchbase-server.babysitter.node")
125    babysittercookiefile = os.path.join(datadir, "couchbase-server.babysitter.cookie")
126
127    args += [
128        "-name", "babysitter_of_n_{0}@::1".format(i) if ipv6 else "babysitter_of_n_{0}@127.0.0.1".format(i),
129        "-proto_dist", "inet6_tcp" if ipv6 else "inet_tcp",
130        "-hidden",
131        "-kernel", "global_enable_tracing", "true",
132        "-ns_babysitter", "cookiefile", quote_string_for_erl(babysittercookiefile),
133        "-ns_babysitter", "nodefile", quote_string_for_erl(babysitternodefile),
134        "-ns_server", "config_path", '"etc/static_config.in"',
135        "-ns_server", "ipv6", "true" if ipv6 else "false",
136        "error_logger_mf_dir", quote_string_for_erl(logdir),
137        "path_config_etcdir", '"priv"',
138        "path_config_bindir", quote_string_for_erl(PREFIX+"/bin"),
139        "path_config_libdir", quote_string_for_erl(PREFIX+"/lib"),
140        "path_config_datadir", quote_string_for_erl(datadir),
141        "path_config_tmpdir", quote_string_for_erl(tempdir),
142        "path_config_secdir", quote_string_for_erl(PREFIX+"/etc/security"),
143        "path_audit_log", quote_string_for_erl(logdir),
144        "rest_port", str(base_api_port + i),
145        "query_port", str(base_couch_port - 1 - i),
146        "ssl_query_port", str(10000 + base_couch_port - 1 - i),
147        "projector_port", str(base_projector_port + i),
148        "ssl_rest_port", str(10000 + base_api_port + i),
149        "capi_port", str(base_couch_port + i),
150        "ssl_capi_port", str(10000 + base_couch_port + i),
151        "memcached_port", str(base_direct_port + i * 2),
152        "moxi_port", str(base_direct_port + i * 2 + 1),
153        "memcached_dedicated_port", str(base_direct_port - 1 - i * 4),
154        "ssl_proxy_downstream_port", str(base_direct_port - 2 - i * 4),
155        "ssl_proxy_upstream_port", str(base_direct_port - 3 - i * 4),
156        "memcached_ssl_port", str(base_direct_port - 4 - i * 4),
157        "nodefile", quote_string_for_erl(nodefile),
158        "short_name", quote_string_for_erl('n_{0}'.format(i)),
159        "xdcr_rest_port", str(base_xdcr_port + i),
160        "indexer_admin_port", str(base_indexer_port + i * 6),
161        "indexer_scan_port", str(base_indexer_port + i * 6 + 1),
162        "indexer_http_port", str(base_indexer_port + i * 6 + 2),
163        "indexer_https_port", str(10000 + base_indexer_port + i * 6 + 2),
164        "indexer_stinit_port", str(base_indexer_port + i * 6 + 3),
165        "indexer_stcatchup_port", str(base_indexer_port + i * 6 + 4),
166        "indexer_stmaint_port", str(base_indexer_port + i * 6 + 5),
167        "fts_http_port", str(base_fts_port + i),
168        "fts_ssl_port", str(10000 + base_fts_port + i),
169        "eventing_http_port", str(base_eventing_port + i),
170        ] + cluster_extra_args
171
172    return args
173
174def start_cluster(num_nodes, start_index, host, extra_args, args_prefix):
175    prepare_start_cluster(extra_args, args_prefix)
176
177    def start_node(i):
178        logdir = "logs/n_{0}".format(i)
179        try:
180            os.makedirs(logdir)
181        except:
182            pass
183
184        args = erlang_args_for_node(i)
185
186        params = {}
187
188        os_specific(args, params)
189
190        if not params.has_key('env'):
191            params['env'] = {}
192            params['env'].update(os.environ)
193        path = params['env']['PATH']
194        path = (PREFIX+"/bin") + os.pathsep + path
195        if not params['env'].has_key('ERL_FULLSWEEP_AFTER'):
196            params['env']['ERL_FULLSWEEP_AFTER'] = '512'
197        params['env']['PATH'] = path
198
199        crash_dump_base = 'erl_crash.dump.n_%d' % i
200        params['env']['ERL_CRASH_DUMP_BASE'] = crash_dump_base
201        params['env']['ERL_CRASH_DUMP'] = crash_dump_base + '.babysitter'
202
203        params['env']['COUCHBASE_SMALLER_PKEYS'] = '1'
204
205        params['close_fds'] = True
206        if platform.system() == "Windows":
207            params['close_fds'] = False
208
209        w = None
210
211        if "-noinput" in args:
212            (r,w) = os.pipe()
213
214            params['stdin'] = r
215
216            if 'setpgrp' in os.__dict__ and params.get('close_fds'):
217                # this puts child out of our process group. So that
218                # Ctrl-C doesn't deliver SIGINT to it, leaving us
219                # ability to it shutdown carefully or otherwise
220                params['preexec_fn'] = os.setpgrp
221
222        pr = subprocess.Popen(args, **params)
223        if w != None:
224            os.close(r)
225        pr.write_side = w
226        return pr
227
228    return [start_node(i + start_index) for i in xrange(num_nodes)]
229
230def usage():
231    sys.exit("Usage: {0} [--nodes=N] [--dont-rename] [--dont-start] "
232             "[--interactive] [--static-cookie] [--start-index=N] "
233             "[--static-cookie] [--host=H] [--loglevel=L] "
234             "[--pluggable-config=File] [--minified] [--disable-autocomplete]"
235             "[ns_server args]".format(sys.argv[0]))
236
237def find_primary_addr():
238    Family = socket.AF_INET6 if ipv6 else socket.AF_INET
239    DnsAddr = "2001:4860:4860::8844" if ipv6 else "8.8.8.8"
240    s = socket.socket(Family, socket.SOCK_DGRAM)
241    try:
242        s.connect((DnsAddr, 53))
243        if ipv6:
244            addr, port, _, _ = s.getsockname()
245        else:
246            addr, port = s.getsockname()
247
248        return addr
249    except socket.error:
250        return None
251    finally:
252        s.close()
253
254def main():
255    try:
256        optlist, args = getopt.gnu_getopt(sys.argv[1:], "hn:i",
257                                          ["help", "start-index=", "nodes=",
258                                           "dont-rename", "interactive",
259                                           "static-cookie", "dont-start",
260                                           "host=", "loglevel=",
261                                           "prepend-extras", "pluggable-config=",
262                                           "minified", "disable-autocomplete"])
263    except getopt.GetoptError, err:
264        # print help information and exit:
265        print str(err) # will print something like "option -a not recognized"
266        usage()
267        sys.exit(2)
268
269    global ipv6
270    ipv6 = is_ipv6_setup()
271
272    dont_rename = False
273    dont_start = False
274    static_cookie = False
275    interactive_shell = False
276    start_index = 0
277    num_nodes = 1
278    prepend_extras = False
279    host = "127.0.0.1"
280    loglevel = 'debug'
281    pluggable_config = []
282    use_minified = False
283    disable_autocomplete = "{disable_autocomplete,false}"
284
285    for o, a in optlist:
286        if o in ("--nodes", "-n"):
287            num_nodes = int(a)
288        elif o in ("--interactive", "-i"):
289            interactive_shell = True
290        elif o == '--dont-start':
291            dont_start = True
292        elif o == '--host':
293            host = a
294        elif o == '--start-index':
295            start_index = int(a)
296        elif o == '--dont-rename':
297            dont_rename = True
298        elif o in ("--help", "-h"):
299            usage()
300            exit(0)
301        elif o in("--static-cookie"):
302            static_cookie = True
303        elif o == '--loglevel':
304            loglevel = a
305        elif o == "--prepend-extras":
306            prepend_extras = True
307        elif o == "--pluggable-config":
308            pluggable_config.append(a)
309        elif o == "--minified":
310            use_minified = True
311        elif o == "--disable-autocomplete":
312            disable_autocomplete = "{disable_autocomplete,true}"
313        else:
314            assert False, "unhandled options"
315
316    nodes = []
317    terminal_attrs = None
318
319    def kill_nodes(*args):
320        for n in nodes:
321            if n.write_side != None:
322                print("Closing %d\n" % n.write_side)
323                # os.write(n.write_side, "shutdown\n") # this line does graceful shutdown versus quick
324                os.close(n.write_side)
325            else:
326                try:
327                    n.kill()
328                except OSError:
329                    pass
330
331        for n in nodes:
332            n.wait()
333
334        if terminal_attrs != None:
335            termios.tcsetattr(sys.stdin, termios.TCSANOW, terminal_attrs)
336
337    atexit.register(kill_nodes)
338
339    try:
340        import termios
341        terminal_attrs = termios.tcgetattr(sys.stdin)
342    except:
343        pass
344
345    extra_args = []
346    if not dont_rename:
347        primary_addr = find_primary_addr()
348        if primary_addr == None:
349            print("was unable to detect 'internet' address of this machine."
350                  + " node rename will be disabled")
351        else:
352            extra_args += ["rename_ip", '"' + primary_addr + '"']
353
354    extra_args += args[1:]
355    if prepend_extras:
356        prepend_args = args[0:]
357    else:
358        prepend_args = []
359        extra_args += args[0:]
360
361    if static_cookie:
362        extra_args += ["-ns_server", "dont_reset_cookie", "true"]
363
364    if not interactive_shell:
365        extra_args += ["-noinput"]
366        if not dont_start:
367            extra_args += ["-run", "child_erlang", "child_start", "ns_babysitter_bootstrap"]
368            extra_args += ["-ns_babysitter", "handle_ctrl_c", "true"]
369    else:
370        if not dont_start:
371            extra_args += ["-run", "ns_babysitter_bootstrap"]
372
373    if loglevel not in LOGLEVELS:
374        print "Valid log levels are the following: %s" % ', '.join(LOGLEVELS)
375        sys.exit(1)
376    extra_args += ["-ns_server", "loglevel_stderr", loglevel]
377
378    plugins_dir = '../build/cluster_run_ui_plugins'
379    if os.path.isdir(plugins_dir):
380        for file in os.listdir(plugins_dir):
381            if fnmatch.fnmatch(file, 'pluggable-ui-*.cluster_run.json'):
382                pluggable_config.append(os.path.join(plugins_dir, file))
383
384    if pluggable_config:
385        extra_args += ["-ns_server", "ui_plugins",
386                        quote_string_for_erl(','.join(pluggable_config))]
387
388    ui_env = [disable_autocomplete]
389
390    extra_args += ["-ns_server", "use_minified", "true" if use_minified else "false"]
391    extra_args += ["-ns_server", "ui_env", '[' + ','.join(ui_env) + ']']
392
393    nodes = start_cluster(num_nodes, start_index, host, extra_args, prepend_args)
394
395    for node in nodes:
396        node.wait()
397
398
399if __name__ == '__main__':
400    main()
401