xref: /6.0.3/ns_server/cluster_run (revision 659a1cbd)
1#!/usr/bin/env python
2import os, os.path, subprocess, sys, signal, atexit, socket, getopt, select, shlex
3import platform, fnmatch
4
5base_direct_port = 12000
6base_api_port = 9000
7base_couch_port = 9500
8base_projector_port = 10000
9base_xdcr_port = 13000
10base_indexer_port = 9100
11base_fts_port = 9200
12base_eventing_port = 9300
13base_cbas_port = 9600
14
15LOGLEVELS = ["debug", "info", "warn", "error", "critical"]
16
17def read_configuration():
18    with open("build/cluster_run.configuration") as f:
19        def fn(line):
20            k, v = line.strip().split('=')
21            return k, shlex.split(v)[0]
22
23        return dict(fn(line) for line in f.readlines())
24
25config = read_configuration()
26PREFIX = config['prefix']
27
28def setup_path():
29    def ebin_search(path):
30        dirs = os.walk(path)
31        ebins = []
32
33        for d, _, _ in dirs:
34            if os.path.basename(d) == "ebin":
35                ebins.append(d)
36
37        return ebins
38
39    path = ebin_search(".")
40    couchpath = ebin_search("{0}/lib/couchdb/erlang/lib".format(PREFIX))
41    couch_plugins = ebin_search("{0}/lib/couchdb/plugins".format(PREFIX))
42
43    if len(couchpath) == 0:
44       sys.exit("Couch libs wasn't found.\nCan't handle it")
45
46    return couchpath + path + couch_plugins
47
48def mk_node_couch_config(i):
49    try:
50        os.mkdir("couch")
51    except os.error:
52        pass
53
54    with open("couch/n_{0}_conf.ini".format(i), "w") as f:
55        f.write("[httpd]\n")
56        f.write("port={0}\n".format(base_couch_port + i))
57        f.write("[couchdb]\n")
58        f.write("database_dir={0}/data/n_{1}/data\n".format(os.getcwd(), i))
59        f.write("view_index_dir={0}/data/n_{1}/data\n".format(os.getcwd(), i))
60        f.write("max_dbs_open=10000\n")
61        f.write("[upr]\n")
62        f.write("port={0}\n".format(base_direct_port + i * 2))
63        f.write("[dcp]\n")
64        f.write("port={0}\n".format(base_direct_port + i * 2))
65
66
67def couch_configs(i):
68    mk_node_couch_config(i)
69    return ["{0}/etc/couchdb/default.ini".format(PREFIX),
70            "{0}/etc/couchdb/default.d/capi.ini".format(PREFIX),
71            "{0}/etc/couchdb/default.d/geocouch.ini".format(PREFIX),
72            "couch/n_{0}_conf.ini".format(i)]
73
74def os_specific(args, params):
75    """Add os-specific junk to the cluster startup."""
76    if platform.system() == 'Windows':
77        args += ["dont_suppress_stderr_logger", "false"]
78    else:
79        args += ["dont_suppress_stderr_logger", "true"]
80    if platform.system() == 'Darwin':
81        import resource
82        ## OS X has a pretty tiny default fd limit.  Let's increase it (if it hasn't already been).
83        (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
84        if soft < 2048:
85            resource.setrlimit(resource.RLIMIT_NOFILE, (2048, 2048))
86        params['env'] = {"ERL_MAX_PORTS": "2048"}
87        params['env'].update(os.environ)
88
89ebin_path = None
90cluster_extra_args = None
91cluster_args_prefix = None
92
93def is_ipv6_setup():
94    return os.getenv("IPV6", "false") == "true"
95
96def prepare_start_cluster(extra_args, args_prefix):
97    global cluster_args_prefix
98    global cluster_extra_args
99    global ebin_path
100
101    ebin_path = setup_path()
102    cluster_extra_args = extra_args
103    cluster_args_prefix = args_prefix
104
105def quote_string_for_erl(s):
106    return '"' + s.replace("\\", "\\\\").replace("\"", "\\\"") + '"'
107
108def erlang_args_for_node(i):
109    logdir = "logs/n_{0}".format(i)
110
111    args = cluster_args_prefix + ["erl", "+MMmcs" "30",
112                                  "+A", "16", "+sbtu",
113                                  "+P", "327680", "-pa"] + ebin_path
114    args += [
115        "-setcookie", "nocookie",
116        "-kernel", "inet_dist_listen_min", "21100",
117        "inet_dist_listen_max", "21199",
118        "error_logger", "false",
119        "-sasl", "sasl_error_logger", "false",
120        "-couch_ini"] + couch_configs(i)
121
122    datadir = os.path.abspath('data/n_{0}'.format(i))
123    tempdir = os.path.abspath('tmp/')
124    nodefile = os.path.join(datadir, "nodefile")
125    babysitternodefile = os.path.join(datadir, "couchbase-server.babysitter.node")
126    babysittercookiefile = os.path.join(datadir, "couchbase-server.babysitter.cookie")
127
128    args += [
129        "-name", "babysitter_of_n_{0}@::1".format(i) if ipv6 else "babysitter_of_n_{0}@127.0.0.1".format(i),
130        "-proto_dist", "inet6_tcp" if ipv6 else "inet_tcp",
131        "-hidden",
132        "-kernel", "global_enable_tracing", "true",
133        "-ns_babysitter", "cookiefile", quote_string_for_erl(babysittercookiefile),
134        "-ns_babysitter", "nodefile", quote_string_for_erl(babysitternodefile),
135        "-ns_server", "config_path", '"etc/static_config.in"',
136        "-ns_server", "ipv6", "true" if ipv6 else "false",
137        "error_logger_mf_dir", quote_string_for_erl(logdir),
138        "path_config_etcdir", '"priv"',
139        "path_config_bindir", quote_string_for_erl(PREFIX+"/bin"),
140        "path_config_libdir", quote_string_for_erl(PREFIX+"/lib"),
141        "path_config_datadir", quote_string_for_erl(datadir),
142        "path_config_tmpdir", quote_string_for_erl(tempdir),
143        "path_config_secdir", quote_string_for_erl(PREFIX+"/etc/security"),
144        "path_audit_log", quote_string_for_erl(logdir),
145        "rest_port", str(base_api_port + i),
146        "query_port", str(base_couch_port - 1 - i),
147        "ssl_query_port", str(10000 + base_couch_port - 1 - i),
148        "projector_port", str(base_projector_port + i),
149        "ssl_rest_port", str(10000 + base_api_port + i),
150        "capi_port", str(base_couch_port + i),
151        "ssl_capi_port", str(10000 + base_couch_port + i),
152        "memcached_port", str(base_direct_port + i * 2),
153        "moxi_port", str(base_direct_port + i * 2 + 1),
154        "memcached_dedicated_port", str(base_direct_port - 1 - i * 4),
155        "ssl_proxy_downstream_port", str(base_direct_port - 2 - i * 4),
156        "ssl_proxy_upstream_port", str(base_direct_port - 3 - i * 4),
157        "memcached_ssl_port", str(base_direct_port - 4 - i * 4),
158        "nodefile", quote_string_for_erl(nodefile),
159        "short_name", quote_string_for_erl('n_{0}'.format(i)),
160        "xdcr_rest_port", str(base_xdcr_port + i),
161        "indexer_admin_port", str(base_indexer_port + i * 6),
162        "indexer_scan_port", str(base_indexer_port + i * 6 + 1),
163        "indexer_http_port", str(base_indexer_port + i * 6 + 2),
164        "indexer_https_port", str(10000 + base_indexer_port + i * 6 + 2),
165        "indexer_stinit_port", str(base_indexer_port + i * 6 + 3),
166        "indexer_stcatchup_port", str(base_indexer_port + i * 6 + 4),
167        "indexer_stmaint_port", str(base_indexer_port + i * 6 + 5),
168        "fts_http_port", str(base_fts_port + i),
169        "fts_ssl_port", str(10000 + base_fts_port + i),
170        "eventing_http_port", str(base_eventing_port + i),
171        "eventing_https_port", str(10000 + base_eventing_port + i),
172        "cbas_http_port", str(base_cbas_port + i * 12),
173        "cbas_cc_http_port", str(base_cbas_port + i * 12 + 1),
174        "cbas_cc_cluster_port", str(base_cbas_port + i * 12 + 2),
175        "cbas_cc_client_port", str(base_cbas_port + i * 12 + 3),
176        "cbas_hyracks_console_port", str(base_cbas_port + i * 12 + 4),
177        "cbas_cluster_port", str(base_cbas_port + i * 12 + 5),
178        "cbas_data_port", str(base_cbas_port + i * 12 + 6),
179        "cbas_result_port", str(base_cbas_port + i * 12 + 7),
180        "cbas_messaging_port", str(base_cbas_port + i * 12 + 8),
181        "cbas_debug_port", str(base_cbas_port + i * 12 + 9),
182        "cbas_auth_port", str(base_cbas_port + i * 12 + 10),
183        "cbas_admin_port", str(base_cbas_port + i * 12 + 11),
184        "cbas_ssl_port", str(10000 + base_cbas_port + i)
185        ] + cluster_extra_args
186
187    return args
188
189def start_cluster(num_nodes, start_index, host, extra_args, args_prefix):
190    prepare_start_cluster(extra_args, args_prefix)
191
192    def start_node(i):
193        logdir = "logs/n_{0}".format(i)
194        try:
195            os.makedirs(logdir)
196        except:
197            pass
198
199        args = erlang_args_for_node(i)
200
201        params = {}
202
203        os_specific(args, params)
204
205        if not params.has_key('env'):
206            params['env'] = {}
207            params['env'].update(os.environ)
208        path = params['env']['PATH']
209        path = (PREFIX+"/bin") + os.pathsep + path
210        if not params['env'].has_key('ERL_FULLSWEEP_AFTER'):
211            params['env']['ERL_FULLSWEEP_AFTER'] = '512'
212        params['env']['PATH'] = path
213
214        crash_dump_base = 'erl_crash.dump.n_%d' % i
215        params['env']['ERL_CRASH_DUMP_BASE'] = crash_dump_base
216        params['env']['ERL_CRASH_DUMP'] = crash_dump_base + '.babysitter'
217
218        params['env']['COUCHBASE_SMALLER_PKEYS'] = '1'
219
220        params['close_fds'] = True
221        if platform.system() == "Windows":
222            params['close_fds'] = False
223
224        w = None
225
226        if "-noinput" in args:
227            (r,w) = os.pipe()
228
229            params['stdin'] = r
230
231            if 'setpgrp' in os.__dict__ and params.get('close_fds'):
232                # this puts child out of our process group. So that
233                # Ctrl-C doesn't deliver SIGINT to it, leaving us
234                # ability to it shutdown carefully or otherwise
235                params['preexec_fn'] = os.setpgrp
236
237        pr = subprocess.Popen(args, **params)
238        if w != None:
239            os.close(r)
240        pr.write_side = w
241        return pr
242
243    return [start_node(i + start_index) for i in xrange(num_nodes)]
244
245def usage():
246    sys.exit("Usage: {0} [--nodes=N] [--dont-rename] [--dont-start] "
247             "[--interactive] [--static-cookie] [--start-index=N] "
248             "[--static-cookie] [--host=H] [--loglevel=L] "
249             "[--pluggable-config=File] [--minified] [--disable-autocomplete]"
250             "[ns_server args]".format(sys.argv[0]))
251
252def find_primary_addr():
253    Family = socket.AF_INET6 if ipv6 else socket.AF_INET
254    DnsAddr = "2001:4860:4860::8844" if ipv6 else "8.8.8.8"
255    s = socket.socket(Family, socket.SOCK_DGRAM)
256    try:
257        s.connect((DnsAddr, 53))
258        if ipv6:
259            addr, port, _, _ = s.getsockname()
260        else:
261            addr, port = s.getsockname()
262
263        return addr
264    except socket.error:
265        return None
266    finally:
267        s.close()
268
269def main():
270    try:
271        optlist, args = getopt.gnu_getopt(sys.argv[1:], "hn:i",
272                                          ["help", "start-index=", "nodes=",
273                                           "dont-rename", "interactive",
274                                           "static-cookie", "dont-start",
275                                           "host=", "loglevel=",
276                                           "prepend-extras", "pluggable-config=",
277                                           "minified", "disable-autocomplete"])
278    except getopt.GetoptError, err:
279        # print help information and exit:
280        print str(err) # will print something like "option -a not recognized"
281        usage()
282        sys.exit(2)
283
284    global ipv6
285    ipv6 = is_ipv6_setup()
286
287    dont_rename = False
288    dont_start = False
289    static_cookie = False
290    interactive_shell = False
291    start_index = 0
292    num_nodes = 1
293    prepend_extras = False
294    host = "127.0.0.1"
295    loglevel = 'debug'
296    pluggable_config = []
297    use_minified = False
298    disable_autocomplete = "{disable_autocomplete,false}"
299
300    for o, a in optlist:
301        if o in ("--nodes", "-n"):
302            num_nodes = int(a)
303        elif o in ("--interactive", "-i"):
304            interactive_shell = True
305        elif o == '--dont-start':
306            dont_start = True
307        elif o == '--host':
308            host = a
309        elif o == '--start-index':
310            start_index = int(a)
311        elif o == '--dont-rename':
312            dont_rename = True
313        elif o in ("--help", "-h"):
314            usage()
315            exit(0)
316        elif o in("--static-cookie"):
317            static_cookie = True
318        elif o == '--loglevel':
319            loglevel = a
320        elif o == "--prepend-extras":
321            prepend_extras = True
322        elif o == "--pluggable-config":
323            pluggable_config.append(a)
324        elif o == "--minified":
325            use_minified = True
326        elif o == "--disable-autocomplete":
327            disable_autocomplete = "{disable_autocomplete,true}"
328        else:
329            assert False, "unhandled options"
330
331    nodes = []
332    terminal_attrs = None
333
334    def kill_nodes(*args):
335        for n in nodes:
336            if n.write_side != None:
337                print("Closing %d\n" % n.write_side)
338                # os.write(n.write_side, "shutdown\n") # this line does graceful shutdown versus quick
339                os.close(n.write_side)
340            else:
341                try:
342                    n.kill()
343                except OSError:
344                    pass
345
346        for n in nodes:
347            n.wait()
348
349        if terminal_attrs != None:
350            termios.tcsetattr(sys.stdin, termios.TCSANOW, terminal_attrs)
351
352    atexit.register(kill_nodes)
353
354    try:
355        import termios
356        terminal_attrs = termios.tcgetattr(sys.stdin)
357    except:
358        pass
359
360    extra_args = []
361    if not dont_rename:
362        primary_addr = find_primary_addr()
363        if primary_addr == None:
364            print("was unable to detect 'internet' address of this machine."
365                  + " node rename will be disabled")
366        else:
367            extra_args += ["rename_ip", '"' + primary_addr + '"']
368
369    extra_args += args[1:]
370    if prepend_extras:
371        prepend_args = args[0:]
372    else:
373        prepend_args = []
374        extra_args += args[0:]
375
376    if static_cookie:
377        extra_args += ["-ns_server", "dont_reset_cookie", "true"]
378
379    if not interactive_shell:
380        extra_args += ["-noinput"]
381        if not dont_start:
382            extra_args += ["-run", "child_erlang", "child_start", "ns_babysitter_bootstrap"]
383            extra_args += ["-ns_babysitter", "handle_ctrl_c", "true"]
384    else:
385        if not dont_start:
386            extra_args += ["-run", "ns_babysitter_bootstrap"]
387
388    if loglevel not in LOGLEVELS:
389        print "Valid log levels are the following: %s" % ', '.join(LOGLEVELS)
390        sys.exit(1)
391    extra_args += ["-ns_server", "loglevel_stderr", loglevel]
392
393    plugins_dir = '../build/cluster_run_ui_plugins'
394    if os.path.isdir(plugins_dir):
395        for file in os.listdir(plugins_dir):
396            if fnmatch.fnmatch(file, 'pluggable-ui-*.cluster_run.json'):
397                pluggable_config.append(os.path.join(plugins_dir, file))
398
399    if pluggable_config:
400        extra_args += ["-ns_server", "ui_plugins",
401                        quote_string_for_erl(','.join(pluggable_config))]
402
403    ui_env = [disable_autocomplete]
404
405    extra_args += ["-ns_server", "use_minified", "true" if use_minified else "false"]
406    extra_args += ["-ns_server", "ui_env", '[' + ','.join(ui_env) + ']']
407
408    nodes = start_cluster(num_nodes, start_index, host, extra_args, prepend_args)
409
410    for node in nodes:
411        node.wait()
412
413
414if __name__ == '__main__':
415    main()
416