xref: /6.0.3/ns_server/cluster_run (revision 4abb10e8)
1#!/usr/bin/env python
2import os, os.path, subprocess, sys, signal, atexit, socket, getopt, select, shlex
3import platform, fnmatch
4
5base_direct_port = 12000
6base_api_port = 9000
7base_couch_port = 9500
8base_projector_port = 10000
9base_xdcr_port = 13000
10base_indexer_port = 9100
11base_fts_port = 9200
12
13LOGLEVELS = ["debug", "info", "warn", "error", "critical"]
14
15def read_configuration():
16    with open("build/cluster_run.configuration") as f:
17        def fn(line):
18            k, v = line.strip().split('=')
19            return k, shlex.split(v)[0]
20
21        return dict(fn(line) for line in f.readlines())
22
23config = read_configuration()
24PREFIX = config['prefix']
25
26def setup_path():
27    def ebin_search(path):
28        dirs = os.walk(path)
29        ebins = []
30
31        for d, _, _ in dirs:
32            if os.path.basename(d) == "ebin":
33                ebins.append(d)
34
35        return ebins
36
37    path = ebin_search(".")
38    couchpath = ebin_search("{0}/lib/couchdb/erlang/lib".format(PREFIX))
39    couch_plugins = ebin_search("{0}/lib/couchdb/plugins".format(PREFIX))
40
41    if len(couchpath) == 0:
42       sys.exit("Couch libs wasn't found.\nCan't handle it")
43
44    return couchpath + path + couch_plugins
45
46def mk_node_couch_config(i):
47    try:
48        os.mkdir("couch")
49    except os.error:
50        pass
51
52    with open("couch/n_{0}_conf.ini".format(i), "w") as f:
53        f.write("[httpd]\n")
54        f.write("port={0}\n".format(base_couch_port + i))
55        f.write("[couchdb]\n")
56        f.write("database_dir={0}/data/n_{1}/data\n".format(os.getcwd(), i))
57        f.write("view_index_dir={0}/data/n_{1}/data\n".format(os.getcwd(), i))
58        f.write("max_dbs_open=10000\n")
59        f.write("[upr]\n")
60        f.write("port={0}\n".format(base_direct_port + i * 2))
61        f.write("[dcp]\n")
62        f.write("port={0}\n".format(base_direct_port + i * 2))
63
64
65def couch_configs(i):
66    mk_node_couch_config(i)
67    return ["{0}/etc/couchdb/default.ini".format(PREFIX),
68            "{0}/etc/couchdb/default.d/capi.ini".format(PREFIX),
69            "{0}/etc/couchdb/default.d/geocouch.ini".format(PREFIX),
70            "couch/n_{0}_conf.ini".format(i)]
71
72def os_specific(args, params):
73    """Add os-specific junk to the cluster startup."""
74    if platform.system() == 'Windows':
75        args += ["dont_suppress_stderr_logger", "false"]
76    else:
77        args += ["dont_suppress_stderr_logger", "true"]
78    if platform.system() == 'Darwin':
79        import resource
80        ## OS X has a pretty tiny default fd limit.  Let's increase it (if it hasn't already been).
81        (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
82        if soft < 2048:
83            resource.setrlimit(resource.RLIMIT_NOFILE, (2048, 2048))
84        params['env'] = {"ERL_MAX_PORTS": "2048"}
85        params['env'].update(os.environ)
86
87ebin_path = None
88cluster_extra_args = None
89cluster_args_prefix = None
90
91def prepare_start_cluster(extra_args, args_prefix):
92    global cluster_args_prefix
93    global cluster_extra_args
94    global ebin_path
95
96    ebin_path = setup_path()
97    cluster_extra_args = extra_args
98    cluster_args_prefix = args_prefix
99
100def quote_string_for_erl(s):
101    return '"' + s.replace("\\", "\\\\").replace("\"", "\\\"") + '"'
102
103def erlang_args_for_node(i):
104    logdir = "logs/n_{0}".format(i)
105
106    args = cluster_args_prefix + ["erl", "+MMmcs" "30",
107                                  "+A", "16", "+sbtu",
108                                  "+P", "327680", "-pa"] + ebin_path
109    args += [
110        "-setcookie", "nocookie",
111        "-kernel", "inet_dist_listen_min", "21100",
112        "inet_dist_listen_max", "21199",
113        "error_logger", "false",
114        "-sasl", "sasl_error_logger", "false",
115        "-couch_ini"] + couch_configs(i)
116
117    datadir = os.path.abspath('data/n_{0}'.format(i))
118    tempdir = os.path.abspath('tmp/')
119    nodefile = os.path.join(datadir, "nodefile")
120    babysitternodefile = os.path.join(datadir, "couchbase-server.babysitter.node")
121    babysittercookiefile = os.path.join(datadir, "couchbase-server.cookie")
122
123    args += [
124        "-name", "babysitter_of_n_{0}@127.0.0.1".format(i),
125        "-hidden",
126        "-kernel", "global_enable_tracing", "true",
127        "-ns_babysitter", "cookiefile", quote_string_for_erl(babysittercookiefile),
128        "-ns_babysitter", "nodefile", quote_string_for_erl(babysitternodefile),
129        "-ns_server", "config_path", '"etc/static_config.in"',
130        "-ns_server", "enable_mlockall", "false",
131        "error_logger_mf_dir", quote_string_for_erl(logdir),
132        "path_config_etcdir", '"priv"',
133        "path_config_bindir", quote_string_for_erl(PREFIX+"/bin"),
134        "path_config_libdir", quote_string_for_erl(PREFIX+"/lib"),
135        "path_config_datadir", quote_string_for_erl(datadir),
136        "path_config_tmpdir", quote_string_for_erl(tempdir),
137        "path_config_secdir", quote_string_for_erl(PREFIX+"/etc/security"),
138        "path_audit_log", quote_string_for_erl(logdir),
139        "rest_port", str(base_api_port + i),
140        "query_port", str(base_couch_port - 1 - i),
141        "ssl_query_port", str(10000 + base_couch_port - 1 - i),
142        "projector_port", str(base_projector_port + i),
143        "ssl_rest_port", str(10000 + base_api_port + i),
144        "capi_port", str(base_couch_port + i),
145        "ssl_capi_port", str(10000 + base_couch_port + i),
146        "memcached_port", str(base_direct_port + i * 2),
147        "moxi_port", str(base_direct_port + i * 2 + 1),
148        "memcached_dedicated_port", str(base_direct_port - 1 - i * 4),
149        "ssl_proxy_downstream_port", str(base_direct_port - 2 - i * 4),
150        "ssl_proxy_upstream_port", str(base_direct_port - 3 - i * 4),
151        "memcached_ssl_port", str(base_direct_port - 4 - i * 4),
152        "nodefile", quote_string_for_erl(nodefile),
153        "short_name", quote_string_for_erl('n_{0}'.format(i)),
154        "xdcr_rest_port", str(base_xdcr_port + i),
155        "indexer_admin_port", str(base_indexer_port + i * 6),
156        "indexer_scan_port", str(base_indexer_port + i * 6 + 1),
157        "indexer_http_port", str(base_indexer_port + i * 6 + 2),
158        "indexer_stinit_port", str(base_indexer_port + i * 6 + 3),
159        "indexer_stcatchup_port", str(base_indexer_port + i * 6 + 4),
160        "indexer_stmaint_port", str(base_indexer_port + i * 6 + 5),
161        "fts_http_port", str(base_fts_port + i),
162        "fts_ssl_port", str(10000 + base_fts_port + i)
163        ] + cluster_extra_args
164
165    return args
166
167def start_cluster(num_nodes, start_index, host, extra_args, args_prefix):
168    prepare_start_cluster(extra_args, args_prefix)
169
170    def start_node(i):
171        logdir = "logs/n_{0}".format(i)
172        try:
173            os.makedirs(logdir)
174        except:
175            pass
176
177        args = erlang_args_for_node(i)
178
179        params = {}
180
181        os_specific(args, params)
182
183        if not params.has_key('env'):
184            params['env'] = {}
185            params['env'].update(os.environ)
186        path = params['env']['PATH']
187        path = (PREFIX+"/bin") + os.pathsep + path
188        if not params['env'].has_key('ERL_FULLSWEEP_AFTER'):
189            params['env']['ERL_FULLSWEEP_AFTER'] = '512'
190        params['env']['PATH'] = path
191
192        crash_dump_base = 'erl_crash.dump.n_%d' % i
193        params['env']['ERL_CRASH_DUMP_BASE'] = crash_dump_base
194        params['env']['ERL_CRASH_DUMP'] = crash_dump_base + '.babysitter'
195
196        params['env']['COUCHBASE_SMALLER_PKEYS'] = '1'
197
198        params['close_fds'] = True
199        if platform.system() == "Windows":
200            params['close_fds'] = False
201
202        w = None
203
204        if "-noinput" in args:
205            (r,w) = os.pipe()
206
207            params['stdin'] = r
208
209            if 'setpgrp' in os.__dict__ and params.get('close_fds'):
210                # this puts child out of our process group. So that
211                # Ctrl-C doesn't deliver SIGINT to it, leaving us
212                # ability to it shutdown carefully or otherwise
213                params['preexec_fn'] = os.setpgrp
214
215        pr = subprocess.Popen(args, **params)
216        if w != None:
217            os.close(r)
218        pr.write_side = w
219        return pr
220
221    return [start_node(i + start_index) for i in xrange(num_nodes)]
222
223def usage():
224    sys.exit("Usage: {0} [--nodes=N] [--dont-rename] [--dont-start] "
225             "[--interactive] [--static-cookie] [--start-index=N] "
226             "[--static-cookie] [--host=H] [--loglevel=L] "
227             "[--pluggable-config=File] [--minified] [--disable-autocomplete]"
228             "[ns_server args]".format(sys.argv[0]))
229
230def find_primary_addr():
231    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
232    try:
233        s.connect(("8.8.8.8", 56))
234        addr, port = s.getsockname()
235        return addr
236    except socket.error:
237        return None
238    finally:
239        s.close()
240
241def main():
242    try:
243        optlist, args = getopt.gnu_getopt(sys.argv[1:], "hn:i",
244                                          ["help", "start-index=", "nodes=",
245                                           "dont-rename", "interactive",
246                                           "static-cookie", "dont-start",
247                                           "host=", "loglevel=",
248                                           "prepend-extras", "pluggable-config=",
249                                           "minified", "disable-autocomplete"])
250    except getopt.GetoptError, err:
251        # print help information and exit:
252        print str(err) # will print something like "option -a not recognized"
253        usage()
254        sys.exit(2)
255
256    dont_rename = False
257    dont_start = False
258    static_cookie = False
259    interactive_shell = False
260    start_index = 0
261    num_nodes = 1
262    prepend_extras = False
263    host = "127.0.0.1"
264    loglevel = 'debug'
265    pluggable_config = []
266    use_minified = False
267    disable_autocomplete = "{disable_autocomplete,false}"
268
269    for o, a in optlist:
270        if o in ("--nodes", "-n"):
271            num_nodes = int(a)
272        elif o in ("--interactive", "-i"):
273            interactive_shell = True
274        elif o == '--dont-start':
275            dont_start = True
276        elif o == '--host':
277            host = a
278        elif o == '--start-index':
279            start_index = int(a)
280        elif o == '--dont-rename':
281            dont_rename = True
282        elif o in ("--help", "-h"):
283            usage()
284            exit(0)
285        elif o in("--static-cookie"):
286            static_cookie = True
287        elif o == '--loglevel':
288            loglevel = a
289        elif o == "--prepend-extras":
290            prepend_extras = True
291        elif o == "--pluggable-config":
292            pluggable_config.append(a)
293        elif o == "--minified":
294            use_minified = True
295        elif o == "--disable-autocomplete":
296            disable_autocomplete = "{disable_autocomplete,true}"
297        else:
298            assert False, "unhandled options"
299
300    nodes = []
301    terminal_attrs = None
302
303    def kill_nodes(*args):
304        for n in nodes:
305            if n.write_side != None:
306                print("Closing %d\n" % n.write_side)
307                # os.write(n.write_side, "shutdown\n") # this line does graceful shutdown versus quick
308                os.close(n.write_side)
309            else:
310                try:
311                    n.kill()
312                except OSError:
313                    pass
314
315        for n in nodes:
316            n.wait()
317
318        if terminal_attrs != None:
319            termios.tcsetattr(sys.stdin, termios.TCSANOW, terminal_attrs)
320
321    atexit.register(kill_nodes)
322
323    try:
324        import termios
325        terminal_attrs = termios.tcgetattr(sys.stdin)
326    except:
327        pass
328
329    extra_args = []
330    if not dont_rename:
331        primary_addr = find_primary_addr()
332        if primary_addr == None:
333            print("was unable to detect 'internet' address of this machine."
334                  + " node rename will be disabled")
335        else:
336            extra_args += ["rename_ip", '"' + primary_addr + '"']
337
338    extra_args += args[1:]
339    if prepend_extras:
340        prepend_args = args[0:]
341    else:
342        prepend_args = []
343        extra_args += args[0:]
344
345    if static_cookie:
346        extra_args += ["-ns_server", "dont_reset_cookie", "true"]
347
348    if not interactive_shell:
349        extra_args += ["-noinput"]
350        if not dont_start:
351            extra_args += ["-run", "child_erlang", "child_start", "ns_babysitter_bootstrap"]
352            extra_args += ["-ns_babysitter", "handle_ctrl_c", "true"]
353    else:
354        if not dont_start:
355            extra_args += ["-run", "ns_babysitter_bootstrap"]
356
357    if loglevel not in LOGLEVELS:
358        print "Valid log levels are the following: %s" % ', '.join(LOGLEVELS)
359        sys.exit(1)
360    extra_args += ["-ns_server", "loglevel_stderr", loglevel]
361
362    plugins_dir = '../build/cluster_run_ui_plugins'
363    if os.path.isdir(plugins_dir):
364        for file in os.listdir(plugins_dir):
365            if fnmatch.fnmatch(file, 'pluggable-ui-*.cluster_run.json'):
366                pluggable_config.append(os.path.join(plugins_dir, file))
367
368    if pluggable_config:
369        extra_args += ["-ns_server", "ui_plugins",
370                        quote_string_for_erl(','.join(pluggable_config))]
371
372    ui_env = [disable_autocomplete]
373
374    extra_args += ["-ns_server", "use_minified", "true" if use_minified else "false"]
375    extra_args += ["-ns_server", "ui_env", '[' + ','.join(ui_env) + ']']
376
377    nodes = start_cluster(num_nodes, start_index, host, extra_args, prepend_args)
378
379    for node in nodes:
380        node.wait()
381
382
383if __name__ == '__main__':
384    main()
385