xref: /6.0.3/ns_server/cluster_run (revision 08a24925)
1#!/usr/bin/env python
2import os, os.path, subprocess, sys, signal, atexit, socket, getopt, select, shlex
3import platform, fnmatch
4
5base_direct_port = 12000
6base_api_port = 9000
7base_couch_port = 9500
8base_projector_port = 10000
9base_xdcr_port = 13000
10base_indexer_port = 9100
11base_fts_port = 9200
12
13LOGLEVELS = ["debug", "info", "warn", "error", "critical"]
14
15def read_configuration():
16    with open("build/cluster_run.configuration") as f:
17        def fn(line):
18            k, v = line.strip().split('=')
19            return k, shlex.split(v)[0]
20
21        return dict(fn(line) for line in f.readlines())
22
23config = read_configuration()
24PREFIX = config['prefix']
25
26def setup_path():
27    def ebin_search(path):
28        dirs = os.walk(path)
29        ebins = []
30
31        for d, _, _ in dirs:
32            if os.path.basename(d) == "ebin":
33                ebins.append(d)
34
35        return ebins
36
37    path = ebin_search(".")
38    couchpath = ebin_search("{0}/lib/couchdb/erlang/lib".format(PREFIX))
39    couch_plugins = ebin_search("{0}/lib/couchdb/plugins".format(PREFIX))
40
41    if len(couchpath) == 0:
42       sys.exit("Couch libs wasn't found.\nCan't handle it")
43
44    return couchpath + path + couch_plugins
45
46def mk_node_couch_config(i):
47    try:
48        os.mkdir("couch")
49    except os.error:
50        pass
51
52    with open("couch/n_{0}_conf.ini".format(i), "w") as f:
53        f.write("[httpd]\n")
54        f.write("port={0}\n".format(base_couch_port + i))
55        f.write("[couchdb]\n")
56        f.write("database_dir={0}/data/n_{1}/data\n".format(os.getcwd(), i))
57        f.write("view_index_dir={0}/data/n_{1}/data\n".format(os.getcwd(), i))
58        f.write("max_dbs_open=10000\n")
59        f.write("[upr]\n")
60        f.write("port={0}\n".format(base_direct_port + i * 2))
61        f.write("[dcp]\n")
62        f.write("port={0}\n".format(base_direct_port + i * 2))
63
64
65def couch_configs(i):
66    mk_node_couch_config(i)
67    return ["{0}/etc/couchdb/default.ini".format(PREFIX),
68            "{0}/etc/couchdb/default.d/capi.ini".format(PREFIX),
69            "{0}/etc/couchdb/default.d/geocouch.ini".format(PREFIX),
70            "couch/n_{0}_conf.ini".format(i)]
71
72def os_specific(args, params):
73    """Add os-specific junk to the cluster startup."""
74    if platform.system() == 'Windows':
75        args += ["dont_suppress_stderr_logger", "false"]
76    else:
77        args += ["dont_suppress_stderr_logger", "true"]
78    if platform.system() == 'Darwin':
79        import resource
80        ## OS X has a pretty tiny default fd limit.  Let's increase it (if it hasn't already been).
81        (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
82        if soft < 2048:
83            resource.setrlimit(resource.RLIMIT_NOFILE, (2048, 2048))
84        params['env'] = {"ERL_MAX_PORTS": "2048"}
85        params['env'].update(os.environ)
86
87ebin_path = None
88cluster_extra_args = None
89cluster_args_prefix = None
90
91def prepare_start_cluster(extra_args, args_prefix):
92    global cluster_args_prefix
93    global cluster_extra_args
94    global ebin_path
95
96    ebin_path = setup_path()
97    cluster_extra_args = extra_args
98    cluster_args_prefix = args_prefix
99
100def quote_string_for_erl(s):
101    return '"' + s.replace("\\", "\\\\").replace("\"", "\\\"") + '"'
102
103def erlang_args_for_node(i):
104    logdir = "logs/n_{0}".format(i)
105
106    args = cluster_args_prefix + ["erl", "+MMmcs" "30",
107                                  "+A", "16", "+sbtu",
108                                  "+P", "327680", "-pa"] + ebin_path
109    args += [
110        "-setcookie", "nocookie",
111        "-kernel", "inet_dist_listen_min", "21100",
112        "inet_dist_listen_max", "21199",
113        "error_logger", "false",
114        "-sasl", "sasl_error_logger", "false",
115        "-couch_ini"] + couch_configs(i)
116
117    datadir = os.path.abspath('data/n_{0}'.format(i))
118    tempdir = os.path.abspath('tmp/')
119    nodefile = os.path.join(datadir, "nodefile")
120    babysitternodefile = os.path.join(datadir, "couchbase-server.babysitter.node")
121    babysittercookiefile = os.path.join(datadir, "couchbase-server.cookie")
122
123    args += [
124        "-name", "babysitter_of_n_{0}@127.0.0.1".format(i),
125        "-hidden",
126        "-kernel", "global_enable_tracing", "true",
127        "-ns_babysitter", "cookiefile", quote_string_for_erl(babysittercookiefile),
128        "-ns_babysitter", "nodefile", quote_string_for_erl(babysitternodefile),
129        "-ns_server", "config_path", '"etc/static_config.in"',
130        "-ns_server", "enable_mlockall", "false",
131        "error_logger_mf_dir", quote_string_for_erl(logdir),
132        "path_config_etcdir", '"priv"',
133        "path_config_bindir", quote_string_for_erl(PREFIX+"/bin"),
134        "path_config_libdir", quote_string_for_erl(PREFIX+"/lib"),
135        "path_config_datadir", quote_string_for_erl(datadir),
136        "path_config_tmpdir", quote_string_for_erl(tempdir),
137        "path_config_secdir", quote_string_for_erl(PREFIX+"/etc/security"),
138        "path_audit_log", quote_string_for_erl(logdir),
139        "rest_port", str(base_api_port + i),
140        "query_port", str(base_couch_port - 1 - i),
141        "ssl_query_port", str(10000 + base_couch_port - 1 - i),
142        "projector_port", str(base_projector_port + i),
143        "ssl_rest_port", str(10000 + base_api_port + i),
144        "capi_port", str(base_couch_port + i),
145        "ssl_capi_port", str(10000 + base_couch_port + i),
146        "memcached_port", str(base_direct_port + i * 2),
147        "moxi_port", str(base_direct_port + i * 2 + 1),
148        "memcached_dedicated_port", str(base_direct_port - 1 - i * 4),
149        "ssl_proxy_downstream_port", str(base_direct_port - 2 - i * 4),
150        "ssl_proxy_upstream_port", str(base_direct_port - 3 - i * 4),
151        "memcached_ssl_port", str(base_direct_port - 4 - i * 4),
152        "nodefile", quote_string_for_erl(nodefile),
153        "short_name", quote_string_for_erl('n_{0}'.format(i)),
154        "xdcr_rest_port", str(base_xdcr_port + i),
155        "indexer_admin_port", str(base_indexer_port + i * 6),
156        "indexer_scan_port", str(base_indexer_port + i * 6 + 1),
157        "indexer_http_port", str(base_indexer_port + i * 6 + 2),
158        "indexer_stinit_port", str(base_indexer_port + i * 6 + 3),
159        "indexer_stcatchup_port", str(base_indexer_port + i * 6 + 4),
160        "indexer_stmaint_port", str(base_indexer_port + i * 6 + 5),
161        "fts_http_port", str(base_fts_port + i)
162        ] + cluster_extra_args
163
164    return args
165
166def start_cluster(num_nodes, start_index, host, extra_args, args_prefix):
167    prepare_start_cluster(extra_args, args_prefix)
168
169    def start_node(i):
170        logdir = "logs/n_{0}".format(i)
171        try:
172            os.makedirs(logdir)
173        except:
174            pass
175
176        args = erlang_args_for_node(i)
177
178        params = {}
179
180        os_specific(args, params)
181
182        if not params.has_key('env'):
183            params['env'] = {}
184            params['env'].update(os.environ)
185        path = params['env']['PATH']
186        path = (PREFIX+"/bin") + os.pathsep + path
187        if not params['env'].has_key('ERL_FULLSWEEP_AFTER'):
188            params['env']['ERL_FULLSWEEP_AFTER'] = '512'
189        params['env']['PATH'] = path
190
191        crash_dump_base = 'erl_crash.dump.n_%d' % i
192        params['env']['ERL_CRASH_DUMP_BASE'] = crash_dump_base
193        params['env']['ERL_CRASH_DUMP'] = crash_dump_base + '.babysitter'
194
195        params['env']['COUCHBASE_SMALLER_PKEYS'] = '1'
196
197        params['close_fds'] = True
198        if platform.system() == "Windows":
199            params['close_fds'] = False
200
201        w = None
202
203        if "-noinput" in args:
204            (r,w) = os.pipe()
205
206            params['stdin'] = r
207
208            if 'setpgrp' in os.__dict__ and params.get('close_fds'):
209                # this puts child out of our process group. So that
210                # Ctrl-C doesn't deliver SIGINT to it, leaving us
211                # ability to it shutdown carefully or otherwise
212                params['preexec_fn'] = os.setpgrp
213
214        pr = subprocess.Popen(args, **params)
215        if w != None:
216            os.close(r)
217        pr.write_side = w
218        return pr
219
220    return [start_node(i + start_index) for i in xrange(num_nodes)]
221
222def usage():
223    sys.exit("Usage: {0} [--nodes=N] [--dont-rename] [--dont-start] "
224             "[--interactive] [--static-cookie] [--start-index=N] "
225             "[--static-cookie] [--host=H] [--loglevel=L] "
226             "[--pluggable-config=File] [--minified] [--disable-autocomplete]"
227             "[ns_server args]".format(sys.argv[0]))
228
229def find_primary_addr():
230    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
231    try:
232        s.connect(("8.8.8.8", 56))
233        addr, port = s.getsockname()
234        return addr
235    except socket.error:
236        return None
237    finally:
238        s.close()
239
240def main():
241    try:
242        optlist, args = getopt.gnu_getopt(sys.argv[1:], "hn:i",
243                                          ["help", "start-index=", "nodes=",
244                                           "dont-rename", "interactive",
245                                           "static-cookie", "dont-start",
246                                           "host=", "loglevel=",
247                                           "prepend-extras", "pluggable-config=",
248                                           "minified", "disable-autocomplete"])
249    except getopt.GetoptError, err:
250        # print help information and exit:
251        print str(err) # will print something like "option -a not recognized"
252        usage()
253        sys.exit(2)
254
255    dont_rename = False
256    dont_start = False
257    static_cookie = False
258    interactive_shell = False
259    start_index = 0
260    num_nodes = 1
261    prepend_extras = False
262    host = "127.0.0.1"
263    loglevel = 'debug'
264    pluggable_config = []
265    use_minified = False
266    disable_autocomplete = "{disable_autocomplete,false}"
267
268    for o, a in optlist:
269        if o in ("--nodes", "-n"):
270            num_nodes = int(a)
271        elif o in ("--interactive", "-i"):
272            interactive_shell = True
273        elif o == '--dont-start':
274            dont_start = True
275        elif o == '--host':
276            host = a
277        elif o == '--start-index':
278            start_index = int(a)
279        elif o == '--dont-rename':
280            dont_rename = True
281        elif o in ("--help", "-h"):
282            usage()
283            exit(0)
284        elif o in("--static-cookie"):
285            static_cookie = True
286        elif o == '--loglevel':
287            loglevel = a
288        elif o == "--prepend-extras":
289            prepend_extras = True
290        elif o == "--pluggable-config":
291            pluggable_config.append(a)
292        elif o == "--minified":
293            use_minified = True
294        elif o == "--disable-autocomplete":
295            disable_autocomplete = "{disable_autocomplete,true}"
296        else:
297            assert False, "unhandled options"
298
299    nodes = []
300    terminal_attrs = None
301
302    def kill_nodes(*args):
303        for n in nodes:
304            if n.write_side != None:
305                print("Closing %d\n" % n.write_side)
306                # os.write(n.write_side, "shutdown\n") # this line does graceful shutdown versus quick
307                os.close(n.write_side)
308            else:
309                try:
310                    n.kill()
311                except OSError:
312                    pass
313
314        for n in nodes:
315            n.wait()
316
317        if terminal_attrs != None:
318            termios.tcsetattr(sys.stdin, termios.TCSANOW, terminal_attrs)
319
320    atexit.register(kill_nodes)
321
322    try:
323        import termios
324        terminal_attrs = termios.tcgetattr(sys.stdin)
325    except:
326        pass
327
328    extra_args = []
329    if not dont_rename:
330        primary_addr = find_primary_addr()
331        if primary_addr == None:
332            print("was unable to detect 'internet' address of this machine."
333                  + " node rename will be disabled")
334        else:
335            extra_args += ["rename_ip", '"' + primary_addr + '"']
336
337    extra_args += args[1:]
338    if prepend_extras:
339        prepend_args = args[0:]
340    else:
341        prepend_args = []
342        extra_args += args[0:]
343
344    if static_cookie:
345        extra_args += ["-ns_server", "dont_reset_cookie", "true"]
346
347    if not interactive_shell:
348        extra_args += ["-noinput"]
349        if not dont_start:
350            extra_args += ["-run", "child_erlang", "child_start", "ns_babysitter_bootstrap"]
351            extra_args += ["-ns_babysitter", "handle_ctrl_c", "true"]
352    else:
353        if not dont_start:
354            extra_args += ["-run", "ns_babysitter_bootstrap"]
355
356    if loglevel not in LOGLEVELS:
357        print "Valid log levels are the following: %s" % ', '.join(LOGLEVELS)
358        sys.exit(1)
359    extra_args += ["-ns_server", "loglevel_stderr", loglevel]
360
361    plugins_dir = '../build/cluster_run_ui_plugins'
362    if os.path.isdir(plugins_dir):
363        for file in os.listdir(plugins_dir):
364            if fnmatch.fnmatch(file, 'pluggable-ui-*.cluster_run.json'):
365                pluggable_config.append(os.path.join(plugins_dir, file))
366
367    if pluggable_config:
368        extra_args += ["-ns_server", "ui_plugins",
369                        quote_string_for_erl(','.join(pluggable_config))]
370
371    ui_env = [disable_autocomplete]
372
373    extra_args += ["-ns_server", "use_minified", "true" if use_minified else "false"]
374    extra_args += ["-ns_server", "ui_env", '[' + ','.join(ui_env) + ']']
375
376    nodes = start_cluster(num_nodes, start_index, host, extra_args, prepend_args)
377
378    for node in nodes:
379        node.wait()
380
381
382if __name__ == '__main__':
383    main()
384