1#!/usr/bin/env python
2#
3# @author Couchbase <info@couchbase.com>
4# @copyright 2011-2018 Couchbase, Inc.
5#
6# Licensed under the Apache License, Version 2.0 (the "License");
7# you may not use this file except in compliance with the License.
8# You may obtain a copy of the License at
9#
10#      http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS,
14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15# See the License for the specific language governing permissions and
16# limitations under the License.
17import os, os.path, subprocess, sys, signal, atexit, socket, getopt, select, shlex
18import platform, fnmatch
19
20base_direct_port = 12000
21base_api_port = 9000
22base_couch_port = 9500
23base_projector_port = 10000
24base_xdcr_port = 13000
25base_indexer_port = 9100
26base_fts_port = 9200
27base_eventing_port = 9300
28base_cbas_port = 9600
29
30LOGLEVELS = ["debug", "info", "warn", "error", "critical"]
31
32def read_configuration():
33    with open("build/cluster_run.configuration") as f:
34        def fn(line):
35            k, v = line.strip().split('=')
36            return k, shlex.split(v)[0]
37
38        return dict(fn(line) for line in f.readlines())
39
40config = read_configuration()
41PREFIX = config['prefix']
42
43def setup_path():
44    def ebin_search(path):
45        dirs = os.walk(path)
46        ebins = []
47
48        for d, _, _ in dirs:
49            if os.path.basename(d) == "ebin":
50                ebins.append(d)
51
52        return ebins
53
54    path = ebin_search(".")
55    couchpath = ebin_search("{0}/lib/couchdb/erlang/lib".format(PREFIX))
56    couch_plugins = ebin_search("{0}/lib/couchdb/plugins".format(PREFIX))
57
58    if len(couchpath) == 0:
59       sys.exit("Couch libs wasn't found.\nCan't handle it")
60
61    return couchpath + path + couch_plugins
62
63def mk_node_couch_config(i):
64    try:
65        os.mkdir("couch")
66    except os.error:
67        pass
68
69    with open("couch/n_{0}_conf.ini".format(i), "w") as f:
70        f.write("[httpd]\n")
71        f.write("port={0}\n".format(base_couch_port + i))
72        f.write("[couchdb]\n")
73        f.write("database_dir={0}/data/n_{1}/data\n".format(os.getcwd(), i))
74        f.write("view_index_dir={0}/data/n_{1}/data\n".format(os.getcwd(), i))
75        f.write("max_dbs_open=10000\n")
76        f.write("[upr]\n")
77        f.write("port={0}\n".format(base_direct_port + i * 2))
78        f.write("[dcp]\n")
79        f.write("port={0}\n".format(base_direct_port + i * 2))
80
81
82def couch_configs(i):
83    mk_node_couch_config(i)
84    return ["{0}/etc/couchdb/default.ini".format(PREFIX),
85            "{0}/etc/couchdb/default.d/capi.ini".format(PREFIX),
86            "{0}/etc/couchdb/default.d/geocouch.ini".format(PREFIX),
87            "couch/n_{0}_conf.ini".format(i)]
88
89def os_specific(args, params):
90    """Add os-specific junk to the cluster startup."""
91    if platform.system() == 'Windows':
92        args += ["dont_suppress_stderr_logger", "false"]
93    else:
94        args += ["dont_suppress_stderr_logger", "true"]
95    if platform.system() == 'Darwin':
96        import resource
97        ## OS X has a pretty tiny default fd limit.  Let's increase it (if it hasn't already been).
98        (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
99        if soft < 2048:
100            resource.setrlimit(resource.RLIMIT_NOFILE, (2048, 2048))
101        params['env'] = {"ERL_MAX_PORTS": "2048"}
102        params['env'].update(os.environ)
103
104ebin_path = None
105cluster_extra_args = None
106cluster_args_prefix = None
107
108def is_ipv6_setup():
109    return os.getenv("IPV6", "false") == "true"
110
111def prepare_start_cluster(extra_args, args_prefix):
112    global cluster_args_prefix
113    global cluster_extra_args
114    global ebin_path
115
116    ebin_path = setup_path()
117    cluster_extra_args = extra_args
118    cluster_args_prefix = args_prefix
119
120def quote_string_for_erl(s):
121    return '"' + s.replace("\\", "\\\\").replace("\"", "\\\"") + '"'
122
123def erlang_args_for_node(i):
124    logdir = "logs/n_{0}".format(i)
125
126    args = cluster_args_prefix + ["erl", "+MMmcs" "30",
127                                  "+A", "16", "+sbtu",
128                                  "+P", "327680", "-pa"] + ebin_path
129    args += [
130        "-setcookie", "nocookie",
131        "-kernel", "inet_dist_listen_min", "21100",
132        "inet_dist_listen_max", "21199",
133        "error_logger", "false",
134        "-sasl", "sasl_error_logger", "false",
135        "-couch_ini"] + couch_configs(i)
136
137    datadir = os.path.abspath('data/n_{0}'.format(i))
138    tempdir = os.path.abspath('tmp/')
139    nodefile = os.path.join(datadir, "nodefile")
140    babysitternodefile = os.path.join(datadir, "couchbase-server.babysitter.node")
141    babysittercookiefile = os.path.join(datadir, "couchbase-server.babysitter.cookie")
142
143    args += [
144        "-name", "babysitter_of_n_{0}@::1".format(i) if ipv6 else "babysitter_of_n_{0}@127.0.0.1".format(i),
145        "-proto_dist", "inet6_tcp" if ipv6 else "inet_tcp",
146        "-hidden",
147        "-kernel", "global_enable_tracing", "true",
148        "-ns_babysitter", "cookiefile", quote_string_for_erl(babysittercookiefile),
149        "-ns_babysitter", "nodefile", quote_string_for_erl(babysitternodefile),
150        "-ns_server", "config_path", '"etc/static_config.in"',
151        "-ns_server", "ipv6", "true" if ipv6 else "false",
152        "error_logger_mf_dir", quote_string_for_erl(logdir),
153        "path_config_etcdir", '"priv"',
154        "path_config_bindir", quote_string_for_erl(PREFIX+"/bin"),
155        "path_config_libdir", quote_string_for_erl(PREFIX+"/lib"),
156        "path_config_datadir", quote_string_for_erl(datadir),
157        "path_config_tmpdir", quote_string_for_erl(tempdir),
158        "path_config_secdir", quote_string_for_erl(PREFIX+"/etc/security"),
159        "path_audit_log", quote_string_for_erl(logdir),
160        "rest_port", str(base_api_port + i),
161        "query_port", str(base_couch_port - 1 - i),
162        "ssl_query_port", str(10000 + base_couch_port - 1 - i),
163        "projector_port", str(base_projector_port + i),
164        "ssl_rest_port", str(10000 + base_api_port + i),
165        "capi_port", str(base_couch_port + i),
166        "ssl_capi_port", str(10000 + base_couch_port + i),
167        "memcached_port", str(base_direct_port + i * 2),
168        "moxi_port", str(base_direct_port + i * 2 + 1),
169        "memcached_dedicated_port", str(base_direct_port - 1 - i * 4),
170        "memcached_ssl_port", str(base_direct_port - 4 - i * 4),
171        "nodefile", quote_string_for_erl(nodefile),
172        "short_name", quote_string_for_erl('n_{0}'.format(i)),
173        "xdcr_rest_port", str(base_xdcr_port + i),
174        "indexer_admin_port", str(base_indexer_port + i * 6),
175        "indexer_scan_port", str(base_indexer_port + i * 6 + 1),
176        "indexer_http_port", str(base_indexer_port + i * 6 + 2),
177        "indexer_https_port", str(10000 + base_indexer_port + i * 6 + 2),
178        "indexer_stinit_port", str(base_indexer_port + i * 6 + 3),
179        "indexer_stcatchup_port", str(base_indexer_port + i * 6 + 4),
180        "indexer_stmaint_port", str(base_indexer_port + i * 6 + 5),
181        "fts_http_port", str(base_fts_port + i),
182        "fts_ssl_port", str(10000 + base_fts_port + i),
183        "eventing_http_port", str(base_eventing_port + i),
184        "eventing_https_port", str(10000 + base_eventing_port + i),
185        "eventing_debug_port", str(base_eventing_port + i * 6 + 1),
186        "cbas_http_port", str(base_cbas_port + i * 15),
187        "cbas_cc_http_port", str(base_cbas_port + i * 15 + 1),
188        "cbas_cc_cluster_port", str(base_cbas_port + i * 15 + 2),
189        "cbas_cc_client_port", str(base_cbas_port + i * 15 + 3),
190        "cbas_console_port", str(base_cbas_port + i * 15 + 4),
191        "cbas_cluster_port", str(base_cbas_port + i * 15 + 5),
192        "cbas_data_port", str(base_cbas_port + i * 15 + 6),
193        "cbas_result_port", str(base_cbas_port + i * 15 + 7),
194        "cbas_messaging_port", str(base_cbas_port + i * 15 + 8),
195        "cbas_debug_port", str(base_cbas_port + i * 15 + 9),
196        "cbas_parent_port", str(base_cbas_port + i * 15 + 10),
197        "cbas_admin_port", str(base_cbas_port + i * 15 + 11),
198        "cbas_replication_port", str(base_cbas_port + i * 15 + 12),
199        "cbas_metadata_port", str(base_cbas_port + i * 15 + 13),
200        "cbas_metadata_callback_port", str(base_cbas_port + i * 15 + 14),
201        "cbas_ssl_port", str(10000 + base_cbas_port + i)
202        ] + cluster_extra_args
203
204    return args
205
206def start_cluster(num_nodes, start_index, host, extra_args, args_prefix):
207    prepare_start_cluster(extra_args, args_prefix)
208
209    def start_node(i):
210        logdir = "logs/n_{0}".format(i)
211        try:
212            os.makedirs(logdir)
213        except:
214            pass
215
216        args = erlang_args_for_node(i)
217
218        params = {}
219
220        os_specific(args, params)
221
222        if not params.has_key('env'):
223            params['env'] = {}
224            params['env'].update(os.environ)
225        path = params['env']['PATH']
226        path = (PREFIX+"/bin") + os.pathsep + path
227        if not params['env'].has_key('ERL_FULLSWEEP_AFTER'):
228            params['env']['ERL_FULLSWEEP_AFTER'] = '512'
229        params['env']['PATH'] = path
230
231        crash_dump_base = 'erl_crash.dump.n_%d' % i
232        params['env']['ERL_CRASH_DUMP_BASE'] = crash_dump_base
233        params['env']['ERL_CRASH_DUMP'] = crash_dump_base + '.babysitter'
234
235        params['env']['COUCHBASE_SMALLER_PKEYS'] = '1'
236
237        params['close_fds'] = True
238        if platform.system() == "Windows":
239            params['close_fds'] = False
240
241        w = None
242
243        if "-noinput" in args:
244            (r,w) = os.pipe()
245
246            params['stdin'] = r
247
248            if 'setpgrp' in os.__dict__ and params.get('close_fds'):
249                # this puts child out of our process group. So that
250                # Ctrl-C doesn't deliver SIGINT to it, leaving us
251                # ability to it shutdown carefully or otherwise
252                params['preexec_fn'] = os.setpgrp
253
254        pr = subprocess.Popen(args, **params)
255        if w != None:
256            os.close(r)
257        pr.write_side = w
258        return pr
259
260    return [start_node(i + start_index) for i in xrange(num_nodes)]
261
262def usage():
263    sys.exit("Usage: {0} [--nodes=N] [--dont-rename] [--dont-start] "
264             "[--interactive] [--static-cookie] [--start-index=N] "
265             "[--static-cookie] [--host=H] [--loglevel=L] "
266             "[--pluggable-config=File] [--minified] [--disable-autocomplete]"
267             "[ns_server args]".format(sys.argv[0]))
268
269def find_primary_addr():
270    Family = socket.AF_INET6 if ipv6 else socket.AF_INET
271    DnsAddr = "2001:4860:4860::8844" if ipv6 else "8.8.8.8"
272    s = socket.socket(Family, socket.SOCK_DGRAM)
273    try:
274        s.connect((DnsAddr, 53))
275        if ipv6:
276            addr, port, _, _ = s.getsockname()
277        else:
278            addr, port = s.getsockname()
279
280        return addr
281    except socket.error:
282        return None
283    finally:
284        s.close()
285
286def main():
287    try:
288        optlist, args = getopt.gnu_getopt(sys.argv[1:], "hn:i",
289                                          ["help", "start-index=", "nodes=",
290                                           "dont-rename", "interactive",
291                                           "static-cookie", "dont-start",
292                                           "host=", "loglevel=",
293                                           "prepend-extras", "pluggable-config=",
294                                           "minified", "disable-autocomplete",
295                                           "pretend-version="])
296    except getopt.GetoptError, err:
297        # print help information and exit:
298        print str(err) # will print something like "option -a not recognized"
299        usage()
300        sys.exit(2)
301
302    global ipv6
303    ipv6 = is_ipv6_setup()
304
305    dont_rename = False
306    dont_start = False
307    static_cookie = False
308    start_index = 0
309    num_nodes = 1
310    prepend_extras = False
311    host = "127.0.0.1"
312    loglevel = 'debug'
313    pluggable_config = []
314    use_minified = False
315    disable_autocomplete = "{disable_autocomplete,false}"
316    pretend_version = None
317
318    for o, a in optlist:
319        if o in ("--nodes", "-n"):
320            num_nodes = int(a)
321        elif o == '--dont-start':
322            dont_start = True
323        elif o == '--host':
324            host = a
325        elif o == '--start-index':
326            start_index = int(a)
327        elif o == '--dont-rename':
328            dont_rename = True
329        elif o in ("--help", "-h"):
330            usage()
331            exit(0)
332        elif o in("--static-cookie"):
333            static_cookie = True
334        elif o == '--loglevel':
335            loglevel = a
336        elif o == "--prepend-extras":
337            prepend_extras = True
338        elif o == "--pluggable-config":
339            pluggable_config.append(a)
340        elif o == "--minified":
341            use_minified = True
342        elif o == "--disable-autocomplete":
343            disable_autocomplete = "{disable_autocomplete,true}"
344        elif o == "--pretend-version":
345            pretend_version = a
346        else:
347            assert False, "unhandled options"
348
349    nodes = []
350    terminal_attrs = None
351
352    def kill_nodes(*args):
353        for n in nodes:
354            if n.write_side != None:
355                print("Closing %d\n" % n.write_side)
356                # os.write(n.write_side, "shutdown\n") # this line does graceful shutdown versus quick
357                os.close(n.write_side)
358            else:
359                try:
360                    n.kill()
361                except OSError:
362                    pass
363
364        for n in nodes:
365            n.wait()
366
367        if terminal_attrs != None:
368            termios.tcsetattr(sys.stdin, termios.TCSANOW, terminal_attrs)
369
370    atexit.register(kill_nodes)
371
372    try:
373        import termios
374        terminal_attrs = termios.tcgetattr(sys.stdin)
375    except:
376        pass
377
378    extra_args = []
379    if not dont_rename:
380        primary_addr = find_primary_addr()
381        if primary_addr == None:
382            print("was unable to detect 'internet' address of this machine."
383                  + " node rename will be disabled")
384        else:
385            extra_args += ["rename_ip", '"' + primary_addr + '"']
386
387    extra_args += args[1:]
388    if prepend_extras:
389        prepend_args = args[0:]
390    else:
391        prepend_args = []
392        extra_args += args[0:]
393
394    if static_cookie:
395        extra_args += ["-ns_server", "dont_reset_cookie", "true"]
396
397    if dont_start:
398        extra_args += ["-run", "t", "fake_loggers"]
399    else:
400        extra_args += ["-noinput"]
401        extra_args += ["-run", "child_erlang", "child_start", "ns_babysitter_bootstrap"]
402        extra_args += ["-ns_babysitter", "handle_ctrl_c", "true"]
403
404    if loglevel not in LOGLEVELS:
405        print "Valid log levels are the following: %s" % ', '.join(LOGLEVELS)
406        sys.exit(1)
407    extra_args += ["-ns_server", "loglevel_stderr", loglevel]
408
409    plugins_dir = '../build/cluster_run_ui_plugins'
410    if os.path.isdir(plugins_dir):
411        for file in os.listdir(plugins_dir):
412            if fnmatch.fnmatch(file, 'pluggable-ui-*.cluster_run.json'):
413                pluggable_config.append(os.path.join(plugins_dir, file))
414
415    if pluggable_config:
416        extra_args += ["-ns_server", "ui_plugins",
417                        quote_string_for_erl(','.join(pluggable_config))]
418
419    ui_env = [disable_autocomplete]
420
421    extra_args += ["-ns_server", "use_minified", "true" if use_minified else "false"]
422    extra_args += ["-ns_server", "ui_env", '[' + ','.join(ui_env) + ']']
423
424    if pretend_version is not None:
425        extra_args += ["-ns_server",
426                       "pretend_version", '"{}"'.format(pretend_version)]
427
428    nodes = start_cluster(num_nodes, start_index, host, extra_args, prepend_args)
429
430    for node in nodes:
431        node.wait()
432
433
434if __name__ == '__main__':
435    main()
436