xref: /6.0.3/ns_server/src/misc.erl (revision e8532222)
1% Copyright (c) 2009-2018, Couchbase, Inc.
2% Copyright (c) 2008, Cliff Moon
3% Copyright (c) 2008, Powerset, Inc
4%
5% All rights reserved.
6%
7% Redistribution and use in source and binary forms, with or without
8% modification, are permitted provided that the following conditions
9% are met:
10%
11% * Redistributions of source code must retain the above copyright
12% notice, this list of conditions and the following disclaimer.
13% * Redistributions in binary form must reproduce the above copyright
14% notice, this list of conditions and the following disclaimer in the
15% documentation and/or other materials provided with the distribution.
16% * Neither the name of Powerset, Inc nor the names of its
17% contributors may be used to endorse or promote products derived from
18% this software without specific prior written permission.
19%
20% THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21% "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22% LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
23% FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
24% COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
25% INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
26% BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
27% LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
28% CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29% LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
30% ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31% POSSIBILITY OF SUCH DAMAGE.
32%
33% Original Author: Cliff Moon
34
35-module(misc).
36
37-include("ns_common.hrl").
38-include_lib("kernel/include/file.hrl").
39
40-include("triq.hrl").
41-include_lib("eunit/include/eunit.hrl").
42
43-include("cut.hrl").
44-include("generic.hrl").
45
46-compile(export_all).
47-export_type([timer/0, timer/1]).
48
49shuffle(List) when is_list(List) ->
50    [N || {_R, N} <- lists:keysort(1, [{random:uniform(), X} || X <- List])].
51
52get_days_list() ->
53    ["Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"].
54
55% formats time (see erlang:localtime/0) as ISO-8601 text
56iso_8601_fmt({{Year,Month,Day},{Hour,Min,Sec}}) ->
57    io_lib:format("~4.10.0B-~2.10.0B-~2.10.0B ~2.10.0B:~2.10.0B:~2.10.0B",
58                  [Year, Month, Day, Hour, Min, Sec]).
59
60%% applies (catch Fun(X)) for each element of list in parallel. If
61%% execution takes longer than given Timeout it'll exit with reason
62%% timeout (which is consistent with behavior of gen_XXX:call
63%% modules).  Care is taken to not leave any messages in calling
64%% process mailbox and to correctly shutdown any worker processes
65%% if/when calling process is killed.
66-spec parallel_map(fun((any()) -> any()), [any()], non_neg_integer() | infinity) -> [any()].
67parallel_map(Fun, List, Timeout) when is_list(List) andalso is_function(Fun) ->
68    case async:run_with_timeout(
69           fun () ->
70                   async:map(Fun, List)
71           end, Timeout) of
72        {ok, R} ->
73            R;
74        {error, timeout} ->
75            exit(timeout)
76    end.
77
78gather_dir_info(Name) ->
79    case file:list_dir(Name) of
80        {ok, Filenames} ->
81            [gather_link_info(filename:join(Name, N)) || N <- Filenames];
82        Error ->
83            Error
84    end.
85
86gather_link_info(Name) ->
87    case file:read_link_info(Name) of
88        {ok, Info} ->
89            case Info#file_info.type of
90                directory ->
91                    {Name, Info, gather_dir_info(Name)};
92                _ ->
93                    {Name, Info}
94            end;
95        Error ->
96            {Name, Error}
97    end.
98
99rm_rf(Name) when is_list(Name) ->
100  case rm_rf_is_dir(Name) of
101      {ok, false} ->
102          file:delete(Name);
103      {ok, true} ->
104          case file:list_dir(Name) of
105              {ok, Filenames} ->
106                  case rm_rf_loop(Name, Filenames) of
107                      ok ->
108                          case file:del_dir(Name) of
109                              ok ->
110                                  ok;
111                              {error, enoent} ->
112                                  ok;
113                              Error ->
114                                  ?log_warning("Cannot delete ~p: ~p~nDir info: ~p",
115                                               [Name, Error, gather_dir_info(Name)]),
116                                  Error
117                          end;
118                      Error ->
119                          Error
120                  end;
121              {error, enoent} ->
122                  ok;
123              {error, Reason} = Error ->
124                  ?log_warning("rm_rf failed because ~p", [Reason]),
125                  Error
126          end;
127      {error, enoent} ->
128          ok;
129      Error ->
130          ?log_warning("stat on ~s failed: ~p", [Name, Error]),
131          Error
132  end.
133
134rm_rf_is_dir(Name) ->
135    case file:read_link_info(Name) of
136        {ok, Info} ->
137            {ok, Info#file_info.type =:= directory};
138        Error ->
139            Error
140    end.
141
142rm_rf_loop(_DirName, []) ->
143    ok;
144rm_rf_loop(DirName, [F | Files]) ->
145    FileName = filename:join(DirName, F),
146    case rm_rf(FileName) of
147        ok ->
148            rm_rf_loop(DirName, Files);
149        {error, enoent} ->
150            rm_rf_loop(DirName, Files);
151        Error ->
152            ?log_warning("Cannot delete ~p: ~p", [FileName, Error]),
153            Error
154    end.
155
156rand_str(N) ->
157  lists:map(fun(_I) ->
158      random:uniform(26) + $a - 1
159    end, lists:seq(1,N)).
160
161nthreplace(N, E, List) ->
162  lists:sublist(List, N-1) ++ [E] ++ lists:nthtail(N, List).
163
164ceiling(X) ->
165  T = erlang:trunc(X),
166  case (X - T) of
167    Neg when Neg < 0 -> T;
168    Pos when Pos > 0 -> T + 1;
169    _ -> T
170  end.
171
172position(Predicate, List) when is_function(Predicate) ->
173  position(Predicate, List, 1);
174
175position(E, List) ->
176  position(E, List, 1).
177
178position(Predicate, [], _N) when is_function(Predicate) -> false;
179
180position(Predicate, [E|List], N) when is_function(Predicate) ->
181  case Predicate(E) of
182    true -> N;
183    false -> position(Predicate, List, N+1)
184  end;
185
186position(_, [], _) -> false;
187
188position(E, [E|_List], N) -> N;
189
190position(E, [_|List], N) -> position(E, List, N+1).
191
192msecs_to_usecs(MilliSec) ->
193    MilliSec * 1000.
194
195% Returns just the node name string that's before the '@' char.
196% For example, returns "test" instead of "test@myhost.com".
197%
198node_name_short() ->
199    node_name_short(node()).
200
201node_name_short(Node) ->
202    [NodeName | _] = string:tokens(atom_to_list(Node), "@"),
203    NodeName.
204
205% Node is an atom like some_name@host.foo.bar.com
206
207node_name_host(Node) ->
208    [Name, Host | _] = string:tokens(atom_to_list(Node), "@"),
209    {Name, Host}.
210
211% Get an environment variable value or return a default value
212getenv_int(VariableName, DefaultValue) ->
213    case (catch list_to_integer(os:getenv(VariableName))) of
214        EnvBuckets when is_integer(EnvBuckets) -> EnvBuckets;
215        _ -> DefaultValue
216    end.
217
218% Get an application environment variable, or a defualt value.
219get_env_default(Var, Def) ->
220    case application:get_env(Var) of
221        {ok, Value} -> Value;
222        undefined -> Def
223    end.
224
225get_env_default(App, Var, Def) ->
226    case application:get_env(App, Var) of
227        {ok, Value} ->
228            Value;
229        undefined ->
230            Def
231    end.
232
233ping_jointo() ->
234    case application:get_env(jointo) of
235        {ok, NodeName} -> ping_jointo(NodeName);
236        X -> X
237    end.
238
239ping_jointo(NodeName) ->
240    ?log_debug("attempting to contact ~p", [NodeName]),
241    case net_adm:ping(NodeName) of
242        pong -> ?log_debug("connected to ~p", [NodeName]);
243        pang -> {error, io_lib:format("could not ping ~p~n", [NodeName])}
244    end.
245
246%% http://github.com/joearms/elib1/blob/master/lib/src/elib1_misc.erl#L1367
247
248%%----------------------------------------------------------------------
249%% @doc remove leading and trailing white space from a string.
250
251-spec trim(string()) -> string().
252
253trim(S) ->
254    remove_leading_and_trailing_whitespace(S).
255
256trim_test() ->
257    "abc" = trim("    abc   ").
258
259%%----------------------------------------------------------------------
260%% @doc remove leading and trailing white space from a string.
261
262-spec remove_leading_and_trailing_whitespace(string()) -> string().
263
264remove_leading_and_trailing_whitespace(X) ->
265    remove_leading_whitespace(remove_trailing_whitespace(X)).
266
267remove_leading_and_trailing_whitespace_test() ->
268    "abc" = remove_leading_and_trailing_whitespace("\r\t  \n \s  abc").
269
270%%----------------------------------------------------------------------
271%% @doc remove leading white space from a string.
272
273-spec remove_leading_whitespace(string()) -> string().
274
275remove_leading_whitespace([$\n|T]) -> remove_leading_whitespace(T);
276remove_leading_whitespace([$\r|T]) -> remove_leading_whitespace(T);
277remove_leading_whitespace([$\s|T]) -> remove_leading_whitespace(T);
278remove_leading_whitespace([$\t|T]) -> remove_leading_whitespace(T);
279remove_leading_whitespace(X) -> X.
280
281%%----------------------------------------------------------------------
282%% @doc remove trailing white space from a string.
283
284-spec remove_trailing_whitespace(string()) -> string().
285
286remove_trailing_whitespace(X) ->
287    lists:reverse(remove_leading_whitespace(lists:reverse(X))).
288
289%% Wait for a process.
290
291wait_for_process(PidOrName, Timeout) ->
292    MRef = erlang:monitor(process, PidOrName),
293    receive
294        {'DOWN', MRef, process, _, _Reason} ->
295            ok
296    after Timeout ->
297            erlang:demonitor(MRef, [flush]),
298            {error, timeout}
299    end.
300
301wait_for_process_test_() ->
302    {spawn,
303     fun () ->
304             %% Normal
305             ok = wait_for_process(spawn(fun() -> ok end), 100),
306             %% Timeout
307             {error, timeout} = wait_for_process(spawn(fun() ->
308                                                               timer:sleep(100), ok end),
309                                                 1),
310             %% Process that exited before we went.
311             Pid = spawn(fun() -> ok end),
312             ok = wait_for_process(Pid, 100),
313             ok = wait_for_process(Pid, 100)
314     end}.
315
316-spec terminate(pid(), atom()) -> true.
317terminate(Pid, normal) ->
318    terminate(Pid, shutdown);
319terminate(Pid, Reason) ->
320    exit(Pid, Reason).
321
322-spec terminate_and_wait(Processes :: pid() | [pid()], Reason :: term()) -> ok.
323terminate_and_wait(Process, Reason) when is_pid(Process) ->
324    terminate_and_wait([Process], Reason);
325terminate_and_wait(Processes, Reason) ->
326    [terminate(P, Reason) || P <- Processes],
327    [misc:wait_for_process(P, infinity) || P <- Processes],
328    ok.
329
330-define(WAIT_FOR_NAME_SLEEP, 200).
331
332%% waits until given name is globally registered. I.e. until calling
333%% {via, leader_registry, Name} starts working
334wait_for_global_name(Name) ->
335    wait_for_global_name(Name, ?get_timeout(wait_for_global_name, 20000)).
336
337wait_for_global_name(Name, TimeoutMillis) ->
338    wait_for_name({via, leader_registry, Name}, TimeoutMillis).
339
340wait_for_local_name(Name, TimeoutMillis) ->
341    wait_for_name({local, Name}, TimeoutMillis).
342
343wait_for_name(Name, TimeoutMillis) ->
344    Tries = (TimeoutMillis + ?WAIT_FOR_NAME_SLEEP-1) div ?WAIT_FOR_NAME_SLEEP,
345    wait_for_name_loop(Name, Tries).
346
347wait_for_name_loop(Name, 0) ->
348    case is_pid(whereis_name(Name)) of
349        true ->
350            ok;
351        _ -> failed
352    end;
353wait_for_name_loop(Name, TriesLeft) ->
354    case is_pid(whereis_name(Name)) of
355        true ->
356            ok;
357        false ->
358            timer:sleep(?WAIT_FOR_NAME_SLEEP),
359            wait_for_name_loop(Name, TriesLeft-1)
360    end.
361
362whereis_name({global, Name}) ->
363    global:whereis_name(Name);
364whereis_name({local, Name}) ->
365    erlang:whereis(Name);
366whereis_name({via, Module, Name}) ->
367    Module:whereis_name(Name).
368
369
370%% Like proc_lib:start_link but allows to specify a node to spawn a process on.
371-spec start_link(node(), module(), atom(), [any()]) -> any() | {error, term()}.
372start_link(Node, M, F, A)
373  when is_atom(Node), is_atom(M), is_atom(F), is_list(A) ->
374    Pid = proc_lib:spawn_link(Node, M, F, A),
375    sync_wait(Pid).
376
377%% turns _this_ process into gen_server loop. Initializing and
378%% registering it properly.
379turn_into_gen_server({local, Name}, Mod, Args, GenServerOpts) ->
380    erlang:register(Name, self()),
381    {ok, State} = Mod:init(Args),
382    proc_lib:init_ack({ok, self()}),
383    gen_server:enter_loop(Mod, GenServerOpts, State, {local, Name}).
384
385sync_wait(Pid) ->
386    receive
387        {ack, Pid, Return} ->
388            Return;
389        {'EXIT', Pid, Reason} ->
390            {error, Reason}
391    end.
392
393spawn_monitor(F) ->
394    Start = make_ref(),
395    Parent = self(),
396
397    Pid = proc_lib:spawn(
398            fun () ->
399                    MRef = erlang:monitor(process, Parent),
400
401                    receive
402                        {'DOWN', MRef, process, Parent, Reason} ->
403                            exit(Reason);
404                        Start ->
405                            erlang:demonitor(MRef, [flush]),
406                            F()
407                    end
408            end),
409
410    MRef = erlang:monitor(process, Pid),
411    Pid ! Start,
412
413    {Pid, MRef}.
414
415poll_for_condition_rec(Condition, _Sleep, 0) ->
416    case Condition() of
417        false -> timeout;
418        Ret -> Ret
419    end;
420poll_for_condition_rec(Condition, Sleep, infinity) ->
421    case Condition() of
422        false ->
423            timer:sleep(Sleep),
424            poll_for_condition_rec(Condition, Sleep, infinity);
425        Ret -> Ret
426    end;
427poll_for_condition_rec(Condition, Sleep, Counter) ->
428    case Condition() of
429        false ->
430            timer:sleep(Sleep),
431            poll_for_condition_rec(Condition, Sleep, Counter-1);
432        Ret -> Ret
433    end.
434
435poll_for_condition(Condition, Timeout, Sleep) ->
436    Times = case Timeout of
437                infinity ->
438                    infinity;
439                _ ->
440                    (Timeout + Sleep - 1) div Sleep
441            end,
442    poll_for_condition_rec(Condition, Sleep, Times).
443
444poll_for_condition_test() ->
445    true = poll_for_condition(fun () -> true end, 0, 10),
446    timeout = poll_for_condition(fun () -> false end, 100, 10),
447    Ref = make_ref(),
448    self() ! {Ref, 0},
449    Fun  = fun() ->
450                   Counter = receive
451                                 {Ref, C} -> R = C + 1,
452                                             self() ! {Ref, R},
453                                             R
454                             after 0 ->
455                                 erlang:error(should_not_happen)
456                             end,
457                   Counter > 5
458           end,
459    true = poll_for_condition(Fun, 300, 10),
460    receive
461        {Ref, _} -> ok
462    after 0 ->
463            erlang:error(should_not_happen)
464    end.
465
466
467%% Remove matching messages from the inbox.
468%% Returns a count of messages removed.
469
470flush(Msg) -> ?flush(Msg).
471
472
473%% You know, like in Python
474enumerate(List) ->
475    enumerate(List, 1).
476
477enumerate([H|T], Start) ->
478    [{Start, H}|enumerate(T, Start + 1)];
479enumerate([], _) ->
480    [].
481
482
483%% Equivalent of sort|uniq -c
484uniqc(List) ->
485    uniqc(List, 1, []).
486
487uniqc([], _, Acc) ->
488    lists:reverse(Acc);
489uniqc([H], Count, Acc) ->
490    uniqc([], 0, [{H, Count}|Acc]);
491uniqc([H,H|T], Count, Acc) ->
492    uniqc([H|T], Count+1, Acc);
493uniqc([H1,H2|T], Count, Acc) ->
494    uniqc([H2|T], 1, [{H1, Count}|Acc]).
495
496uniqc_test() ->
497    [{a, 2}, {b, 5}] = uniqc([a, a, b, b, b, b, b]),
498    [] = uniqc([]),
499    [{c, 1}] = uniqc([c]).
500
501unique(Xs) ->
502    [X || {X, _} <- uniqc(Xs)].
503
504groupby_map(Fun, List) ->
505    Groups  = sort_and_keygroup(1, lists:map(Fun, List)),
506    [{Key, [X || {_, X} <- Group]} || {Key, Group} <- Groups].
507
508groupby(Fun, List) ->
509    groupby_map(?cut({Fun(_1), _1}), List).
510
511-ifdef(EUNIT).
512groupby_map_test() ->
513    List = [{a, 1}, {a, 2}, {b, 2}, {b, 3}],
514    ?assertEqual([{a, [1, 2]}, {b, [2, 3]}],
515                 groupby_map(fun functools:id/1, List)),
516
517    ?assertEqual([{a, [-1, -2]}, {b, [-2, -3]}],
518                 groupby_map(fun ({K, V}) ->
519                                     {K, -V}
520                             end, List)).
521
522groupby_test() ->
523    Groups = groupby(_ rem 2, lists:seq(0, 10)),
524
525    {0, [0,2,4,6,8,10]} = lists:keyfind(0, 1, Groups),
526    {1, [1,3,5,7,9]}    = lists:keyfind(1, 1, Groups).
527-endif.
528
529keygroup(Index, List) ->
530    keygroup(Index, List, []).
531
532keygroup(_, [], Groups) ->
533    lists:reverse(Groups);
534keygroup(Index, [H|T], Groups) ->
535    Key = element(Index, H),
536    {G, Rest} = lists:splitwith(fun (Elem) -> element(Index, Elem) == Key end, T),
537    keygroup(Index, Rest, [{Key, [H|G]}|Groups]).
538
539keygroup_test() ->
540    [{a, [{a, 1}, {a, 2}]},
541     {b, [{b, 2}, {b, 3}]}] = keygroup(1, [{a, 1}, {a, 2}, {b, 2}, {b, 3}]),
542    [] = keygroup(1, []).
543
544sort_and_keygroup(Index, List) ->
545    keygroup(Index, lists:keysort(Index, List)).
546
547%% Turn [[1, 2, 3], [4, 5, 6], [7, 8, 9]] info
548%% [[1, 4, 7], [2, 5, 8], [3, 6, 9]]
549rotate(List) ->
550    rotate(List, [], [], []).
551
552rotate([], [], [], Acc) ->
553    lists:reverse(Acc);
554rotate([], Heads, Tails, Acc) ->
555    rotate(lists:reverse(Tails), [], [], [lists:reverse(Heads)|Acc]);
556rotate([[H|T]|Rest], Heads, Tails, Acc) ->
557    rotate(Rest, [H|Heads], [T|Tails], Acc);
558rotate(_, [], [], Acc) ->
559    lists:reverse(Acc).
560
561rotate_test() ->
562    [[1, 4, 7], [2, 5, 8], [3, 6, 9]] =
563        rotate([[1, 2, 3], [4, 5, 6], [7, 8, 9]]),
564    [] = rotate([]).
565
566rewrite(Fun, Term) ->
567    generic:maybe_transform(
568      fun (T) ->
569              case Fun(T) of
570                  continue ->
571                      {continue, T};
572                  {stop, NewTerm} ->
573                      {stop, NewTerm}
574              end
575      end, Term).
576
577rewrite_correctly_callbacks_on_tuples_test() ->
578    executing_on_new_process(
579      fun () ->
580              {a, b, c} =
581                  rewrite(
582                    fun (Term) ->
583                            self() ! {term, Term},
584                            continue
585                    end, {a, b, c}),
586              Terms =
587                  letrec(
588                    [[]],
589                    fun (Rec, Acc) ->
590                            receive
591                                X ->
592                                    {term, T} = X,
593                                    Rec(Rec, [T | Acc])
594                            after 0 ->
595                                    lists:reverse(Acc)
596                            end
597                    end),
598              [{a, b, c}, a, b, c] = Terms
599      end).
600
601rewrite_value(Old, New, Struct) ->
602    generic:transformb(?transform(Old, New), Struct).
603
604rewrite_key_value_tuple(Key, NewValue, Struct) ->
605    generic:transformb(?transform({Key, _}, {Key, NewValue}), Struct).
606
607rewrite_tuples(Fun, Struct) ->
608    rewrite(
609      fun (Term) ->
610              case is_tuple(Term) of
611                  true ->
612                      Fun(Term);
613                  false ->
614                      continue
615              end
616      end,
617      Struct).
618
619rewrite_value_test() ->
620    x = rewrite_value(a, b, x),
621    b = rewrite_value(a, b, a),
622    b = rewrite_value(a, b, b),
623
624    [x, y, z] = rewrite_value(a, b, [x, y, z]),
625
626    [x, b, c, b] = rewrite_value(a, b, [x, a, c, a]),
627
628    {x, y} = rewrite_value(a, b, {x, y}),
629    {x, b} = rewrite_value(a, b, {x, a}),
630
631    X = rewrite_value(a, b,
632                      [ {"a string", 1, x},
633                        {"b string", 4, a, {blah, a, b}}]),
634    X = [{"a string", 1, x},
635         {"b string", 4, b, {blah, b, b}}],
636
637    % handling of improper list
638    [a, [x|c]] = rewrite_value(b, x, [a, [b|c]]),
639    [a, [b|x]] = rewrite_value(c, x, [a, [b|c]]).
640
641rewrite_key_value_tuple_test() ->
642    x = rewrite_key_value_tuple(a, b, x),
643    {a, b} = rewrite_key_value_tuple(a, b, {a, c}),
644    {b, x} = rewrite_key_value_tuple(a, b, {b, x}),
645
646    Orig = [ {"a string", x},
647             {"b string", 4, {a, x, y}, {a, c}, {a, [b, c]}}],
648    X = rewrite_key_value_tuple(a, b, Orig),
649    X = [{"a string", x},
650         {"b string", 4, {a, x, y}, {a, b}, {a, b}}],
651
652    X1 = rewrite_tuples(fun (T) ->
653                                case T of
654                                    {"a string", _} ->
655                                        {stop, {a_string, xxx}};
656                                    {a, _} ->
657                                        {stop, {a, xxx}};
658                                    _ ->
659                                        continue
660                                end
661                        end, Orig),
662
663    X1 = [{a_string, xxx}, {"b string", 4, {a, x, y}, {a, xxx}, {a, xxx}}].
664
665sanitize_url(Url) when is_binary(Url) ->
666    list_to_binary(sanitize_url(binary_to_list(Url)));
667sanitize_url(Url) when is_list(Url) ->
668    HostIndex = string:chr(Url, $@),
669    case HostIndex of
670        0 ->
671            Url;
672        _ ->
673            AfterScheme = string:str(Url, "://"),
674            case AfterScheme of
675                0 ->
676                    "*****" ++ string:substr(Url, HostIndex);
677                _ ->
678                    string:substr(Url, 1, AfterScheme + 2) ++ "*****" ++
679                        string:substr(Url, HostIndex)
680            end
681    end;
682sanitize_url(Url) ->
683    Url.
684
685sanitize_url_test() ->
686    "blah.com/a/b/c" = sanitize_url("blah.com/a/b/c"),
687    "ftp://blah.com" = sanitize_url("ftp://blah.com"),
688    "http://*****@blah.com" = sanitize_url("http://user:password@blah.com"),
689    "*****@blah.com" = sanitize_url("user:password@blah.com").
690
691ukeymergewith(Fun, N, L1, L2) ->
692    ukeymergewith(Fun, N, L1, L2, []).
693
694ukeymergewith(_, _, [], [], Out) ->
695    lists:reverse(Out);
696ukeymergewith(_, _, L1, [], Out) ->
697    lists:reverse(Out, L1);
698ukeymergewith(_, _, [], L2, Out) ->
699    lists:reverse(Out, L2);
700ukeymergewith(Fun, N, L1 = [T1|R1], L2 = [T2|R2], Out) ->
701    K1 = element(N, T1),
702    K2 = element(N, T2),
703    case K1 of
704        K2 ->
705            ukeymergewith(Fun, N, R1, R2, [Fun(T1, T2) | Out]);
706        K when K < K2 ->
707            ukeymergewith(Fun, N, R1, L2, [T1|Out]);
708        _ ->
709            ukeymergewith(Fun, N, L1, R2, [T2|Out])
710    end.
711
712ukeymergewith_test() ->
713    Fun = fun ({K, A}, {_, B}) ->
714                  {K, A + B}
715          end,
716    [{a, 3}] = ukeymergewith(Fun, 1, [{a, 1}], [{a, 2}]),
717    [{a, 3}, {b, 1}] = ukeymergewith(Fun, 1, [{a, 1}], [{a, 2}, {b, 1}]),
718    [{a, 1}, {b, 3}] = ukeymergewith(Fun, 1, [{b, 1}], [{a, 1}, {b, 2}]).
719
720
721%% Given two sorted lists, return a 3-tuple containing the elements
722%% that appear only in the first list, only in the second list, or are
723%% common to both lists, respectively.
724comm([H1|T1] = L1, [H2|T2] = L2) ->
725    if H1 == H2 ->
726            {R1, R2, R3} = comm(T1, T2),
727            {R1, R2, [H1|R3]};
728       H1 < H2 ->
729            {R1, R2, R3} = comm(T1, L2),
730            {[H1|R1], R2, R3};
731       true ->
732            {R1, R2, R3} = comm(L1, T2),
733            {R1, [H2|R2], R3}
734    end;
735comm(L1, L2) when L1 == []; L2 == [] ->
736    {L1, L2, []}.
737
738
739comm_test() ->
740    {[1], [2], [3]} = comm([1,3], [2,3]),
741    {[1,2,3], [], []} = comm([1,2,3], []),
742    {[], [], []} = comm([], []).
743
744
745
746start_singleton(Module, Name, Args, Opts) ->
747    case Module:start_link({via, leader_registry, Name}, Name, Args, Opts) of
748        {error, {already_started, Pid}} ->
749            ?log_warning("start_singleton(~p, ~p, ~p, ~p) -> already started:"
750                         " monitoring ~p from ~p",
751                         [Module, Name, Args, Opts, Pid, node()]),
752            {ok, spawn_link(fun () ->
753                                    misc:wait_for_process(Pid, infinity),
754                                    ?log_info("~p saw ~p exit (was pid ~p).",
755                                              [self(), Name, Pid])
756                            end)};
757        {ok, Pid} = X ->
758            ?log_info("start_singleton(~p, ~p, ~p, ~p):"
759                      " started as ~p on ~p~n",
760                      [Module, Name, Args, Opts, Pid, node()]),
761            X;
762        X -> X
763    end.
764
765
766%% Verify that a given global name belongs to the local pid, exiting
767%% if it doesn't.
768-spec verify_name(atom()) ->
769                         ok | no_return().
770verify_name(Name) ->
771    case leader_registry:whereis_name(Name) of
772        Pid when Pid == self() ->
773            ok;
774        Pid ->
775            ?log_error("~p is registered to ~p. Killing ~p.",
776                       [Name, Pid, self()]),
777            exit(kill)
778    end.
779
780
781key_update_rec(Key, List, Fun, Acc) ->
782    case List of
783        [{Key, OldValue} | Rest] ->
784            %% once we found our key, compute new value and don't recurse anymore
785            %% just append rest of list to reversed accumulator
786            lists:reverse([{Key, Fun(OldValue)} | Acc],
787                          Rest);
788        [] ->
789            %% if we reach here, then we didn't found our tuple
790            false;
791        [X | XX] ->
792            %% anything that's not our pair is just kept intact
793            key_update_rec(Key, XX, Fun, [X | Acc])
794    end.
795
796%% replace value of given Key with result of applying Fun on it in
797%% given proplist. Preserves order of keys. Assumes Key occurs only
798%% once.
799key_update(Key, PList, Fun) ->
800    key_update_rec(Key, PList, Fun, []).
801
802%% replace values from OldPList with values from NewPList
803update_proplist(OldPList, NewPList) ->
804    NewPList ++
805        lists:filter(fun ({K, _}) ->
806                             case lists:keyfind(K, 1, NewPList) of
807                                 false -> true;
808                                 _ -> false
809                             end
810                     end, OldPList).
811
812update_proplist_test() ->
813    [{a, 1}, {b, 2}, {c,3}] =:= update_proplist([{a,2}, {c,3}],
814                                                [{a,1}, {b,2}]).
815
816%% get proplist value or fail
817expect_prop_value(K, List) ->
818    Ref = make_ref(),
819    try
820        case proplists:get_value(K, List, Ref) of
821            RV when RV =/= Ref -> RV
822        end
823    catch
824        error:X -> erlang:error(X, [K, List])
825    end.
826
827%% true iff given path is absolute
828is_absolute_path(Path) ->
829    Normalized = filename:join([Path]),
830    filename:absname(Normalized) =:= Normalized.
831
832%% @doc Truncate a timestamp to the nearest multiple of N seconds.
833trunc_ts(TS, N) ->
834    TS - (TS rem (N*1000)).
835
836%% alternative of file:read_file/1 that reads file until EOF is
837%% reached instead of relying on file length. See
838%% http://groups.google.com/group/erlang-programming/browse_thread/thread/fd1ec67ff690d8eb
839%% for more information. This piece of code was borrowed from above mentioned URL.
840raw_read_file(Path) ->
841    case file:open(Path, [read, binary]) of
842        {ok, File} -> raw_read_loop(File, []);
843        Crap -> Crap
844    end.
845raw_read_loop(File, Acc) ->
846    case file:read(File, 16384) of
847        {ok, Bytes} ->
848            raw_read_loop(File, [Acc | Bytes]);
849        eof ->
850            file:close(File),
851            {ok, iolist_to_binary(Acc)};
852        {error, Reason} ->
853            file:close(File),
854            erlang:error(Reason)
855    end.
856
857assoc_multicall_results_rec([], _ResL, _BadNodes, SuccessAcc, ErrorAcc) ->
858    {SuccessAcc, ErrorAcc};
859assoc_multicall_results_rec([N | Nodes], Results, BadNodes,
860                            SuccessAcc, ErrorAcc) ->
861    case lists:member(N, BadNodes) of
862        true ->
863            assoc_multicall_results_rec(Nodes, Results, BadNodes, SuccessAcc, ErrorAcc);
864        _ ->
865            [Res | ResRest] = Results,
866
867            case Res of
868                {badrpc, Reason} ->
869                    NewErrAcc = [{N, Reason} | ErrorAcc],
870                    assoc_multicall_results_rec(Nodes, ResRest, BadNodes,
871                                                SuccessAcc, NewErrAcc);
872                _ ->
873                    NewOkAcc = [{N, Res} | SuccessAcc],
874                    assoc_multicall_results_rec(Nodes, ResRest, BadNodes,
875                                                NewOkAcc, ErrorAcc)
876            end
877    end.
878
879%% Returns a pair of proplists and list of nodes. First element is a
880%% mapping from Nodes to return values for nodes that
881%% succeeded. Second one is a mapping from Nodes to error reason for
882%% failed nodes. And third tuple element is BadNodes argument unchanged.
883-spec assoc_multicall_results([node()], [any() | {badrpc, any()}], [node()]) ->
884                                    {OkNodeResults::[{node(), any()}],
885                                     BadRPCNodeResults::[{node(), any()}],
886                                     BadNodes::[node()]}.
887assoc_multicall_results(Nodes, ResL, BadNodes) ->
888    {OkNodeResults, BadRPCNodeResults} = assoc_multicall_results_rec(Nodes, ResL, BadNodes, [], []),
889    {OkNodeResults, BadRPCNodeResults, BadNodes}.
890
891%% Performs rpc:multicall and massages results into "normal results",
892%% {badrpc, ...} results and timeouts/disconnects. Returns triple
893%% produced by assoc_multicall_results/3 above.
894rpc_multicall_with_plist_result(Nodes, M, F, A, Timeout) ->
895    {ResL, BadNodes} = rpc:multicall(Nodes, M, F, A, Timeout),
896    assoc_multicall_results(Nodes, ResL, BadNodes).
897
898rpc_multicall_with_plist_result(Nodes, M, F, A) ->
899    rpc_multicall_with_plist_result(Nodes, M, F, A, infinity).
900
901-spec realpath(string(), string()) -> {ok, string()} |
902                                      {error, atom(), string(), list(), any()}.
903realpath(Path, BaseDir) ->
904    case erlang:system_info(system_architecture) of
905        "win32" ->
906            {ok, filename:absname(Path, BaseDir)};
907        _ -> case realpath_full(Path, BaseDir, 32) of
908                 {ok, X, _} -> {ok, X};
909                 X -> X
910             end
911    end.
912
913-spec realpath_full(string(), string(), integer()) ->
914                           {ok, string(), integer()} |
915                           {error, atom(), string(), list(), any()}.
916realpath_full(Path, BaseDir, SymlinksLimit) ->
917    NormalizedPath = filename:join([Path]),
918    Tokens = string:tokens(NormalizedPath, "/"),
919    case Path of
920        [$/ | _] ->
921            %% if we're absolute path then start with root
922            realpath_rec_check("/", Tokens, SymlinksLimit);
923        _ ->
924            %% otherwise start walking from BaseDir
925            realpath_rec_info(#file_info{type = other}, BaseDir,
926                              Tokens, SymlinksLimit)
927    end.
928
929%% this is called to check type of Current pathname and expand
930%% it if it's symlink.
931-spec realpath_rec_check(string(), [string()], integer()) ->
932                                {ok, string(), integer()} |
933                                {error, atom(), string(), list(), any()}.
934realpath_rec_check(Current, Tokens, SymlinksLimit) ->
935    case file:read_link_info(Current) of
936        {ok, Info} ->
937            realpath_rec_info(Info, Current, Tokens, SymlinksLimit);
938        Crap -> {error, read_file_info, Current, Tokens, Crap}
939    end.
940
941%% this implements 'meat' of path name lookup
942-spec realpath_rec_info(tuple(), string(), [string()], integer()) ->
943                               {ok, string(), integer()} |
944                               {error, atom(), string(), list(), any()}.
945%% this case handles Current being symlink. Symlink is recursively
946%% expanded and we continue path name walking from expanded place
947realpath_rec_info(Info, Current, Tokens, SymlinksLimit)
948  when Info#file_info.type =:= symlink ->
949    case file:read_link(Current) of
950        {error, _} = Crap -> {error, read_link, Current, Tokens, Crap};
951        {ok, LinkDestination} ->
952            case SymlinksLimit of
953                0 -> {error, symlinks_limit_reached, Current, Tokens, undefined};
954                _ ->
955                    case realpath_full(LinkDestination,
956                                       filename:dirname(Current),
957                                       SymlinksLimit - 1) of
958                        {ok, Expanded, NewSymlinksLimit} ->
959                            realpath_rec_check(Expanded, Tokens,
960                                               NewSymlinksLimit);
961                        Error -> Error
962                    end
963            end
964    end;
965%% this case handles end of path name walking
966realpath_rec_info(_, Current, [], SymlinksLimit) ->
967    {ok, Current, SymlinksLimit};
968%% this case just removes single dot
969realpath_rec_info(Info, Current, ["." | Tokens], SymlinksLimit) ->
970    realpath_rec_info(Info, Current, Tokens, SymlinksLimit);
971%% this case implements ".."
972realpath_rec_info(_Info, Current, [".." | Tokens], SymlinksLimit) ->
973    realpath_rec_check(filename:dirname(Current), Tokens, SymlinksLimit);
974%% this handles most common case of walking single level of file tree
975realpath_rec_info(_Info, Current, [FirstToken | Tokens], SymlinksLimit) ->
976    NewCurrent = filename:absname(FirstToken, Current),
977    realpath_rec_check(NewCurrent, Tokens, SymlinksLimit).
978
979-spec split_binary_at_char(binary(), char()) -> binary() | {binary(), binary()}.
980split_binary_at_char(Binary, Chr) ->
981    case binary:split(Binary, <<Chr:8>>) of
982        [_] -> Binary;
983        [Part1, Part2] -> {Part1, Part2}
984    end.
985
986is_binary_ends_with(Binary, Suffix) ->
987    binary:longest_common_suffix([Binary, Suffix]) =:= size(Suffix).
988
989absname(Name) ->
990    PathType = filename:pathtype(Name),
991    case PathType of
992        absolute ->
993            filename:absname(Name, "/");
994        _ ->
995            filename:absname(Name)
996    end.
997
998start_event_link(SubscriptionBody) ->
999    {ok,
1000     spawn_link(fun () ->
1001                        SubscriptionBody(),
1002                        receive
1003                            _ -> ok
1004                        end
1005                end)}.
1006
1007%% Writes to file atomically using write_file + atomic_rename
1008atomic_write_file(Path, BodyOrBytes)
1009  when is_function(BodyOrBytes);
1010       is_binary(BodyOrBytes);
1011       is_list(BodyOrBytes) ->
1012    DirName = filename:dirname(Path),
1013    FileName = filename:basename(Path),
1014    TmpPath = path_config:tempfile(DirName, FileName, ".tmp"),
1015    try
1016        case misc:write_file(TmpPath, BodyOrBytes) of
1017            ok ->
1018                atomic_rename(TmpPath, Path);
1019            X ->
1020                X
1021        end
1022    after
1023        (catch file:delete(TmpPath))
1024    end.
1025
1026%% Rename file (more or less) atomically.
1027%% See https://lwn.net/Articles/351422/ for some details.
1028%%
1029%% NB: this does not work on Windows
1030%% (http://osdir.com/ml/racket.development/2011-01/msg00149.html).
1031atomic_rename(From, To) ->
1032    case file:open(From, [raw, binary, read, write]) of
1033        {ok, IO} ->
1034            SyncRV =
1035                try
1036                    file:sync(IO)
1037                after
1038                    ok = file:close(IO)
1039                end,
1040            case SyncRV of
1041                ok ->
1042                    %% NOTE: linux manpages also mention sync
1043                    %% on directory, but erlang can't do that
1044                    %% and that's not portable
1045                    file:rename(From, To);
1046                _ ->
1047                    SyncRV
1048            end;
1049        Err ->
1050            Err
1051    end.
1052
1053%% Get an item from from a dict, if it doesnt exist return default
1054-spec dict_get(term(), dict(), term()) -> term().
1055dict_get(Key, Dict, Default) ->
1056    case dict:find(Key, Dict) of
1057        {ok, Value} ->
1058            Value;
1059        error ->
1060            Default
1061    end.
1062
1063%% like dict:update/4 but calls the function on initial value instead of just
1064%% storing it in the dict
1065dict_update(Key, Fun, Initial, Dict) ->
1066    try
1067        dict:update(Key, Fun, Dict)
1068    catch
1069        %% key not found
1070        error:badarg ->
1071            dict:store(Key, Fun(Initial), Dict)
1072    end.
1073
1074%% Parse version of the form 1.7.0r_252_g1e1c2c0 or 1.7.0r-252-g1e1c2c0 into a
1075%% list {[1,7,0],candidate,252}.  1.8.0 introduces a license type suffix,
1076%% like: 1.8.0r-25-g1e1c2c0-enterprise.  Note that we should never
1077%% see something like 1.7.0-enterprise, as older nodes won't carry
1078%% the license type information.
1079-spec parse_version(string()) -> version().
1080parse_version(VersionStr) ->
1081    Parts = string:tokens(VersionStr, "_-"),
1082    case Parts of
1083        [BaseVersionStr] ->
1084            {BaseVersion, Type} = parse_base_version(BaseVersionStr),
1085            {BaseVersion, Type, 0};
1086        [BaseVersionStr, OffsetStr | _Rest] ->
1087            {BaseVersion, Type} = parse_base_version(BaseVersionStr),
1088            {BaseVersion, Type, list_to_integer(OffsetStr)}
1089    end.
1090
1091-define(VERSION_REGEXP,
1092        "^((?:[0-9]+\.)*[0-9]+).*?(r)?$"). % unbreak font-lock "
1093
1094parse_base_version(BaseVersionStr) ->
1095    case re:run(BaseVersionStr, ?VERSION_REGEXP,
1096                [{capture, all_but_first, list}]) of
1097        {match, [NumericVersion, "r"]} ->
1098            Type = candidate;
1099        {match, [NumericVersion]} ->
1100            Type = release
1101    end,
1102
1103    {lists:map(fun list_to_integer/1,
1104               string:tokens(NumericVersion, ".")), Type}.
1105
1106this_node_rest_port() ->
1107    node_rest_port(node()).
1108
1109node_rest_port(Node) ->
1110    node_rest_port(ns_config:latest(), Node).
1111
1112node_rest_port(Config, Node) ->
1113    case ns_config:search_node_prop(Node, Config, rest, port_meta, local) of
1114        local ->
1115            ns_config:search_node_prop(Node, Config, rest, port, 8091);
1116        global ->
1117            ns_config:search_prop(Config, rest, port, 8091)
1118    end.
1119
1120-ifdef(EUNIT).
1121parse_version_test() ->
1122    ?assertEqual({[1,7,0],release,252},
1123                 parse_version("1.7.0_252_g1e1c2c0")),
1124    ?assertEqual({[1,7,0],release,252},
1125                 parse_version("1.7.0-252-g1e1c2c0")),
1126    ?assertEqual({[1,7,0],candidate,252},
1127                 parse_version("1.7.0r_252_g1e1c2c0")),
1128    ?assertEqual({[1,7,0],candidate,252},
1129                 parse_version("1.7.0r-252-g1e1c2c0")),
1130    ?assertEqual({[1,7,0],release,0},
1131                 parse_version("1.7.0")),
1132    ?assertEqual({[1,7,0],candidate,0},
1133                 parse_version("1.7.0r")),
1134    ?assertEqual(true,
1135                 parse_version("1.7.0") >
1136                     parse_version("1.7.0r_252_g1e1c2c0")),
1137    ?assertEqual(true,
1138                 parse_version("1.7.0") >
1139                     parse_version("1.7.0r")),
1140    ?assertEqual(true,
1141                 parse_version("1.7.1r") >
1142                     parse_version("1.7.0")),
1143    ?assertEqual(true,
1144                 parse_version("1.7.1_252_g1e1c2c0") >
1145                     parse_version("1.7.1_251_g1e1c2c1")),
1146    ?assertEqual({[1,8,0],release,25},
1147                 parse_version("1.8.0_25_g1e1c2c0-enterprise")),
1148    ?assertEqual({[1,8,0],release,25},
1149                 parse_version("1.8.0-25-g1e1c2c0-enterprise")),
1150    ?assertEqual({[2,0,0],candidate,702},
1151                 parse_version("2.0.0dp4r-702")),
1152    ?assertEqual({[2,0,0],candidate,702},
1153                 parse_version("2.0.0dp4r-702-g1e1c2c0")),
1154    ?assertEqual({[2,0,0],candidate,702},
1155                 parse_version("2.0.0dp4r-702-g1e1c2c0-enterprise")),
1156    ?assertEqual({[2,0,0],release,702},
1157                 parse_version("2.0.0dp4-702")),
1158    ?assertEqual({[2,0,0],release,702},
1159                 parse_version("2.0.0dp4-702-g1e1c2c0")),
1160    ?assertEqual({[2,0,0],release,702},
1161                 parse_version("2.0.0dp4-702-g1e1c2c0-enterprise")),
1162    ok.
1163
1164ceiling_test() ->
1165    ?assertEqual(4, ceiling(4)),
1166    ?assertEqual(4, ceiling(4.0)),
1167    ?assertEqual(4, ceiling(3.99)),
1168    ?assertEqual(4, ceiling(3.01)),
1169    ?assertEqual(-4, ceiling(-4)),
1170    ?assertEqual(-4, ceiling(-4.0)),
1171    ?assertEqual(-4, ceiling(-4.99)),
1172    ?assertEqual(-4, ceiling(-4.01)),
1173    ok.
1174
1175-endif.
1176
1177compute_map_diff(undefined, OldMap) ->
1178    compute_map_diff([], OldMap);
1179compute_map_diff(NewMap, undefined) ->
1180    compute_map_diff(NewMap, []);
1181compute_map_diff([], []) ->
1182    [];
1183compute_map_diff(NewMap, []) when NewMap =/= [] ->
1184    compute_map_diff(NewMap, [[] || _ <- NewMap]);
1185compute_map_diff([], OldMap) when OldMap =/= [] ->
1186    compute_map_diff([[] || _ <- OldMap], OldMap);
1187compute_map_diff(NewMap, OldMap) ->
1188    VBucketsCount = erlang:length(NewMap),
1189    [{I, ChainOld, ChainNew} ||
1190        {I, ChainOld, ChainNew} <-
1191            lists:zip3(lists:seq(0, VBucketsCount-1), OldMap, NewMap),
1192        ChainOld =/= ChainNew].
1193
1194%% execute body in newly spawned process. Function returns when Body
1195%% returns and with it's return value. If body produced any exception
1196%% it will be rethrown. Care is taken to propagate exits of 'parent'
1197%% process to this worker process.
1198executing_on_new_process(Fun) ->
1199    executing_on_new_process(Fun, []).
1200
1201executing_on_new_process(Fun, Options) ->
1202    {trap_exit, TrapExit} = process_info(self(), trap_exit),
1203
1204    StartOptions = executing_on_new_process_handle_options(Options),
1205    executing_on_new_process_handle_trap_exit(TrapExit, Fun, StartOptions).
1206
1207executing_on_new_process_handle_options(Options) ->
1208    PassThrough = [abort_after],
1209    proplist_keyfilter(lists:member(_, PassThrough), Options).
1210
1211executing_on_new_process_handle_trap_exit(true, Fun, StartOptions) ->
1212    %% If the caller had trap_exit set, we can't really make the execution
1213    %% interruptlible, after all it was, hopefully, a deliberate choice to set
1214    %% trap_exit, so we need to abide by it.
1215    executing_on_new_process_body(Fun, StartOptions, []);
1216executing_on_new_process_handle_trap_exit(false, Fun, StartOptions) ->
1217    with_trap_exit(?cut(executing_on_new_process_body(
1218                          Fun, StartOptions, [interruptible]))).
1219
1220executing_on_new_process_body(Fun, StartOptions, WaitOptions) ->
1221    async:with(
1222      Fun, StartOptions,
1223      fun (A) ->
1224              try
1225                  async:wait(A, WaitOptions)
1226              catch
1227                  throw:{interrupted, {'EXIT', _, Reason} = Exit} ->
1228                      true = proplists:get_bool(interruptible, WaitOptions),
1229
1230                      ?log_debug("Aborting ~p (body is ~p) because "
1231                                 "we are interrupted by an exit message ~p",
1232                                 [A, Fun, Exit]),
1233
1234                      async:abort(A, Reason),
1235                      %% will be processed by the with_trap_exit
1236                      self() ! Exit
1237              end
1238      end).
1239
1240-ifdef(EUNIT).
1241executing_on_new_process_test() ->
1242    lists:foreach(
1243      fun (_) ->
1244              P = spawn(?cut(misc:executing_on_new_process(
1245                               fun () ->
1246                                       register(grandchild, self()),
1247                                       timer:sleep(3600 * 1000)
1248                               end))),
1249              timer:sleep(random:uniform(5) - 1),
1250              exit(P, shutdown),
1251              ok = wait_for_process(P, 500),
1252              undefined = whereis(grandchild)
1253      end, lists:seq(1, 1000)).
1254
1255%% Check that exit signals are propagated without any mangling.
1256executing_on_new_process_exit_test() ->
1257    try
1258        misc:executing_on_new_process(?cut(exit(shutdown)))
1259    catch
1260        exit:shutdown ->
1261            ok
1262    end.
1263
1264executing_on_new_process_abort_after_test() ->
1265    ?assertExit(timeout,
1266                misc:executing_on_new_process(
1267                  fun () ->
1268                          register(child, self()),
1269                          timer:sleep(10000)
1270                  end,
1271                  [{abort_after, 100}])),
1272
1273    %% the child must be dead
1274    undefined = whereis(child),
1275
1276    ok = misc:executing_on_new_process(?cut(timer:sleep(100)),
1277                                       [{abort_after, 1000}]),
1278
1279
1280    %% must be no messages left in mailbox
1281    0 = ?flush(_).
1282
1283-endif.
1284
1285%% returns if Reason is EXIT caused by undefined function/module
1286is_undef_exit(M, F, A, {undef, [{M, F, A, []} | _]}) -> true; % R15, R16
1287is_undef_exit(M, F, A, {undef, [{M, F, A} | _]}) -> true; % R14
1288is_undef_exit(_M, _F, _A, _Reason) -> false.
1289
1290is_timeout_exit({'EXIT', timeout}) -> true;
1291is_timeout_exit({'EXIT', {timeout, _}}) -> true;
1292is_timeout_exit(_) -> false.
1293
1294-spec sync_shutdown_many_i_am_trapping_exits(Pids :: [pid()]) -> ok.
1295sync_shutdown_many_i_am_trapping_exits(Pids) ->
1296    {trap_exit, true} = erlang:process_info(self(), trap_exit),
1297    [(catch erlang:exit(Pid, shutdown)) || Pid <- Pids],
1298    BadShutdowns = [{P, RV} || P <- Pids,
1299                               (RV = inner_wait_shutdown(P)) =/= shutdown],
1300    case BadShutdowns of
1301        [] -> ok;
1302        _ ->
1303            ?log_error("Shutdown of the following failed: ~p", [BadShutdowns])
1304    end,
1305    [] = BadShutdowns,
1306    ok.
1307
1308%% NOTE: this is internal helper, despite everything being exported
1309%% from here
1310-spec inner_wait_shutdown(Pid :: pid()) -> term().
1311inner_wait_shutdown(Pid) ->
1312    MRef = erlang:monitor(process, Pid),
1313    MRefReason = receive
1314                     {'DOWN', MRef, _, _, MRefReason0} ->
1315                         MRefReason0
1316                 end,
1317    receive
1318        {'EXIT', Pid, Reason} ->
1319            Reason
1320    after 5000 ->
1321            ?log_error("Expected exit signal from ~p but could not get it in 5 seconds. This is a bug, but process we're waiting for is dead (~p), so trying to ignore...", [Pid, MRefReason]),
1322            ?log_debug("Here's messages:~n~p", [erlang:process_info(self(), messages)]),
1323            MRefReason
1324    end.
1325
1326%% @doc works like try/after but when try has raised exception, any
1327%% exception from AfterBody is logged and ignored. I.e. when we face
1328%% exceptions from both try-block and after-block, exception from
1329%% after-block is logged and ignored and exception from try-block is
1330%% rethrown. Use this when exception from TryBody is more important
1331%% than exception from AfterBody.
1332try_with_maybe_ignorant_after(TryBody, AfterBody) ->
1333    RV =
1334        try TryBody()
1335        catch T:E ->
1336                Stacktrace = erlang:get_stacktrace(),
1337                try AfterBody()
1338                catch T2:E2 ->
1339                        ?log_error("Eating exception from ignorant after-block:~n~p", [{T2, E2, erlang:get_stacktrace()}])
1340                end,
1341                erlang:raise(T, E, Stacktrace)
1342        end,
1343    AfterBody(),
1344    RV.
1345
1346letrec(Args, F) ->
1347    erlang:apply(F, [F | Args]).
1348
1349-spec is_ipv6() -> true | false.
1350is_ipv6() ->
1351    get_env_default(ns_server, ipv6, false).
1352
1353-spec get_net_family() -> inet:address_family().
1354get_net_family() ->
1355    case is_ipv6() of
1356        true ->
1357            inet6;
1358        false ->
1359            inet
1360    end.
1361
1362-spec get_proto_dist_type() -> string().
1363get_proto_dist_type() ->
1364    case is_ipv6() of
1365        true ->
1366            "inet6_tcp";
1367        false ->
1368            "inet_tcp"
1369    end.
1370
1371-spec localhost() -> string().
1372localhost() ->
1373    localhost([]).
1374
1375-spec localhost([] | [url]) -> string().
1376localhost(Options) ->
1377    case is_ipv6() of
1378        true ->
1379            case Options of
1380                [] ->
1381                    "::1";
1382                [url] ->
1383                    "[::1]"
1384            end;
1385        false ->
1386            "127.0.0.1"
1387    end.
1388
1389-spec inaddr_any() -> string().
1390inaddr_any() ->
1391    inaddr_any([]).
1392
1393-spec inaddr_any([] | [url]) -> string().
1394inaddr_any(Options) ->
1395    case is_ipv6() of
1396        true ->
1397            case Options of
1398                [] ->
1399                    "::";
1400                [url] ->
1401                    "[::]"
1402            end;
1403        false ->
1404            "0.0.0.0"
1405    end.
1406
1407-spec local_url(integer(),
1408                [] | [no_scheme |
1409                      {user_info, {string(), string()}}]) -> string().
1410local_url(Port, Options) ->
1411    local_url(Port, "", Options).
1412
1413-spec local_url(integer(), string(),
1414                [] | [no_scheme |
1415                      {user_info, {string(), string()}}]) -> string().
1416local_url(Port, [H | _] = Path, Options) when H =/= $/ ->
1417    local_url(Port, "/" ++ Path, Options);
1418local_url(Port, Path, Options) ->
1419    Scheme = case lists:member(no_scheme, Options) of
1420                 true -> "";
1421                 false -> "http://"
1422             end,
1423    User = case lists:keysearch(user_info, 1, Options) of
1424               false -> "";
1425               {value, {_, {U, P}}} -> U ++ ":" ++ P ++ "@"
1426           end,
1427    Scheme ++ User ++ localhost([url]) ++ ":" ++ integer_to_list(Port) ++ Path.
1428
1429-spec is_good_address(string()) -> ok | {cannot_resolve, inet:posix()}
1430                                       | {cannot_listen, inet:posix()}
1431                                       | {address_not_allowed, string()}.
1432is_good_address(Address) ->
1433    is_good_address(Address, is_ipv6()).
1434
1435is_good_address(Address, false) ->
1436    check_short_name(Address, ".");
1437is_good_address(Address, true) ->
1438    case inet:getaddr(Address, inet6) of
1439        {error, Err} ->
1440            Msg = io_lib:format("~s doesn't look to refer to a valid IPv6 address as it doesn't "
1441                                "resolve. (Posix error code: '~p')", [Address, Err]),
1442            {address_not_allowed, Msg};
1443        _ ->
1444            check_short_name(Address, ".:")
1445    end.
1446
1447check_short_name(Address, Separators) ->
1448    case lists:subtract(Address, Separators) of
1449        Address ->
1450            {address_not_allowed,
1451             "Short names are not allowed. Please use a Fully Qualified Domain Name."};
1452        _ ->
1453            is_good_address_when_allowed(Address)
1454    end.
1455
1456is_good_address_when_allowed(Address) ->
1457    NetFamily = get_net_family(),
1458    case inet:getaddr(Address, NetFamily) of
1459        {error, Errno} ->
1460            {cannot_resolve, Errno};
1461        {ok, IpAddr} ->
1462            case gen_udp:open(0, [NetFamily, {ip, IpAddr}]) of
1463                {error, Errno} ->
1464                    {cannot_listen, Errno};
1465                {ok, Socket} ->
1466                    gen_udp:close(Socket),
1467                    ok
1468            end
1469    end.
1470
1471is_local_port_open(Port, Timeout) ->
1472    case gen_tcp:connect(localhost(), Port, [get_net_family()], Timeout) of
1473        {ok, Socket} ->
1474            gen_tcp:close(Socket),
1475            true;
1476        {error, _} ->
1477            false
1478    end.
1479
1480delaying_crash(DelayBy, Body) ->
1481    try
1482        Body()
1483    catch T:E ->
1484            ST = erlang:get_stacktrace(),
1485            ?log_debug("Delaying crash ~p:~p by ~pms~nStacktrace: ~p", [T, E, DelayBy, ST]),
1486            timer:sleep(DelayBy),
1487            erlang:raise(T, E, ST)
1488    end.
1489
1490%% Like erlang:memory but returns 'notsup' if it's impossible to get this
1491%% information.
1492memory() ->
1493    try
1494        erlang:memory()
1495    catch
1496        error:notsup ->
1497            notsup
1498    end.
1499
1500%% Wraps erlang:system_info(logical_processors).
1501cpu_count() ->
1502    erlang:system_info(logical_processors).
1503
1504ensure_writable_dir(Path) ->
1505    filelib:ensure_dir(Path),
1506    case filelib:is_dir(Path) of
1507        true ->
1508            TouchPath = filename:join(Path, ".touch"),
1509            case misc:write_file(TouchPath, <<"">>) of
1510                ok ->
1511                    file:delete(TouchPath),
1512                    ok;
1513                _ -> error
1514            end;
1515        _ ->
1516            case file:make_dir(Path) of
1517                ok -> ok;
1518                _ ->
1519                    error
1520            end
1521    end.
1522
1523ensure_writable_dirs([]) ->
1524    ok;
1525ensure_writable_dirs([Path | Rest]) ->
1526    case ensure_writable_dir(Path) of
1527        ok ->
1528            ensure_writable_dirs(Rest);
1529        X -> X
1530    end.
1531
1532%% Like lists:split but does not fail if N > length(List).
1533safe_split(N, List) ->
1534    do_safe_split(N, List, []).
1535
1536do_safe_split(_, [], Acc) ->
1537    {lists:reverse(Acc), []};
1538do_safe_split(0, List, Acc) ->
1539    {lists:reverse(Acc), List};
1540do_safe_split(N, [H|T], Acc) ->
1541    do_safe_split(N - 1, T, [H|Acc]).
1542
1543-spec run_external_tool(string(), [string()]) -> {non_neg_integer(), binary()}.
1544run_external_tool(Path, Args) ->
1545    run_external_tool(Path, Args, []).
1546
1547-spec run_external_tool(string(), [string()], [{string(), string()}]) -> {non_neg_integer(), binary()}.
1548run_external_tool(Path, Args, Env) ->
1549    executing_on_new_process(
1550      fun () ->
1551              Port = erlang:open_port({spawn_executable, Path},
1552                                      [stderr_to_stdout, binary,
1553                                       stream, exit_status, hide,
1554                                       {args, Args},
1555                                       {env, Env}]),
1556              collect_external_tool_output(Port, [])
1557      end).
1558
1559collect_external_tool_output(Port, Acc) ->
1560    receive
1561        {Port, {data, Data}} ->
1562            collect_external_tool_output(Port, [Data | Acc]);
1563        {Port, {exit_status, Status}} ->
1564            {Status, iolist_to_binary(lists:reverse(Acc))};
1565        Msg ->
1566            ?log_error("Got unexpected message"),
1567            exit({unexpected_message, Msg})
1568    end.
1569
1570min_by(Less, Items) ->
1571    lists:foldl(
1572      fun (Elem, Acc) ->
1573              case Less(Elem, Acc) of
1574                  true ->
1575                      Elem;
1576                  false ->
1577                      Acc
1578              end
1579      end, hd(Items), tl(Items)).
1580
1581inspect_term(Value) ->
1582    binary_to_list(iolist_to_binary(io_lib:format("~p", [Value]))).
1583
1584with_file(Path, Mode, Body) ->
1585    case file:open(Path, Mode) of
1586        {ok, F} ->
1587            try
1588                Body(F)
1589            after
1590                (catch file:close(F))
1591            end;
1592        Error ->
1593            Error
1594    end.
1595
1596write_file(Path, Bytes) when is_binary(Bytes); is_list(Bytes) ->
1597    misc:write_file(Path,
1598                    fun (F) ->
1599                            file:write(F, Bytes)
1600                    end);
1601write_file(Path, Body) when is_function(Body) ->
1602    with_file(Path, [raw, binary, write], Body).
1603
1604halt(Status) ->
1605    try
1606        erlang:halt(Status, [{flush, false}])
1607    catch
1608        error:undef ->
1609            erlang:halt(Status)
1610    end.
1611
1612%% Ensure that directory exists. Analogous to running mkdir -p in shell.
1613mkdir_p(Path) ->
1614    case filelib:ensure_dir(Path) of
1615        ok ->
1616            case file:make_dir(Path) of
1617                {error, eexist} ->
1618                    case file:read_file_info(Path) of
1619                        {ok, Info} ->
1620                            case Info#file_info.type of
1621                                directory ->
1622                                    ok;
1623                                _ ->
1624                                    {error, eexist}
1625                            end;
1626                        Error ->
1627                            Error
1628                    end;
1629                %% either ok or other error
1630                Other ->
1631                    Other
1632            end;
1633        Error ->
1634            Error
1635    end.
1636
1637create_marker(Path, Data)
1638  when is_list(Data);
1639       is_binary(Data) ->
1640    ok = atomic_write_file(Path, Data).
1641
1642create_marker(Path) ->
1643    create_marker(Path, <<"">>).
1644
1645remove_marker(Path) ->
1646    ok = file:delete(Path).
1647
1648marker_exists(Path) ->
1649    case file:read_file_info(Path) of
1650        {ok, _} ->
1651            true;
1652        {error, enoent} ->
1653            false;
1654        Other ->
1655            ?log_error("Unexpected error when reading marker ~p: ~p", [Path, Other]),
1656            exit({failed_to_read_marker, Path, Other})
1657    end.
1658
1659read_marker(Path) ->
1660    case file:read_file(Path) of
1661        {ok, BinaryContents} ->
1662            {ok, binary_to_list(BinaryContents)};
1663        {error, enoent} ->
1664            false;
1665        Other ->
1666            ?log_error("Unexpected error when reading marker ~p: ~p", [Path, Other]),
1667            exit({failed_to_read_marker, Path, Other})
1668    end.
1669
1670take_marker(Path) ->
1671    Result = read_marker(Path),
1672    case Result of
1673        {ok, _} ->
1674            remove_marker(Path);
1675        false ->
1676            ok
1677    end,
1678
1679    Result.
1680
1681is_free_nodename(ShortName) ->
1682    {ok, Names} = erl_epmd:names({127,0,0,1}),
1683    not lists:keymember(ShortName, 1, Names).
1684
1685wait_for_nodename(ShortName) ->
1686    wait_for_nodename(ShortName, 5).
1687
1688wait_for_nodename(ShortName, Attempts) ->
1689    case is_free_nodename(ShortName) of
1690        true ->
1691            ok;
1692        false ->
1693            case Attempts of
1694                0 ->
1695                    {error, duplicate_name};
1696                _ ->
1697                    ?log_info("Short name ~s is still occupied. "
1698                              "Will try again after a bit", [ShortName]),
1699                    timer:sleep(500),
1700                    wait_for_nodename(ShortName, Attempts - 1)
1701            end
1702    end.
1703
1704is_prefix(KeyPattern, K) ->
1705    KPL = size(KeyPattern),
1706    case K of
1707        <<KeyPattern:KPL/binary, _/binary>> ->
1708            true;
1709        _ ->
1710            false
1711    end.
1712
1713eval(Str,Binding) ->
1714    {ok,Ts,_} = erl_scan:string(Str),
1715    Ts1 = case lists:reverse(Ts) of
1716              [{dot,_}|_] -> Ts;
1717              TsR -> lists:reverse([{dot,1} | TsR])
1718          end,
1719    {ok,Expr} = erl_parse:parse_exprs(Ts1),
1720    erl_eval:exprs(Expr, Binding).
1721
1722get_ancestors() ->
1723    erlang:get('$ancestors').
1724
1725multi_call(Nodes, Name, Request, Timeout) ->
1726    multi_call(Nodes, Name, Request, Timeout, fun (_) -> true end).
1727
1728%% Behaves like gen_server:multi_call except that instead of just returning a
1729%% list of "bad nodes" it returns some details about why a call failed.
1730%%
1731%% In addition it takes a predicate that can classify an ok reply as an
1732%% error. Such a reply will be put into the bad replies list.
1733-spec multi_call(Nodes, Name, Request, Timeout, OkPred) -> Result
1734  when Nodes   :: [node()],
1735       Name    :: atom(),
1736       Request :: any(),
1737       Timeout :: infinity | non_neg_integer(),
1738       Result  :: {Good, Bad},
1739       Good    :: [{node(), any()}],
1740       Bad     :: [{node(), any()}],
1741
1742       OkPred   :: fun((any()) -> OkPredRV),
1743       OkPredRV :: boolean() | {false, ErrorTerm :: term()}.
1744multi_call(Nodes, Name, Request, Timeout, OkPred) ->
1745    Ref = erlang:make_ref(),
1746    Parent = self(),
1747    try
1748        parallel_map(
1749          fun (N) ->
1750                  RV = try gen_server:call({Name, N}, Request, infinity) of
1751                           Res ->
1752                               case OkPred(Res) of
1753                                   true ->
1754                                       {ok, Res};
1755                                   false ->
1756                                       {error, Res};
1757                                   {false, ErrorTerm} ->
1758                                       {error, ErrorTerm}
1759                               end
1760                       catch T:E ->
1761                               {error, {T, E}}
1762                       end,
1763                  Parent ! {Ref, {N, RV}}
1764          end, Nodes, Timeout)
1765    catch exit:timeout ->
1766            ok
1767    end,
1768    multi_call_collect(ordsets:from_list(Nodes), [], [], [], Ref).
1769
1770multi_call_collect(Nodes, GotNodes, AccGood, AccBad, Ref) ->
1771    receive
1772        {Ref, {N, Result}} ->
1773            {NewGood, NewBad} =
1774                case Result of
1775                    {ok, Good} ->
1776                        {[{N, Good} | AccGood], AccBad};
1777                    {error, Bad} ->
1778                        {AccGood, [{N, Bad} | AccBad]}
1779                end,
1780
1781            multi_call_collect(Nodes, [N | GotNodes], NewGood, NewBad, Ref)
1782    after 0 ->
1783            TimeoutNodes = ordsets:subtract(Nodes, ordsets:from_list(GotNodes)),
1784            BadNodes = [{N, timeout} || N <- TimeoutNodes] ++ AccBad,
1785            {AccGood, BadNodes}
1786    end.
1787
1788-ifdef(EUNIT).
1789
1790multi_call_test_() ->
1791    {setup, fun multi_call_test_setup/0, fun multi_call_test_teardown/1,
1792     [fun do_test_multi_call/0]}.
1793
1794multi_call_test_setup_server() ->
1795    meck:new(multi_call_server, [non_strict, no_link]),
1796    meck:expect(multi_call_server, init, fun([]) -> {ok, {}} end),
1797    meck:expect(multi_call_server, handle_call,
1798                fun(Request, _From, _State) ->
1799                        Reply = case Request of
1800                                    {echo, V} ->
1801                                        V;
1802                                    {sleep, Time} ->
1803                                        timer:sleep(Time);
1804                                    {eval, Fun} ->
1805                                        Fun()
1806                                end,
1807                        {reply, Reply, {}}
1808                end),
1809    {ok, _} = gen_server:start_link({local, multi_call_server}, multi_call_server, [], []),
1810    ok.
1811
1812multi_call_test_setup() ->
1813    NodeNames = [a, b, c, d, e],
1814    {TestNode, Host0} = misc:node_name_host(node()),
1815    Host = list_to_atom(Host0),
1816
1817    CodePath = code:get_path(),
1818    Nodes = lists:map(
1819              fun (N) ->
1820                      FullName = list_to_atom(atom_to_list(N) ++ "-" ++ TestNode),
1821                      {ok, Node} = slave:start(Host, FullName),
1822                      true = rpc:call(Node, code, set_path, [CodePath]),
1823                      ok = rpc:call(Node, misc, multi_call_test_setup_server, []),
1824                      Node
1825              end, NodeNames),
1826    erlang:put(nodes, Nodes).
1827
1828multi_call_test_teardown(_) ->
1829    Nodes = erlang:get(nodes),
1830    lists:foreach(
1831      fun (Node) ->
1832              ok = slave:stop(Node)
1833      end, Nodes).
1834
1835multi_call_test_assert_bad_nodes(Bad, Expected) ->
1836    BadNodes = [N || {N, _} <- Bad],
1837    ?assertEqual(lists:sort(BadNodes), lists:sort(Expected)).
1838
1839multi_call_test_assert_results(RVs, Nodes, Result) ->
1840    lists:foreach(
1841      fun (N) ->
1842              RV = proplists:get_value(N, RVs),
1843              ?assertEqual(RV, Result)
1844      end, Nodes).
1845
1846do_test_multi_call() ->
1847    Nodes = nodes(),
1848    BadNodes = [bad_node],
1849
1850    {R1, Bad1} = misc:multi_call(Nodes, multi_call_server, {echo, ok}, 100),
1851
1852    multi_call_test_assert_results(R1, Nodes, ok),
1853    multi_call_test_assert_bad_nodes(Bad1, []),
1854
1855    {R2, Bad2} = misc:multi_call(BadNodes ++ Nodes,
1856                                 multi_call_server, {echo, ok}, 100),
1857    multi_call_test_assert_results(R2, Nodes, ok),
1858    multi_call_test_assert_bad_nodes(Bad2, BadNodes),
1859
1860    [FirstNode | RestNodes] = Nodes,
1861    catch gen_server:call({multi_call_server, FirstNode}, {sleep, 100000}, 100),
1862
1863    {R3, Bad3} = misc:multi_call(Nodes, multi_call_server, {echo, ok}, 100),
1864    multi_call_test_assert_results(R3, RestNodes, ok),
1865    ?assertEqual(Bad3, [{FirstNode, timeout}]),
1866
1867    true = rpc:call(FirstNode, erlang, apply,
1868                    [fun () ->
1869                             erlang:exit(whereis(multi_call_server), kill)
1870                     end, []]),
1871    {R4, Bad4} = misc:multi_call(Nodes, multi_call_server, {echo, ok}, 100),
1872    multi_call_test_assert_results(R4, RestNodes, ok),
1873    ?assertMatch([{FirstNode, {exit, {noproc, _}}}], Bad4).
1874
1875multi_call_ok_pred_test_() ->
1876    {setup, fun multi_call_test_setup/0, fun multi_call_test_teardown/1,
1877     [fun do_test_multi_call_ok_pred/0]}.
1878
1879do_test_multi_call_ok_pred() ->
1880    Nodes = nodes(),
1881    BadNodes = [bad_node],
1882    AllNodes = BadNodes ++ Nodes,
1883
1884    {R1, Bad1} = misc:multi_call(AllNodes, multi_call_server, {echo, ok}, 100,
1885                                 fun (_) -> false end),
1886
1887    ?assertEqual(R1, []),
1888    multi_call_test_assert_bad_nodes(Bad1, AllNodes),
1889
1890    {R2, Bad2} = misc:multi_call(AllNodes, multi_call_server, {echo, ok}, 100,
1891                                 fun (_) -> {false, some_error} end),
1892    ?assertEqual(R2, []),
1893    multi_call_test_assert_bad_nodes(Bad2, AllNodes),
1894    multi_call_test_assert_results(Bad2, Nodes, some_error),
1895
1896    {OkNodes, ErrorNodes} = lists:split(length(Nodes) div 2, misc:shuffle(Nodes)),
1897    {R3, Bad3} = misc:multi_call(AllNodes, multi_call_server,
1898                                 {eval, fun () ->
1899                                                case lists:member(node(), OkNodes) of
1900                                                    true ->
1901                                                        ok;
1902                                                    false ->
1903                                                        error
1904                                                end
1905                                        end},
1906                                 100,
1907                                 fun (RV) ->
1908                                         RV =:= ok
1909                                 end),
1910
1911    multi_call_test_assert_results(R3, OkNodes, ok),
1912    multi_call_test_assert_bad_nodes(Bad3, BadNodes ++ ErrorNodes),
1913    multi_call_test_assert_results(Bad3, ErrorNodes, error).
1914
1915-endif.
1916
1917intersperse([], _) ->
1918    [];
1919intersperse([_] = List, _) ->
1920    List;
1921intersperse([X | Rest], Sep) ->
1922    [X, Sep | intersperse(Rest, Sep)].
1923
1924-ifdef(EUNIT).
1925intersperse_test() ->
1926    ?assertEqual([], intersperse([], x)),
1927    ?assertEqual([a], intersperse([a], x)),
1928    ?assertEqual([a,x,b,x,c], intersperse([a,b,c], x)).
1929-endif.
1930
1931hexify(Binary) ->
1932    << <<(hexify_digit(High)), (hexify_digit(Low))>>
1933       || <<High:4, Low:4>> <= Binary >>.
1934
1935hexify_digit(0) -> $0;
1936hexify_digit(1) -> $1;
1937hexify_digit(2) -> $2;
1938hexify_digit(3) -> $3;
1939hexify_digit(4) -> $4;
1940hexify_digit(5) -> $5;
1941hexify_digit(6) -> $6;
1942hexify_digit(7) -> $7;
1943hexify_digit(8) -> $8;
1944hexify_digit(9) -> $9;
1945hexify_digit(10) -> $a;
1946hexify_digit(11) -> $b;
1947hexify_digit(12) -> $c;
1948hexify_digit(13) -> $d;
1949hexify_digit(14) -> $e;
1950hexify_digit(15) -> $f.
1951
1952-ifdef(EUNIT).
1953hexify_test() ->
1954    lists:foreach(
1955      fun (_) ->
1956              R = crypto:rand_bytes(256),
1957              Hex = hexify(R),
1958
1959              Etalon0 = string:to_lower(integer_to_list(binary:decode_unsigned(R), 16)),
1960              Etalon1 =
1961                  case erlang:byte_size(R) * 2 - length(Etalon0) of
1962                      0 ->
1963                          Etalon0;
1964                      N when N > 0 ->
1965                          lists:duplicate(N, $0) ++ Etalon0
1966                  end,
1967              Etalon = list_to_binary(Etalon1),
1968
1969              ?assertEqual(Hex, Etalon)
1970      end, lists:seq(1, 100)).
1971-endif.
1972
1973iolist_is_empty(<<>>) ->
1974    true;
1975iolist_is_empty([]) ->
1976    true;
1977iolist_is_empty([H|T]) ->
1978    iolist_is_empty(H) andalso iolist_is_empty(T);
1979iolist_is_empty(_) ->
1980    false.
1981
1982-ifdef(EUNIT).
1983iolist_is_empty_test() ->
1984    ?assertEqual(iolist_is_empty(""), true),
1985    ?assertEqual(iolist_is_empty(<<>>), true),
1986    ?assertEqual(iolist_is_empty([[[<<>>]]]), true),
1987    ?assertEqual(iolist_is_empty([[]|<<>>]), true),
1988    ?assertEqual(iolist_is_empty([<<>>|[]]), true),
1989    ?assertEqual(iolist_is_empty([[[]], <<"test">>]), false),
1990    ?assertEqual(iolist_is_empty([[<<>>]|"test"]), false).
1991-endif.
1992
1993ejson_encode_pretty(Json) ->
1994    iolist_to_binary(
1995      pipes:run(sjson:stream_json(Json),
1996                sjson:encode_json([{compact, false},
1997                                   {strict, false}]),
1998                pipes:collect())).
1999
2000upermutations(Xs) ->
2001    do_upermutations(lists:sort(Xs)).
2002
2003do_upermutations([]) ->
2004    [[]];
2005do_upermutations(Xs) ->
2006    [[X|Ys] || X <- unique(Xs), Ys <- do_upermutations(Xs -- [X])].
2007
2008prop_upermutations() ->
2009    ?FORALL(Xs, resize(10, list(int(0,5))),
2010            begin
2011                Perms = upermutations(Xs),
2012
2013                NoDups = (lists:usort(Perms) =:= Perms),
2014
2015                XsSorted = lists:sort(Xs),
2016                ProperPerms =
2017                    lists:all(
2018                      fun (P) ->
2019                              lists:sort(P) =:= XsSorted
2020                      end, Perms),
2021
2022                N = length(Xs),
2023                Counts = [C || {_, C} <- uniqc(XsSorted)],
2024                ExpectedSize =
2025                    fact(N) div lists:foldl(
2026                                  fun (X, Y) -> X * Y end,
2027                                  1,
2028                                  lists:map(fun fact/1, Counts)),
2029                ProperSize = (ExpectedSize =:= length(Perms)),
2030
2031                NoDups andalso ProperPerms andalso ProperSize
2032            end).
2033
2034fact(0) ->
2035    1;
2036fact(N) ->
2037    N * fact(N-1).
2038
2039-spec item_count(list(), term()) -> non_neg_integer().
2040item_count(List, Item) ->
2041    lists:foldl(
2042      fun(Ele, Acc) ->
2043              if Ele =:= Item -> Acc + 1;
2044                 true -> Acc
2045              end
2046      end, 0, List).
2047
2048canonical_proplist(List) ->
2049    lists:usort(compact_proplist(List)).
2050
2051%% This is similar to proplists:compact() in spirit, except that it
2052%% [1] drops the item only if it's tuple and its second field is false
2053%% [2] retains already compacted props & tuples whose second field is not false
2054%%
2055%% Ex:
2056%%  compact([{a,true}, {d,false}, 1, {"x",true}, {a,b}]) =:= [a, 1, "x", {a,b}]
2057compact_proplist(List) ->
2058    lists:filtermap(
2059      fun(Elem) ->
2060              case Elem of
2061                  {Key, true} ->
2062                      {true, Key};
2063                  {_Key, false} ->
2064                      false;
2065                  _ ->
2066                      {true, Elem}
2067              end
2068      end, List).
2069
2070proplist_keyfilter(Pred, PropList) ->
2071    lists:filter(proplist_keyfilter_pred(_, Pred), PropList).
2072
2073proplist_keyfilter_pred(Tuple, Pred)
2074  when is_tuple(Tuple) ->
2075    Pred(element(1, Tuple));
2076proplist_keyfilter_pred(Key, Pred) ->
2077    Pred(Key).
2078
2079-ifdef(EUNIT).
2080
2081compact_test() ->
2082    ?assertEqual([a, c], compact_proplist([{a,true}, {b,false}, {c,true}])),
2083    ?assertEqual(["b", {a,b}], compact_proplist([{"b",true}, {a,b}])).
2084
2085canonical_proplist_test() ->
2086    ?assertEqual([a], canonical_proplist([{a,true}, a, {b,false}])),
2087    ?assertEqual([123, x, {c,"x"}, {e, "y"}],
2088                 canonical_proplist([{x,true}, {e,"y"}, 123, {c,"x"}])).
2089
2090proplist_keyfilter_test() ->
2091    ?assertEqual([{a, 0}, a, {c, 2}, {e, 4}],
2092                 proplist_keyfilter(
2093                   lists:member(_, [a, c, e]),
2094                   [{a, 0}, b, a, {b, 23}, {c, 2}, d, {e, 4}])).
2095
2096-endif.
2097
2098%% TODO: Use string:prefix API (not exported in R16) and remove this when we
2099%%       upgrade to newer versions of Erlang.
2100-spec string_prefix(string(), string()) -> string() | nomatch.
2101string_prefix([C | Str], [C | Pre]) ->
2102    string_prefix(Str, Pre);
2103string_prefix(Str, []) when is_list(Str) ->
2104    Str;
2105string_prefix(Str, Pre) when is_list(Pre), is_list(Str) ->
2106    nomatch.
2107
2108compress(Term) ->
2109    zlib:compress(term_to_binary(Term)).
2110
2111decompress(Blob) ->
2112    binary_to_term(zlib:uncompress(Blob)).
2113
2114-spec split_host_port(list(), list()) -> tuple().
2115split_host_port(HostPort, DefaultPort) ->
2116    split_host_port(HostPort, DefaultPort, is_ipv6()).
2117
2118-spec split_host_port(list(), list(), boolean()) -> tuple().
2119split_host_port("[" ++ Rest, DefaultPort, true) ->
2120    case string:tokens(Rest, "]") of
2121        [Host] ->
2122            {Host, DefaultPort};
2123        [Host, ":" ++ Port] when Port =/= [] ->
2124            {Host, Port};
2125        _ ->
2126            throw({error, [<<"The hostname is malformed.">>]})
2127    end;
2128split_host_port("[" ++ _Rest, _DefaultPort, false) ->
2129    throw({error, [<<"The hostname is malformed.">>]});
2130split_host_port(HostPort, DefaultPort, _) ->
2131    case item_count(HostPort, $:) > 1 of
2132        true ->
2133            throw({error, [<<"The hostname is malformed. If using an IPv6 address, "
2134                             "please enclose the address within '[' and ']'">>]});
2135        false ->
2136            case string:tokens(HostPort, ":") of
2137                [Host] ->
2138                    {Host, DefaultPort};
2139                [Host, Port] ->
2140                    {Host, Port};
2141                _ ->
2142                    throw({error, [<<"The hostname is malformed.">>]})
2143            end
2144    end.
2145
2146-spec maybe_add_brackets(string()) -> string().
2147maybe_add_brackets("[" ++ _Rest = Address) ->
2148    Address;
2149maybe_add_brackets(Address) ->
2150    case lists:member($:, Address) of
2151        true -> "[" ++ Address ++ "]";
2152        false -> Address
2153    end.
2154
2155-spec join_host_port(string(), string() | integer()) -> string().
2156join_host_port(Host, Port) when is_integer(Port) ->
2157    join_host_port(Host, integer_to_list(Port));
2158join_host_port(Host, Port) ->
2159    maybe_add_brackets(Host) ++ ":" ++ Port.
2160
2161-ifdef(EUNIT).
2162join_host_port_test() ->
2163    ?assertEqual("127.0.0.1:1234", join_host_port("127.0.0.1", 1234)),
2164    ?assertEqual("abc.xyz.com:1234", join_host_port("abc.xyz.com", "1234")),
2165    ?assertEqual("[fc00::11]:1234", join_host_port("fc00::11", 1234)).
2166-endif.
2167
2168%% Convert OTP-18+ style time to the traditional now()-like timestamp.
2169%%
2170%% Time should be the system time (as returned by time_compat:system_time/1)
2171%% to be converted.
2172%%
2173%% Unit specifies the unit used.
2174time_to_timestamp(Time, Unit) ->
2175    Micro = time_compat:convert_time_unit(Time, Unit, microsecond),
2176
2177    Sec = Micro div 1000000,
2178    Mega = Sec div 1000000,
2179    {Mega, Sec - Mega * 1000000, Micro - Sec * 1000000}.
2180
2181time_to_timestamp(Time) ->
2182    time_to_timestamp(Time, native).
2183
2184%% Convert traditional now()-like timestamp to OTP-18+ system time.
2185%%
2186%% Unit designates the unit to convert to.
2187timestamp_to_time({MegaSec, Sec, MicroSec}, Unit) ->
2188    Time = MicroSec + 1000000 * (Sec + 1000000 * MegaSec),
2189    time_compat:convert_time_unit(Time, microsecond, Unit).
2190
2191timestamp_to_time(TimeStamp) ->
2192    timestamp_to_time(TimeStamp, native).
2193
2194time_to_epoch_float(Time) when is_integer(Time) or is_float(Time) ->
2195    Time;
2196
2197time_to_epoch_float({_, _, _} = TS) ->
2198    timestamp_to_time(TS, microsecond) / 1000000;
2199
2200time_to_epoch_float(_) ->
2201    undefined.
2202
2203%% Shortcut convert_time_unit/3. Always assumes that time to convert
2204%% is in native units.
2205convert_time_unit(Time, TargetUnit) ->
2206    time_compat:convert_time_unit(Time, native, TargetUnit).
2207
2208update_field(Field, Record, Fun) ->
2209    setelement(Field, Record, Fun(element(Field, Record))).
2210
2211dump_term(Term) ->
2212    true = can_recover_term(Term),
2213
2214    [io_lib:write(Term), $.].
2215
2216can_recover_term(Term) ->
2217    generic:query(fun erlang:'and'/2,
2218                  ?cut(not (is_pid(_1) orelse is_reference(_1) orelse is_port(_1))),
2219                  Term).
2220
2221parse_term(Term) when is_binary(Term) ->
2222    do_parse_term(binary_to_list(Term));
2223parse_term(Term) when is_list(Term) ->
2224    do_parse_term(lists:flatten(Term)).
2225
2226do_parse_term(Term) ->
2227    {ok, Tokens, _} = erl_scan:string(Term),
2228    {ok, Parsed} = erl_parse:parse_term(Tokens),
2229    Parsed.
2230
2231forall_recoverable_terms(Body) ->
2232    ?FORALL(T, ?SUCHTHAT(T1, any(), can_recover_term(T1)), Body(T)).
2233
2234prop_dump_parse_term() ->
2235    forall_recoverable_terms(?cut(_1 =:= parse_term(dump_term(_1)))).
2236
2237prop_dump_parse_term_binary() ->
2238    forall_recoverable_terms(?cut(_1 =:= parse_term(iolist_to_binary(dump_term(_1))))).
2239
2240-record(timer, {tref, msg}).
2241
2242-type timeout()   :: non_neg_integer() | infinity.
2243-type timer()     :: timer(any()).
2244-type timer(Type) :: #timer{tref :: undefined | reference(),
2245                            msg  :: Type}.
2246
2247-spec create_timer(Msg :: Type) -> timer(Type).
2248create_timer(Msg) ->
2249    #timer{tref = undefined,
2250           msg  = Msg}.
2251
2252-spec create_timer(timeout(), Msg :: Type) -> timer(Type).
2253create_timer(Timeout, Msg) ->
2254    arm_timer(Timeout, create_timer(Msg)).
2255
2256-spec arm_timer(timeout(), timer(Type)) -> timer(Type).
2257arm_timer(Timeout, Timer) ->
2258    do_arm_timer(Timeout, cancel_timer(Timer)).
2259
2260do_arm_timer(infinity, Timer) ->
2261    Timer;
2262do_arm_timer(0, Timer) ->
2263    self() ! Timer#timer.msg,
2264    Timer;
2265do_arm_timer(Timeout, #timer{msg = Msg} = Timer) ->
2266    TRef = erlang:send_after(Timeout, self(), Msg),
2267    Timer#timer{tref = TRef}.
2268
2269-spec cancel_timer(timer(Type)) -> timer(Type).
2270cancel_timer(#timer{tref = undefined} = Timer) ->
2271    Timer;
2272cancel_timer(#timer{tref = TRef,
2273                    msg  = Msg} = Timer) ->
2274    erlang:cancel_timer(TRef),
2275    flush(Msg),
2276
2277    Timer#timer{tref = undefined}.
2278
2279-spec read_timer(timer()) -> false | non_neg_integer().
2280read_timer(#timer{tref = undefined}) ->
2281    false;
2282read_timer(Timer) ->
2283    case erlang:read_timer(Timer#timer.tref) of
2284        false ->
2285            %% Since we change tref to undefined when the timer is
2286            %% canceled, here we can be confident that the timer has
2287            %% fired. The user might or might not have processed the
2288            %% message, regardless, we return 0 here.
2289            0;
2290        TimeLeft when is_integer(TimeLeft) ->
2291            TimeLeft
2292    end.
2293
2294is_normal_termination(normal) ->
2295    true;
2296is_normal_termination(Reason) ->
2297    is_shutdown(Reason).
2298
2299is_shutdown(shutdown) ->
2300    true;
2301is_shutdown({shutdown, _}) ->
2302    true;
2303is_shutdown(_) ->
2304    false.
2305
2306with_trap_exit(Fun) ->
2307    Old = process_flag(trap_exit, true),
2308    try
2309        Fun()
2310    after
2311        case Old of
2312            true ->
2313                ok;
2314            false ->
2315                process_flag(trap_exit, false),
2316                with_trap_exit_maybe_exit()
2317        end
2318    end.
2319
2320with_trap_exit_maybe_exit() ->
2321    receive
2322        {'EXIT', _Pid, normal} = Exit ->
2323            ?log_debug("Ignoring exit message with reason normal: ~p", [Exit]),
2324            with_trap_exit_maybe_exit();
2325        {'EXIT', _Pid, Reason} = Exit ->
2326            ?log_debug("Terminating due to exit message ~p", [Exit]),
2327            exit_async(Reason)
2328    after
2329        0 ->
2330            ok
2331    end.
2332
2333%% Like exit(reason), but can't be catched like such:
2334%%
2335%% try exit_async(evasive) catch exit:evasive -> ok end
2336exit_async(Reason) ->
2337    Self = self(),
2338    Pid = spawn(fun () ->
2339                        exit(Self, Reason)
2340                end),
2341    wait_for_process(Pid, infinity),
2342    exit(must_not_happen).
2343
2344-ifdef(EUNIT).
2345with_trap_exit_test_() ->
2346    {spawn,
2347     fun () ->
2348             ?assertExit(
2349                finished,
2350                begin
2351                    with_trap_exit(fun () ->
2352                                           spawn_link(fun () ->
2353                                                              exit(crash)
2354                                                      end),
2355
2356                                           receive
2357                                               {'EXIT', _, crash} ->
2358                                                   ok
2359                                           end
2360                                   end),
2361
2362                    false = process_flag(trap_exit, false),
2363
2364                    Parent = self(),
2365                    {_, MRef} =
2366                        erlang:spawn_monitor(
2367                          fun () ->
2368                                  try
2369                                      with_trap_exit(
2370                                        fun () ->
2371                                                spawn_link(fun () ->
2372                                                                   exit(blah)
2373                                                           end),
2374
2375                                                timer:sleep(100),
2376                                                Parent ! msg
2377                                        end)
2378                                  catch
2379                                      exit:blah ->
2380                                          %% we must not be able to catch the
2381                                          %% shutdown
2382                                          throw(bad)
2383                                  end
2384                          end),
2385
2386                    receive
2387                        {'DOWN', MRef, _, _, Reason} ->
2388                            ?assertEqual(blah, Reason)
2389                    end,
2390
2391                    %% must still receive the message
2392                    1 = misc:flush(msg),
2393
2394                    with_trap_exit(fun () ->
2395                                           spawn_link(fun () ->
2396                                                              exit(normal)
2397                                                      end),
2398                                           timer:sleep(100)
2399                                   end),
2400
2401                    exit(finished)
2402                end)
2403     end}.
2404-endif.
2405
2406%% Like a sequence of unlink(Pid) and exit(Pid, Reason). But care is taken
2407%% that the Pid is terminated even the caller dies right in between unlink and
2408%% exit.
2409unlink_terminate(Pid, Reason) ->
2410    with_trap_exit(
2411      fun () ->
2412              terminate(Pid, Reason),
2413              unlink(Pid),
2414              %% the process might have died before we unlinked
2415              ?flush({'EXIT', Pid, _})
2416      end).
2417
2418%% Unlink, terminate and wait for the completion.
2419unlink_terminate_and_wait(Pid, Reason) ->
2420    unlink_terminate(Pid, Reason),
2421
2422    %% keeping this out of with_trap_exit body to make sure that if somebody
2423    %% wants to kill us quickly, we let them
2424    wait_for_process(Pid, infinity).
2425
2426-ifdef(EUNIT).
2427unlink_terminate_and_wait_simple_test() ->
2428    do_test_unlink_terminate_and_wait_simple(normal),
2429    do_test_unlink_terminate_and_wait_simple(shutdown),
2430    do_test_unlink_terminate_and_wait_simple(kill).
2431
2432do_test_unlink_terminate_and_wait_simple(Reason) ->
2433    Pid = proc_lib:spawn_link(fun () -> timer:sleep(10000) end),
2434    unlink_terminate_and_wait(Pid, Reason),
2435    false = is_process_alive(Pid),
2436
2437    %% make sure dead procecesses are handled too
2438    unlink_terminate_and_wait(Pid, Reason).
2439
2440%% Test that if the killing process doesn't trap exits, we can still kill it
2441%% promptly.
2442unlink_terminate_and_wait_wont_block_test() ->
2443    One = proc_lib:spawn(
2444            fun () ->
2445                    process_flag(trap_exit, true),
2446                    timer:sleep(2000),
2447                    1 = ?flush({'EXIT', _, _})
2448            end),
2449    Two = proc_lib:spawn(
2450            fun () ->
2451                    link(One),
2452                    timer:sleep(50),
2453                    unlink_terminate_and_wait(One, shutdown)
2454            end),
2455
2456    timer:sleep(100),
2457    exit(Two, shutdown),
2458    ok = wait_for_process(Two, 500),
2459    %% the other process terminates eventually
2460    ok = wait_for_process(One, 5000).
2461
2462
2463%% This tries to test that it's never possible to kill the killing process at
2464%% an unfortunate moment and leave the the linked process
2465%% alive. Unfortunately, timings are making it quite hard to test. I managed
2466%% to catch the original issue only when added explicit erlang:yield() between
2467%% unlink and exit. But leaving the test here anyway, it's better than nothing
2468%% after all.
2469unlink_terminate_and_wait_kill_the_killer_test_() ->
2470    {spawn,
2471     fun () ->
2472             lists:foreach(
2473               fun (_) ->
2474                       Self = self(),
2475
2476                       Pid = proc_lib:spawn(fun () -> timer:sleep(10000) end),
2477                       Killer = proc_lib:spawn(
2478                                  fun () ->
2479                                          link(Pid),
2480                                          Self ! linked,
2481                                          receive
2482                                              kill ->
2483                                                  unlink_terminate_and_wait(Pid, kill)
2484                                          end
2485                                  end),
2486
2487                       receive linked -> ok end,
2488                       Killer ! kill,
2489                       delay(random:uniform(10000)),
2490                       exit(Killer, kill),
2491
2492                       ok = wait_for_process(Killer, 1000),
2493                       ok = wait_for_process(Pid, 1000)
2494               end, lists:seq(1, 10000))
2495     end}.
2496
2497delay(0) ->
2498    ok;
2499delay(I) ->
2500    delay(I-1).
2501-endif.
2502
2503%% Compare two strings or binaries for equality without short-circuits
2504%% to avoid timing attacks.
2505compare_secure(<<X/binary>>, <<Y/binary>>) ->
2506    compare_secure(binary_to_list(X), binary_to_list(Y));
2507compare_secure(X, Y) when is_list(X), is_list(Y) ->
2508    case length(X) == length(Y) of
2509        true ->
2510            compare_secure(X, Y, 0);
2511        false ->
2512            false
2513    end.
2514
2515compare_secure([X | RestX], [Y | RestY], Result) ->
2516    compare_secure(RestX, RestY, (X bxor Y) bor Result);
2517compare_secure([], [], Result) ->
2518    Result == 0.
2519
2520bin_bxor(<<Bin1/binary>>, <<Bin2/binary>>) ->
2521    Size = size(Bin1),
2522    Size = size(Bin2),
2523    SizeBits = Size * 8,
2524    <<Int1:SizeBits>> = Bin1,
2525    <<Int2:SizeBits>> = Bin2,
2526    Int3 = Int1 bxor Int2,
2527    <<Int3:SizeBits>>.
2528
2529duplicates(List) when is_list(List) ->
2530    List -- lists:usort(List).
2531
2532-ifdef(EUNIT).
2533no_duplicates_test() ->
2534    ?assertEqual([],  duplicates([])),
2535    ?assertEqual([],  duplicates([1])),
2536    ?assertEqual([],  duplicates([1,2,3,"1"])),
2537    ?assertEqual([1,2,2], duplicates([1,2,1,2,3,2])).
2538-endif.
2539