xref: /5.5.2/couchdb/src/etap/etap.erl (revision 04bc29f3)
1%% Copyright (c) 2008-2009 Nick Gerakines <nick@gerakines.net>
2%%
3%% Permission is hereby granted, free of charge, to any person
4%% obtaining a copy of this software and associated documentation
5%% files (the "Software"), to deal in the Software without
6%% restriction, including without limitation the rights to use,
7%% copy, modify, merge, publish, distribute, sublicense, and/or sell
8%% copies of the Software, and to permit persons to whom the
9%% Software is furnished to do so, subject to the following
10%% conditions:
11%%
12%% The above copyright notice and this permission notice shall be
13%% included in all copies or substantial portions of the Software.
14%%
15%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16%% EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
17%% OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18%% NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
19%% HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
20%% WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21%% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22%% OTHER DEALINGS IN THE SOFTWARE.
23%%
24%% @author Nick Gerakines <nick@gerakines.net> [http://socklabs.com/]
25%% @author Jeremy Wall <jeremy@marzhillstudios.com>
26%% @version 0.3.4
27%% @copyright 2007-2008 Jeremy Wall, 2008-2009 Nick Gerakines
28%% @reference http://testanything.org/wiki/index.php/Main_Page
29%% @reference http://en.wikipedia.org/wiki/Test_Anything_Protocol
30%% @todo Finish implementing the skip directive.
31%% @todo Document the messages handled by this receive loop.
32%% @todo Explain in documentation why we use a process to handle test input.
33%% @doc etap is a TAP testing module for Erlang components and applications.
34%% This module allows developers to test their software using the TAP method.
35%%
36%% <blockquote cite="http://en.wikipedia.org/wiki/Test_Anything_Protocol"><p>
37%% TAP, the Test Anything Protocol, is a simple text-based interface between
38%% testing modules in a test harness. TAP started life as part of the test
39%% harness for Perl but now has implementations in C/C++, Python, PHP, Perl
40%% and probably others by the time you read this.
41%% </p></blockquote>
42%%
43%% The testing process begins by defining a plan using etap:plan/1, running
44%% a number of etap tests and then calling eta:end_tests/0. Please refer to
45%% the Erlang modules in the t directory of this project for example tests.
46-module(etap).
47-vsn("0.3.4").
48
49-export([
50    ensure_test_server/0,
51    start_etap_server/0,
52    test_server/1,
53    msg/1, msg/2,
54    diag/1, diag/2,
55    expectation_mismatch_message/3,
56    plan/1,
57    end_tests/0,
58    not_ok/2, ok/2, is_ok/2, is/3, isnt/3, any/3, none/3,
59    fun_is/3, expect_fun/3, expect_fun/4,
60    is_greater/3,
61    skip/1, skip/2,
62    datetime/1,
63    skip/3,
64    bail/0, bail/1,
65    test_state/0, failure_count/0
66]).
67
68-export([
69    contains_ok/3,
70    is_before/4
71]).
72
73-export([
74    is_pid/2,
75    is_alive/2,
76    is_mfa/3
77]).
78
79-export([
80    loaded_ok/2,
81    can_ok/2, can_ok/3,
82    has_attrib/2, is_attrib/3,
83    is_behaviour/2
84]).
85
86-export([
87    dies_ok/2,
88    lives_ok/2,
89    throws_ok/3
90]).
91
92
93-record(test_state, {
94    planned = 0,
95    count = 0,
96    pass = 0,
97    fail = 0,
98    skip = 0,
99    skip_reason = ""
100}).
101
102%% @spec plan(N) -> Result
103%%       N = unknown | skip | {skip, string()} | integer()
104%%       Result = ok
105%% @doc Create a test plan and boot strap the test server.
106plan(unknown) ->
107    ensure_test_server(),
108    etap_server ! {self(), plan, unknown},
109    ok;
110plan(skip) ->
111    io:format("1..0 # skip~n");
112plan({skip, Reason}) ->
113    io:format("1..0 # skip ~s~n", [Reason]);
114plan(N) when is_integer(N), N > 0 ->
115    ensure_test_server(),
116    etap_server ! {self(), plan, N},
117    ok.
118
119%% @spec end_tests() -> ok
120%% @doc End the current test plan and output test results.
121%% @todo This should probably be done in the test_server process.
122end_tests() ->
123    Ref = make_ref(),
124    case whereis(etap_server) of
125        undefined -> self() ! {Ref, true};
126        _ -> etap_server ! {self(), state, Ref}
127    end,
128    State = receive {Ref, X} -> X end,
129    if
130        is_record(State, test_state) andalso State#test_state.planned == -1 ->
131            io:format("1..~p~n", [State#test_state.count]);
132        true ->
133            ok
134    end,
135    kill().
136
137bail() ->
138    bail("").
139
140bail(Reason) ->
141    etap_server ! {self(), diag, "Bail out! " ++ Reason},
142    kill().
143
144kill() ->
145    case whereis(etap_server) of
146        undefined ->
147            ok;
148        Pid ->
149            MRef = erlang:monitor(process, Pid),
150            Pid ! done,
151            receive
152                {'DOWN', MRef, process, _, _} ->
153                    ok
154            end
155    end.
156
157%% @spec test_state() -> Return
158%%       Return = test_state_record() | {error, string()}
159%% @doc Return the current test state
160test_state() ->
161    etap_server ! {self(), state},
162    receive
163	X when is_record(X, test_state) -> X
164    after
165	1000 -> {error, "Timed out waiting for etap server reply.~n"}
166    end.
167
168%% @spec failure_count() -> Return
169%%       Return = integer() | {error, string()}
170%% @doc Return the current failure count
171failure_count() ->
172    case test_state() of
173        #test_state{fail=FailureCount} -> FailureCount;
174        X -> X
175    end.
176
177%% @spec msg(S) -> ok
178%%       S = string()
179%% @doc Print a message in the test output.
180msg(S) -> etap_server ! {self(), diag, S}, ok.
181
182%% @spec msg(Format, Data) -> ok
183%%      Format = atom() | string() | binary()
184%%      Data = [term()]
185%%      UnicodeList = [Unicode]
186%%      Unicode = int()
187%% @doc Print a message in the test output.
188%% Function arguments are passed through io_lib:format/2.
189msg(Format, Data) -> msg(io_lib:format(Format, Data)).
190
191%% @spec diag(S) -> ok
192%%       S = string()
193%% @doc Print a debug/status message related to the test suite.
194diag(S) -> msg("# " ++ S).
195
196%% @spec diag(Format, Data) -> ok
197%%      Format = atom() | string() | binary()
198%%      Data = [term()]
199%%      UnicodeList = [Unicode]
200%%      Unicode = int()
201%% @doc Print a debug/status message related to the test suite.
202%% Function arguments are passed through io_lib:format/2.
203diag(Format, Data) -> diag(io_lib:format(Format, Data)).
204
205%% @spec expectation_mismatch_message(Got, Expected, Desc) -> ok
206%%       Got = any()
207%%       Expected = any()
208%%       Desc = string()
209%% @doc Print an expectation mismatch message in the test output.
210expectation_mismatch_message(Got, Expected, Desc) ->
211    msg("    ---"),
212    msg("    description: ~p", [Desc]),
213    msg("    found:       ~p", [Got]),
214    msg("    wanted:      ~p", [Expected]),
215    msg("    ..."),
216    ok.
217
218% @spec evaluate(Pass, Got, Expected, Desc) -> Result
219%%       Pass = true | false
220%%       Got = any()
221%%       Expected = any()
222%%       Desc = string()
223%%       Result = true | false
224%% @doc Evaluate a test statement, printing an expectation mismatch message
225%%       if the test failed.
226evaluate(Pass, Got, Expected, Desc) ->
227    case mk_tap(Pass, Desc) of
228        false ->
229            expectation_mismatch_message(Got, Expected, Desc),
230            false;
231        true ->
232            true
233    end.
234
235%% @spec ok(Expr, Desc) -> Result
236%%       Expr = true | false
237%%       Desc = string()
238%%       Result = true | false
239%% @doc Assert that a statement is true.
240ok(Expr, Desc) -> evaluate(Expr == true, Expr, true, Desc).
241
242%% @spec not_ok(Expr, Desc) -> Result
243%%       Expr = true | false
244%%       Desc = string()
245%%       Result = true | false
246%% @doc Assert that a statement is false.
247not_ok(Expr, Desc) -> evaluate(Expr == false, Expr, false, Desc).
248
249%% @spec is_ok(Expr, Desc) -> Result
250%%       Expr = any()
251%%       Desc = string()
252%%       Result = true | false
253%% @doc Assert that two values are the same.
254is_ok(Expr, Desc) -> evaluate(Expr == ok, Expr, ok, Desc).
255
256%% @spec is(Got, Expected, Desc) -> Result
257%%       Got = any()
258%%       Expected = any()
259%%       Desc = string()
260%%       Result = true | false
261%% @doc Assert that two values are the same.
262is(Got, Expected, Desc) -> evaluate(Got == Expected, Got, Expected, Desc).
263
264%% @spec isnt(Got, Expected, Desc) -> Result
265%%       Got = any()
266%%       Expected = any()
267%%       Desc = string()
268%%       Result = true | false
269%% @doc Assert that two values are not the same.
270isnt(Got, Expected, Desc) -> evaluate(Got /= Expected, Got, Expected, Desc).
271
272%% @spec is_greater(ValueA, ValueB, Desc) -> Result
273%%       ValueA = number()
274%%       ValueB = number()
275%%       Desc = string()
276%%       Result = true | false
277%% @doc Assert that an integer is greater than another.
278is_greater(ValueA, ValueB, Desc) when is_integer(ValueA), is_integer(ValueB) ->
279    mk_tap(ValueA > ValueB, Desc).
280
281%% @spec any(Got, Items, Desc) -> Result
282%%       Got = any()
283%%       Items = [any()]
284%%       Desc = string()
285%%       Result = true | false
286%% @doc Assert that an item is in a list.
287any(Got, Items, Desc) when is_function(Got) ->
288    is(lists:any(Got, Items), true, Desc);
289any(Got, Items, Desc) ->
290    is(lists:member(Got, Items), true, Desc).
291
292%% @spec none(Got, Items, Desc) -> Result
293%%       Got = any()
294%%       Items = [any()]
295%%       Desc = string()
296%%       Result = true | false
297%% @doc Assert that an item is not in a list.
298none(Got, Items, Desc) when is_function(Got) ->
299    is(lists:any(Got, Items), false, Desc);
300none(Got, Items, Desc) ->
301    is(lists:member(Got, Items), false, Desc).
302
303%% @spec fun_is(Fun, Expected, Desc) -> Result
304%%       Fun = function()
305%%       Expected = any()
306%%       Desc = string()
307%%       Result = true | false
308%% @doc Use an anonymous function to assert a pattern match.
309fun_is(Fun, Expected, Desc) when is_function(Fun) ->
310    is(Fun(Expected), true, Desc).
311
312%% @spec expect_fun(ExpectFun, Got, Desc) -> Result
313%%       ExpectFun = function()
314%%       Got = any()
315%%       Desc = string()
316%%       Result = true | false
317%% @doc Use an anonymous function to assert a pattern match, using actual
318%%       value as the argument to the function.
319expect_fun(ExpectFun, Got, Desc) ->
320    evaluate(ExpectFun(Got), Got, ExpectFun, Desc).
321
322%% @spec expect_fun(ExpectFun, Got, Desc, ExpectStr) -> Result
323%%       ExpectFun = function()
324%%       Got = any()
325%%       Desc = string()
326%%       ExpectStr = string()
327%%       Result = true | false
328%% @doc Use an anonymous function to assert a pattern match, using actual
329%%       value as the argument to the function.
330expect_fun(ExpectFun, Got, Desc, ExpectStr) ->
331    evaluate(ExpectFun(Got), Got, ExpectStr, Desc).
332
333%% @equiv skip(TestFun, "")
334skip(TestFun) when is_function(TestFun) ->
335    skip(TestFun, "").
336
337%% @spec skip(TestFun, Reason) -> ok
338%%       TestFun = function()
339%%       Reason = string()
340%% @doc Skip a test.
341skip(TestFun, Reason) when is_function(TestFun), is_list(Reason) ->
342    begin_skip(Reason),
343    catch TestFun(),
344    end_skip(),
345    ok.
346
347%% @spec skip(Q, TestFun, Reason) -> ok
348%%       Q = true | false | function()
349%%       TestFun = function()
350%%       Reason = string()
351%% @doc Skips a test conditionally. The first argument to this function can
352%% either be the 'true' or 'false' atoms or a function that returns 'true' or
353%% 'false'.
354skip(QFun, TestFun, Reason) when is_function(QFun), is_function(TestFun), is_list(Reason) ->
355    case QFun() of
356        true -> begin_skip(Reason), TestFun(), end_skip();
357        _ -> TestFun()
358    end,
359    ok;
360
361skip(Q, TestFun, Reason) when is_function(TestFun), is_list(Reason), Q == true ->
362    begin_skip(Reason),
363    TestFun(),
364    end_skip(),
365    ok;
366
367skip(_, TestFun, Reason) when is_function(TestFun), is_list(Reason) ->
368    TestFun(),
369    ok.
370
371%% @private
372begin_skip(Reason) ->
373    etap_server ! {self(), begin_skip, Reason}.
374
375%% @private
376end_skip() ->
377    etap_server ! {self(), end_skip}.
378
379%% @spec contains_ok(string(), string(), string()) -> true | false
380%% @doc Assert that a string is contained in another string.
381contains_ok(Source, String, Desc) ->
382    etap:isnt(
383        string:str(Source, String),
384        0,
385        Desc
386    ).
387
388%% @spec is_before(string(), string(), string(), string()) -> true | false
389%% @doc Assert that a string comes before another string within a larger body.
390is_before(Source, StringA, StringB, Desc) ->
391    etap:is_greater(
392        string:str(Source, StringB),
393        string:str(Source, StringA),
394        Desc
395    ).
396
397%% @doc Assert that a given variable is a pid.
398is_pid(Pid, Desc) when is_pid(Pid) -> etap:ok(true, Desc);
399is_pid(_, Desc) -> etap:ok(false, Desc).
400
401%% @doc Assert that a given process/pid is alive.
402is_alive(Pid, Desc) ->
403    etap:ok(erlang:is_process_alive(Pid), Desc).
404
405%% @doc Assert that the current function of a pid is a given {M, F, A} tuple.
406is_mfa(Pid, MFA, Desc) ->
407    etap:is({current_function, MFA}, erlang:process_info(Pid, current_function), Desc).
408
409%% @spec loaded_ok(atom(), string()) -> true | false
410%% @doc Assert that a module has been loaded successfully.
411loaded_ok(M, Desc) when is_atom(M) ->
412    etap:fun_is(fun({module, _}) -> true; (_) -> false end, code:load_file(M), Desc).
413
414%% @spec can_ok(atom(), atom()) -> true | false
415%% @doc Assert that a module exports a given function.
416can_ok(M, F) when is_atom(M), is_atom(F) ->
417    Matches = [X || {X, _} <- M:module_info(exports), X == F],
418    etap:ok(Matches > 0, lists:concat([M, " can ", F])).
419
420%% @spec can_ok(atom(), atom(), integer()) -> true | false
421%% @doc Assert that a module exports a given function with a given arity.
422can_ok(M, F, A) when is_atom(M); is_atom(F), is_number(A) ->
423    Matches = [X || X <- M:module_info(exports), X == {F, A}],
424    etap:ok(Matches > 0, lists:concat([M, " can ", F, "/", A])).
425
426%% @spec has_attrib(M, A) -> true | false
427%%       M = atom()
428%%       A = atom()
429%% @doc Asserts that a module has a given attribute.
430has_attrib(M, A) when is_atom(M), is_atom(A) ->
431    etap:isnt(
432        proplists:get_value(A, M:module_info(attributes), 'asdlkjasdlkads'),
433        'asdlkjasdlkads',
434        lists:concat([M, " has attribute ", A])
435    ).
436
437%% @spec has_attrib(M, A. V) -> true | false
438%%       M = atom()
439%%       A = atom()
440%%       V = any()
441%% @doc Asserts that a module has a given attribute with a given value.
442is_attrib(M, A, V) when is_atom(M) andalso is_atom(A) ->
443    etap:is(
444        proplists:get_value(A, M:module_info(attributes)),
445        [V],
446        lists:concat([M, "'s ", A, " is ", V])
447    ).
448
449%% @spec is_behavior(M, B) -> true | false
450%%       M = atom()
451%%       B = atom()
452%% @doc Asserts that a given module has a specific behavior.
453is_behaviour(M, B) when is_atom(M) andalso is_atom(B) ->
454    is_attrib(M, behaviour, B).
455
456%% @doc Assert that an exception is raised when running a given function.
457dies_ok(F, Desc) ->
458    case (catch F()) of
459        {'EXIT', _} -> etap:ok(true, Desc);
460        _ -> etap:ok(false, Desc)
461    end.
462
463%% @doc Assert that an exception is not raised when running a given function.
464lives_ok(F, Desc) ->
465    etap:is(try_this(F), success, Desc).
466
467%% @doc Assert that the exception thrown by a function matches the given exception.
468throws_ok(F, Exception, Desc) ->
469    try F() of
470        _ -> etap:ok(nok, Desc)
471    catch
472        _:E ->
473            etap:is(E, Exception, Desc)
474    end.
475
476%% @private
477%% @doc Run a function and catch any exceptions.
478try_this(F) when is_function(F, 0) ->
479    try F() of
480        _ -> success
481    catch
482        throw:E -> {throw, E};
483        error:E -> {error, E};
484        exit:E -> {exit, E}
485    end.
486
487%% @private
488%% @doc Start the etap_server process if it is not running already.
489ensure_test_server() ->
490    case whereis(etap_server) of
491        undefined ->
492            proc_lib:start(?MODULE, start_etap_server,[]);
493        _ ->
494            diag("The test server is already running.")
495    end.
496
497%% @private
498%% @doc Start the etap_server loop and register itself as the etap_server
499%% process.
500start_etap_server() ->
501    catch register(etap_server, self()),
502    proc_lib:init_ack(ok),
503    etap:test_server(#test_state{
504        planned = 0,
505        count = 0,
506        pass = 0,
507        fail = 0,
508        skip = 0,
509        skip_reason = ""
510    }).
511
512
513%% @private
514%% @doc The main etap_server receive/run loop. The etap_server receive loop
515%% responds to seven messages apperatining to failure or passing of tests.
516%% It is also used to initiate the testing process with the {_, plan, _}
517%% message that clears the current test state.
518test_server(State) ->
519    NewState = receive
520        {_From, plan, unknown} ->
521            io:format("# Current time local ~s~n", [datetime(erlang:localtime())]),
522            io:format("# Using etap version ~p~n", [ proplists:get_value(vsn, proplists:get_value(attributes, etap:module_info())) ]),
523            State#test_state{
524                planned = -1,
525                count = 0,
526                pass = 0,
527                fail = 0,
528                skip = 0,
529                skip_reason = ""
530            };
531        {_From, plan, N} ->
532            io:format("# Current time local ~s~n", [datetime(erlang:localtime())]),
533            io:format("# Using etap version ~p~n", [ proplists:get_value(vsn, proplists:get_value(attributes, etap:module_info())) ]),
534            io:format("1..~p~n", [N]),
535            State#test_state{
536                planned = N,
537                count = 0,
538                pass = 0,
539                fail = 0,
540                skip = 0,
541                skip_reason = ""
542            };
543        {_From, begin_skip, Reason} ->
544            State#test_state{
545                skip = 1,
546                skip_reason = Reason
547            };
548        {_From, end_skip} ->
549            State#test_state{
550                skip = 0,
551                skip_reason = ""
552            };
553        {_From, pass, Desc} ->
554            FullMessage = skip_diag(
555                " - " ++ Desc,
556                State#test_state.skip,
557                State#test_state.skip_reason
558            ),
559            io:format("ok ~p ~s~n", [State#test_state.count + 1, FullMessage]),
560            State#test_state{
561                count = State#test_state.count + 1,
562                pass = State#test_state.pass + 1
563            };
564
565        {_From, fail, Desc} ->
566            FullMessage = skip_diag(
567                " - " ++ Desc,
568                State#test_state.skip,
569                State#test_state.skip_reason
570            ),
571            io:format("not ok ~p ~s~n", [State#test_state.count + 1, FullMessage]),
572            State#test_state{
573                count = State#test_state.count + 1,
574                fail = State#test_state.fail + 1
575            };
576        {From, state, Ref} ->
577            From ! {Ref, State},
578            State;
579        {_From, diag, Message} ->
580            io:format("~s~n", [Message]),
581            State;
582        {From, count} ->
583            From ! State#test_state.count,
584            State;
585        {From, is_skip, Ref} ->
586            From ! {Ref, State#test_state.skip},
587            State;
588        done ->
589            exit(normal)
590    end,
591    test_server(NewState).
592
593%% @private
594%% @doc Process the result of a test and send it to the etap_server process.
595mk_tap(Result, Desc) ->
596    etap_server ! {self(), is_skip, Ref = make_ref()} ,
597    receive {Ref, IsSkip} -> ok end,
598    case [IsSkip, Result] of
599        [_, true] ->
600            etap_server ! {self(), pass, Desc},
601            true;
602        [1, _] ->
603            etap_server ! {self(), pass, Desc},
604            true;
605        _ ->
606            etap_server ! {self(), fail, Desc},
607            false
608    end.
609
610%% @private
611%% @doc Format a date/time string.
612datetime(DateTime) ->
613    {{Year, Month, Day}, {Hour, Min, Sec}} = DateTime,
614    io_lib:format("~4.10.0B-~2.10.0B-~2.10.0B ~2.10.0B:~2.10.0B:~2.10.0B", [Year, Month, Day, Hour, Min, Sec]).
615
616%% @private
617%% @doc Craft an output message taking skip/todo into consideration.
618skip_diag(Message, 0, _) ->
619    Message;
620skip_diag(_Message, 1, "") ->
621    " # SKIP";
622skip_diag(_Message, 1, Reason) ->
623    " # SKIP : " ++ Reason.
624