xref: /5.5.2/couchdb/src/etap/etap.erl (revision bc6c5ea6)
1%% Copyright (c) 2008-2009 Nick Gerakines <nick@gerakines.net>
2%%
3%% Permission is hereby granted, free of charge, to any person
4%% obtaining a copy of this software and associated documentation
5%% files (the "Software"), to deal in the Software without
6%% restriction, including without limitation the rights to use,
7%% copy, modify, merge, publish, distribute, sublicense, and/or sell
8%% copies of the Software, and to permit persons to whom the
9%% Software is furnished to do so, subject to the following
10%% conditions:
11%%
12%% The above copyright notice and this permission notice shall be
13%% included in all copies or substantial portions of the Software.
14%%
15%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16%% EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
17%% OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18%% NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
19%% HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
20%% WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21%% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22%% OTHER DEALINGS IN THE SOFTWARE.
23%%
24%% @author Nick Gerakines <nick@gerakines.net> [http://socklabs.com/]
25%% @author Jeremy Wall <jeremy@marzhillstudios.com>
26%% @version 0.3.4
27%% @copyright 2007-2008 Jeremy Wall, 2008-2009 Nick Gerakines
28%% @reference http://testanything.org/wiki/index.php/Main_Page
29%% @reference http://en.wikipedia.org/wiki/Test_Anything_Protocol
30%% @todo Finish implementing the skip directive.
31%% @todo Document the messages handled by this receive loop.
32%% @todo Explain in documentation why we use a process to handle test input.
33%% @doc etap is a TAP testing module for Erlang components and applications.
34%% This module allows developers to test their software using the TAP method.
35%%
36%% <blockquote cite="http://en.wikipedia.org/wiki/Test_Anything_Protocol"><p>
37%% TAP, the Test Anything Protocol, is a simple text-based interface between
38%% testing modules in a test harness. TAP started life as part of the test
39%% harness for Perl but now has implementations in C/C++, Python, PHP, Perl
40%% and probably others by the time you read this.
41%% </p></blockquote>
42%%
43%% The testing process begins by defining a plan using etap:plan/1, running
44%% a number of etap tests and then calling eta:end_tests/0. Please refer to
45%% the Erlang modules in the t directory of this project for example tests.
46-module(etap).
47-vsn("0.3.4").
48
49-export([
50    ensure_test_server/0,
51    start_etap_server/0,
52    test_server/1,
53    msg/1, msg/2,
54    diag/1, diag/2,
55    expectation_mismatch_message/3,
56    plan/1,
57    end_tests/0,
58    not_ok/2, ok/2, is_ok/2, is/3, isnt/3, any/3, none/3,
59    fun_is/3, expect_fun/3, expect_fun/4,
60    is_greater/3,
61    skip/1, skip/2,
62    datetime/1,
63    skip/3,
64    bail/0, bail/1,
65    test_state/0, failure_count/0
66]).
67
68-export([
69    contains_ok/3,
70    is_before/4
71]).
72
73-export([
74    is_pid/2,
75    is_alive/2,
76    is_mfa/3
77]).
78
79-export([
80    loaded_ok/2,
81    can_ok/2, can_ok/3,
82    has_attrib/2, is_attrib/3,
83    is_behaviour/2
84]).
85
86-export([
87    dies_ok/2,
88    lives_ok/2,
89    throws_ok/3
90]).
91
92
93-record(test_state, {
94    planned = 0,
95    count = 0,
96    pass = 0,
97    fail = 0,
98    skip = 0,
99    skip_reason = ""
100}).
101
102%% @spec plan(N) -> Result
103%%       N = unknown | skip | {skip, string()} | integer()
104%%       Result = ok
105%% @doc Create a test plan and boot strap the test server.
106plan(unknown) ->
107    ensure_test_server(),
108    etap_server ! {self(), plan, unknown},
109    ok;
110plan(skip) ->
111    io:format("1..0 # skip~n");
112plan({skip, Reason}) ->
113    io:format("1..0 # skip ~s~n", [Reason]);
114plan(N) when is_integer(N), N > 0 ->
115    ensure_test_server(),
116    etap_server ! {self(), plan, N},
117    ok.
118
119%% @spec end_tests() -> ok
120%% @doc End the current test plan and output test results.
121%% @todo This should probably be done in the test_server process.
122end_tests() ->
123    Ref = make_ref(),
124    case whereis(etap_server) of
125        undefined -> self() ! {Ref, true};
126        _ -> etap_server ! {self(), state, Ref}
127    end,
128    State = receive {Ref, X} -> X end,
129    if
130        is_record(State, test_state) andalso State#test_state.planned == -1 ->
131            io:format("1..~p~n", [State#test_state.count]);
132        true ->
133            ok
134    end,
135    case whereis(etap_server) of
136        undefined -> ok;
137        _ -> etap_server ! done, ok
138    end.
139
140bail() ->
141    bail("").
142
143bail(Reason) ->
144    etap_server ! {self(), diag, "Bail out! " ++ Reason},
145    etap_server ! done, ok,
146    ok.
147
148%% @spec test_state() -> Return
149%%       Return = test_state_record() | {error, string()}
150%% @doc Return the current test state
151test_state() ->
152    etap_server ! {self(), state},
153    receive
154	X when is_record(X, test_state) -> X
155    after
156	1000 -> {error, "Timed out waiting for etap server reply.~n"}
157    end.
158
159%% @spec failure_count() -> Return
160%%       Return = integer() | {error, string()}
161%% @doc Return the current failure count
162failure_count() ->
163    case test_state() of
164        #test_state{fail=FailureCount} -> FailureCount;
165        X -> X
166    end.
167
168%% @spec msg(S) -> ok
169%%       S = string()
170%% @doc Print a message in the test output.
171msg(S) -> etap_server ! {self(), diag, S}, ok.
172
173%% @spec msg(Format, Data) -> ok
174%%      Format = atom() | string() | binary()
175%%      Data = [term()]
176%%      UnicodeList = [Unicode]
177%%      Unicode = int()
178%% @doc Print a message in the test output.
179%% Function arguments are passed through io_lib:format/2.
180msg(Format, Data) -> msg(io_lib:format(Format, Data)).
181
182%% @spec diag(S) -> ok
183%%       S = string()
184%% @doc Print a debug/status message related to the test suite.
185diag(S) -> msg("# " ++ S).
186
187%% @spec diag(Format, Data) -> ok
188%%      Format = atom() | string() | binary()
189%%      Data = [term()]
190%%      UnicodeList = [Unicode]
191%%      Unicode = int()
192%% @doc Print a debug/status message related to the test suite.
193%% Function arguments are passed through io_lib:format/2.
194diag(Format, Data) -> diag(io_lib:format(Format, Data)).
195
196%% @spec expectation_mismatch_message(Got, Expected, Desc) -> ok
197%%       Got = any()
198%%       Expected = any()
199%%       Desc = string()
200%% @doc Print an expectation mismatch message in the test output.
201expectation_mismatch_message(Got, Expected, Desc) ->
202    msg("    ---"),
203    msg("    description: ~p", [Desc]),
204    msg("    found:       ~p", [Got]),
205    msg("    wanted:      ~p", [Expected]),
206    msg("    ..."),
207    ok.
208
209% @spec evaluate(Pass, Got, Expected, Desc) -> Result
210%%       Pass = true | false
211%%       Got = any()
212%%       Expected = any()
213%%       Desc = string()
214%%       Result = true | false
215%% @doc Evaluate a test statement, printing an expectation mismatch message
216%%       if the test failed.
217evaluate(Pass, Got, Expected, Desc) ->
218    case mk_tap(Pass, Desc) of
219        false ->
220            expectation_mismatch_message(Got, Expected, Desc),
221            false;
222        true ->
223            true
224    end.
225
226%% @spec ok(Expr, Desc) -> Result
227%%       Expr = true | false
228%%       Desc = string()
229%%       Result = true | false
230%% @doc Assert that a statement is true.
231ok(Expr, Desc) -> evaluate(Expr == true, Expr, true, Desc).
232
233%% @spec not_ok(Expr, Desc) -> Result
234%%       Expr = true | false
235%%       Desc = string()
236%%       Result = true | false
237%% @doc Assert that a statement is false.
238not_ok(Expr, Desc) -> evaluate(Expr == false, Expr, false, Desc).
239
240%% @spec is_ok(Expr, Desc) -> Result
241%%       Expr = any()
242%%       Desc = string()
243%%       Result = true | false
244%% @doc Assert that two values are the same.
245is_ok(Expr, Desc) -> evaluate(Expr == ok, Expr, ok, Desc).
246
247%% @spec is(Got, Expected, Desc) -> Result
248%%       Got = any()
249%%       Expected = any()
250%%       Desc = string()
251%%       Result = true | false
252%% @doc Assert that two values are the same.
253is(Got, Expected, Desc) -> evaluate(Got == Expected, Got, Expected, Desc).
254
255%% @spec isnt(Got, Expected, Desc) -> Result
256%%       Got = any()
257%%       Expected = any()
258%%       Desc = string()
259%%       Result = true | false
260%% @doc Assert that two values are not the same.
261isnt(Got, Expected, Desc) -> evaluate(Got /= Expected, Got, Expected, Desc).
262
263%% @spec is_greater(ValueA, ValueB, Desc) -> Result
264%%       ValueA = number()
265%%       ValueB = number()
266%%       Desc = string()
267%%       Result = true | false
268%% @doc Assert that an integer is greater than another.
269is_greater(ValueA, ValueB, Desc) when is_integer(ValueA), is_integer(ValueB) ->
270    mk_tap(ValueA > ValueB, Desc).
271
272%% @spec any(Got, Items, Desc) -> Result
273%%       Got = any()
274%%       Items = [any()]
275%%       Desc = string()
276%%       Result = true | false
277%% @doc Assert that an item is in a list.
278any(Got, Items, Desc) when is_function(Got) ->
279    is(lists:any(Got, Items), true, Desc);
280any(Got, Items, Desc) ->
281    is(lists:member(Got, Items), true, Desc).
282
283%% @spec none(Got, Items, Desc) -> Result
284%%       Got = any()
285%%       Items = [any()]
286%%       Desc = string()
287%%       Result = true | false
288%% @doc Assert that an item is not in a list.
289none(Got, Items, Desc) when is_function(Got) ->
290    is(lists:any(Got, Items), false, Desc);
291none(Got, Items, Desc) ->
292    is(lists:member(Got, Items), false, Desc).
293
294%% @spec fun_is(Fun, Expected, Desc) -> Result
295%%       Fun = function()
296%%       Expected = any()
297%%       Desc = string()
298%%       Result = true | false
299%% @doc Use an anonymous function to assert a pattern match.
300fun_is(Fun, Expected, Desc) when is_function(Fun) ->
301    is(Fun(Expected), true, Desc).
302
303%% @spec expect_fun(ExpectFun, Got, Desc) -> Result
304%%       ExpectFun = function()
305%%       Got = any()
306%%       Desc = string()
307%%       Result = true | false
308%% @doc Use an anonymous function to assert a pattern match, using actual
309%%       value as the argument to the function.
310expect_fun(ExpectFun, Got, Desc) ->
311    evaluate(ExpectFun(Got), Got, ExpectFun, Desc).
312
313%% @spec expect_fun(ExpectFun, Got, Desc, ExpectStr) -> Result
314%%       ExpectFun = function()
315%%       Got = any()
316%%       Desc = string()
317%%       ExpectStr = string()
318%%       Result = true | false
319%% @doc Use an anonymous function to assert a pattern match, using actual
320%%       value as the argument to the function.
321expect_fun(ExpectFun, Got, Desc, ExpectStr) ->
322    evaluate(ExpectFun(Got), Got, ExpectStr, Desc).
323
324%% @equiv skip(TestFun, "")
325skip(TestFun) when is_function(TestFun) ->
326    skip(TestFun, "").
327
328%% @spec skip(TestFun, Reason) -> ok
329%%       TestFun = function()
330%%       Reason = string()
331%% @doc Skip a test.
332skip(TestFun, Reason) when is_function(TestFun), is_list(Reason) ->
333    begin_skip(Reason),
334    catch TestFun(),
335    end_skip(),
336    ok.
337
338%% @spec skip(Q, TestFun, Reason) -> ok
339%%       Q = true | false | function()
340%%       TestFun = function()
341%%       Reason = string()
342%% @doc Skips a test conditionally. The first argument to this function can
343%% either be the 'true' or 'false' atoms or a function that returns 'true' or
344%% 'false'.
345skip(QFun, TestFun, Reason) when is_function(QFun), is_function(TestFun), is_list(Reason) ->
346    case QFun() of
347        true -> begin_skip(Reason), TestFun(), end_skip();
348        _ -> TestFun()
349    end,
350    ok;
351
352skip(Q, TestFun, Reason) when is_function(TestFun), is_list(Reason), Q == true ->
353    begin_skip(Reason),
354    TestFun(),
355    end_skip(),
356    ok;
357
358skip(_, TestFun, Reason) when is_function(TestFun), is_list(Reason) ->
359    TestFun(),
360    ok.
361
362%% @private
363begin_skip(Reason) ->
364    etap_server ! {self(), begin_skip, Reason}.
365
366%% @private
367end_skip() ->
368    etap_server ! {self(), end_skip}.
369
370%% @spec contains_ok(string(), string(), string()) -> true | false
371%% @doc Assert that a string is contained in another string.
372contains_ok(Source, String, Desc) ->
373    etap:isnt(
374        string:str(Source, String),
375        0,
376        Desc
377    ).
378
379%% @spec is_before(string(), string(), string(), string()) -> true | false
380%% @doc Assert that a string comes before another string within a larger body.
381is_before(Source, StringA, StringB, Desc) ->
382    etap:is_greater(
383        string:str(Source, StringB),
384        string:str(Source, StringA),
385        Desc
386    ).
387
388%% @doc Assert that a given variable is a pid.
389is_pid(Pid, Desc) when is_pid(Pid) -> etap:ok(true, Desc);
390is_pid(_, Desc) -> etap:ok(false, Desc).
391
392%% @doc Assert that a given process/pid is alive.
393is_alive(Pid, Desc) ->
394    etap:ok(erlang:is_process_alive(Pid), Desc).
395
396%% @doc Assert that the current function of a pid is a given {M, F, A} tuple.
397is_mfa(Pid, MFA, Desc) ->
398    etap:is({current_function, MFA}, erlang:process_info(Pid, current_function), Desc).
399
400%% @spec loaded_ok(atom(), string()) -> true | false
401%% @doc Assert that a module has been loaded successfully.
402loaded_ok(M, Desc) when is_atom(M) ->
403    etap:fun_is(fun({module, _}) -> true; (_) -> false end, code:load_file(M), Desc).
404
405%% @spec can_ok(atom(), atom()) -> true | false
406%% @doc Assert that a module exports a given function.
407can_ok(M, F) when is_atom(M), is_atom(F) ->
408    Matches = [X || {X, _} <- M:module_info(exports), X == F],
409    etap:ok(Matches > 0, lists:concat([M, " can ", F])).
410
411%% @spec can_ok(atom(), atom(), integer()) -> true | false
412%% @doc Assert that a module exports a given function with a given arity.
413can_ok(M, F, A) when is_atom(M); is_atom(F), is_number(A) ->
414    Matches = [X || X <- M:module_info(exports), X == {F, A}],
415    etap:ok(Matches > 0, lists:concat([M, " can ", F, "/", A])).
416
417%% @spec has_attrib(M, A) -> true | false
418%%       M = atom()
419%%       A = atom()
420%% @doc Asserts that a module has a given attribute.
421has_attrib(M, A) when is_atom(M), is_atom(A) ->
422    etap:isnt(
423        proplists:get_value(A, M:module_info(attributes), 'asdlkjasdlkads'),
424        'asdlkjasdlkads',
425        lists:concat([M, " has attribute ", A])
426    ).
427
428%% @spec has_attrib(M, A. V) -> true | false
429%%       M = atom()
430%%       A = atom()
431%%       V = any()
432%% @doc Asserts that a module has a given attribute with a given value.
433is_attrib(M, A, V) when is_atom(M) andalso is_atom(A) ->
434    etap:is(
435        proplists:get_value(A, M:module_info(attributes)),
436        [V],
437        lists:concat([M, "'s ", A, " is ", V])
438    ).
439
440%% @spec is_behavior(M, B) -> true | false
441%%       M = atom()
442%%       B = atom()
443%% @doc Asserts that a given module has a specific behavior.
444is_behaviour(M, B) when is_atom(M) andalso is_atom(B) ->
445    is_attrib(M, behaviour, B).
446
447%% @doc Assert that an exception is raised when running a given function.
448dies_ok(F, Desc) ->
449    case (catch F()) of
450        {'EXIT', _} -> etap:ok(true, Desc);
451        _ -> etap:ok(false, Desc)
452    end.
453
454%% @doc Assert that an exception is not raised when running a given function.
455lives_ok(F, Desc) ->
456    etap:is(try_this(F), success, Desc).
457
458%% @doc Assert that the exception thrown by a function matches the given exception.
459throws_ok(F, Exception, Desc) ->
460    try F() of
461        _ -> etap:ok(nok, Desc)
462    catch
463        _:E ->
464            etap:is(E, Exception, Desc)
465    end.
466
467%% @private
468%% @doc Run a function and catch any exceptions.
469try_this(F) when is_function(F, 0) ->
470    try F() of
471        _ -> success
472    catch
473        throw:E -> {throw, E};
474        error:E -> {error, E};
475        exit:E -> {exit, E}
476    end.
477
478%% @private
479%% @doc Start the etap_server process if it is not running already.
480ensure_test_server() ->
481    case whereis(etap_server) of
482        undefined ->
483            proc_lib:start(?MODULE, start_etap_server,[]);
484        _ ->
485            diag("The test server is already running.")
486    end.
487
488%% @private
489%% @doc Start the etap_server loop and register itself as the etap_server
490%% process.
491start_etap_server() ->
492    catch register(etap_server, self()),
493    proc_lib:init_ack(ok),
494    etap:test_server(#test_state{
495        planned = 0,
496        count = 0,
497        pass = 0,
498        fail = 0,
499        skip = 0,
500        skip_reason = ""
501    }).
502
503
504%% @private
505%% @doc The main etap_server receive/run loop. The etap_server receive loop
506%% responds to seven messages apperatining to failure or passing of tests.
507%% It is also used to initiate the testing process with the {_, plan, _}
508%% message that clears the current test state.
509test_server(State) ->
510    NewState = receive
511        {_From, plan, unknown} ->
512            io:format("# Current time local ~s~n", [datetime(erlang:localtime())]),
513            io:format("# Using etap version ~p~n", [ proplists:get_value(vsn, proplists:get_value(attributes, etap:module_info())) ]),
514            State#test_state{
515                planned = -1,
516                count = 0,
517                pass = 0,
518                fail = 0,
519                skip = 0,
520                skip_reason = ""
521            };
522        {_From, plan, N} ->
523            io:format("# Current time local ~s~n", [datetime(erlang:localtime())]),
524            io:format("# Using etap version ~p~n", [ proplists:get_value(vsn, proplists:get_value(attributes, etap:module_info())) ]),
525            io:format("1..~p~n", [N]),
526            State#test_state{
527                planned = N,
528                count = 0,
529                pass = 0,
530                fail = 0,
531                skip = 0,
532                skip_reason = ""
533            };
534        {_From, begin_skip, Reason} ->
535            State#test_state{
536                skip = 1,
537                skip_reason = Reason
538            };
539        {_From, end_skip} ->
540            State#test_state{
541                skip = 0,
542                skip_reason = ""
543            };
544        {_From, pass, Desc} ->
545            FullMessage = skip_diag(
546                " - " ++ Desc,
547                State#test_state.skip,
548                State#test_state.skip_reason
549            ),
550            io:format("ok ~p ~s~n", [State#test_state.count + 1, FullMessage]),
551            State#test_state{
552                count = State#test_state.count + 1,
553                pass = State#test_state.pass + 1
554            };
555
556        {_From, fail, Desc} ->
557            FullMessage = skip_diag(
558                " - " ++ Desc,
559                State#test_state.skip,
560                State#test_state.skip_reason
561            ),
562            io:format("not ok ~p ~s~n", [State#test_state.count + 1, FullMessage]),
563            State#test_state{
564                count = State#test_state.count + 1,
565                fail = State#test_state.fail + 1
566            };
567        {From, state, Ref} ->
568            From ! {Ref, State},
569            State;
570        {_From, diag, Message} ->
571            io:format("~s~n", [Message]),
572            State;
573        {From, count} ->
574            From ! State#test_state.count,
575            State;
576        {From, is_skip, Ref} ->
577            From ! {Ref, State#test_state.skip},
578            State;
579        done ->
580            exit(normal)
581    end,
582    test_server(NewState).
583
584%% @private
585%% @doc Process the result of a test and send it to the etap_server process.
586mk_tap(Result, Desc) ->
587    etap_server ! {self(), is_skip, Ref = make_ref()} ,
588    receive {Ref, IsSkip} -> ok end,
589    case [IsSkip, Result] of
590        [_, true] ->
591            etap_server ! {self(), pass, Desc},
592            true;
593        [1, _] ->
594            etap_server ! {self(), pass, Desc},
595            true;
596        _ ->
597            etap_server ! {self(), fail, Desc},
598            false
599    end.
600
601%% @private
602%% @doc Format a date/time string.
603datetime(DateTime) ->
604    {{Year, Month, Day}, {Hour, Min, Sec}} = DateTime,
605    io_lib:format("~4.10.0B-~2.10.0B-~2.10.0B ~2.10.0B:~2.10.0B:~2.10.0B", [Year, Month, Day, Hour, Min, Sec]).
606
607%% @private
608%% @doc Craft an output message taking skip/todo into consideration.
609skip_diag(Message, 0, _) ->
610    Message;
611skip_diag(_Message, 1, "") ->
612    " # SKIP";
613skip_diag(_Message, 1, Reason) ->
614    " # SKIP : " ++ Reason.
615