xref: /5.5.2/couchdb/src/etap/etap.erl (revision 610eba80)
1%% Copyright (c) 2008-2009 Nick Gerakines <nick@gerakines.net>
2%%
3%% Permission is hereby granted, free of charge, to any person
4%% obtaining a copy of this software and associated documentation
5%% files (the "Software"), to deal in the Software without
6%% restriction, including without limitation the rights to use,
7%% copy, modify, merge, publish, distribute, sublicense, and/or sell
8%% copies of the Software, and to permit persons to whom the
9%% Software is furnished to do so, subject to the following
10%% conditions:
11%%
12%% The above copyright notice and this permission notice shall be
13%% included in all copies or substantial portions of the Software.
14%%
15%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16%% EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
17%% OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18%% NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
19%% HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
20%% WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21%% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22%% OTHER DEALINGS IN THE SOFTWARE.
23%%
24%% @author Nick Gerakines <nick@gerakines.net> [http://socklabs.com/]
25%% @author Jeremy Wall <jeremy@marzhillstudios.com>
26%% @version 0.3.4
27%% @copyright 2007-2008 Jeremy Wall, 2008-2009 Nick Gerakines
28%% @reference http://testanything.org/wiki/index.php/Main_Page
29%% @reference http://en.wikipedia.org/wiki/Test_Anything_Protocol
30%% @todo Finish implementing the skip directive.
31%% @todo Document the messages handled by this receive loop.
32%% @todo Explain in documentation why we use a process to handle test input.
33%% @doc etap is a TAP testing module for Erlang components and applications.
34%% This module allows developers to test their software using the TAP method.
35%%
36%% <blockquote cite="http://en.wikipedia.org/wiki/Test_Anything_Protocol"><p>
37%% TAP, the Test Anything Protocol, is a simple text-based interface between
38%% testing modules in a test harness. TAP started life as part of the test
39%% harness for Perl but now has implementations in C/C++, Python, PHP, Perl
40%% and probably others by the time you read this.
41%% </p></blockquote>
42%%
43%% The testing process begins by defining a plan using etap:plan/1, running
44%% a number of etap tests and then calling eta:end_tests/0. Please refer to
45%% the Erlang modules in the t directory of this project for example tests.
46-module(etap).
47-export([
48    ensure_test_server/0, start_etap_server/0, test_server/1,
49    diag/1, diag/2, plan/1, end_tests/0, not_ok/2, ok/2, is/3, isnt/3,
50    any/3, none/3, fun_is/3, is_greater/3, skip/1, skip/2,
51    ensure_coverage_starts/0, ensure_coverage_ends/0, coverage_report/0,
52    datetime/1, skip/3, bail/0, bail/1
53]).
54-record(test_state, {planned = 0, count = 0, pass = 0, fail = 0, skip = 0, skip_reason = ""}).
55-vsn("0.3.4").
56
57%% @spec plan(N) -> Result
58%%       N = unknown | skip | {skip, string()} | integer()
59%%       Result = ok
60%% @doc Create a test plan and boot strap the test server.
61plan(unknown) ->
62    ensure_coverage_starts(),
63    ensure_test_server(),
64    etap_server ! {self(), plan, unknown},
65    ok;
66plan(skip) ->
67    io:format("1..0 # skip~n");
68plan({skip, Reason}) ->
69    io:format("1..0 # skip ~s~n", [Reason]);
70plan(N) when is_integer(N), N > 0 ->
71    ensure_coverage_starts(),
72    ensure_test_server(),
73    etap_server ! {self(), plan, N},
74    ok.
75
76%% @spec end_tests() -> ok
77%% @doc End the current test plan and output test results.
78%% @todo This should probably be done in the test_server process.
79end_tests() ->
80    ensure_coverage_ends(),
81    etap_server ! {self(), state, Ref = make_ref()},
82    State = receive {Ref, X} -> X end,
83    if
84        State#test_state.planned == -1 ->
85            io:format("1..~p~n", [State#test_state.count]);
86        true ->
87            ok
88    end,
89    case whereis(etap_server) of
90        undefined -> ok;
91        _ -> etap_server ! done, ok
92    end.
93
94%% @private
95ensure_coverage_starts() ->
96    case os:getenv("COVER") of
97        false -> ok;
98        _ ->
99            BeamDir = case os:getenv("COVER_BIN") of false -> "ebin"; X -> X end,
100            cover:compile_beam_directory(BeamDir)
101    end.
102
103%% @private
104%% @doc Attempts to write out any collected coverage data to the cover/
105%% directory. This function should not be called externally, but it could be.
106ensure_coverage_ends() ->
107    case os:getenv("COVER") of
108        false -> ok;
109        _ ->
110            filelib:ensure_dir("cover/"),
111            Name = lists:flatten([
112                io_lib:format("~.16b", [X]) || X <- binary_to_list(erlang:md5(
113                     term_to_binary({make_ref(), now()})
114                ))
115            ]),
116            cover:export("cover/" ++ Name ++ ".coverdata")
117    end.
118
119%% @spec coverage_report() -> ok
120%% @doc Use the cover module's covreage report builder to create code coverage
121%% reports from recently created coverdata files.
122coverage_report() ->
123    [cover:import(File) || File <- filelib:wildcard("cover/*.coverdata")],
124    lists:foreach(
125        fun(Mod) ->
126            cover:analyse_to_file(Mod, atom_to_list(Mod) ++ "_coverage.txt", [])
127        end,
128        cover:imported_modules()
129    ),
130    ok.
131
132bail() ->
133    bail("").
134
135bail(Reason) ->
136    etap_server ! {self(), diag, "Bail out! " ++ Reason},
137    ensure_coverage_ends(),
138    etap_server ! done, ok,
139    ok.
140
141
142%% @spec diag(S) -> ok
143%%       S = string()
144%% @doc Print a debug/status message related to the test suite.
145diag(S) -> etap_server ! {self(), diag, "# " ++ S}, ok.
146
147%% @spec diag(Format, Data) -> ok
148%%      Format = atom() | string() | binary()
149%%      Data = [term()]
150%%      UnicodeList = [Unicode]
151%%      Unicode = int()
152%% @doc Print a debug/status message related to the test suite.
153%% Function arguments are passed through io_lib:format/2.
154diag(Format, Data) -> diag(io_lib:format(Format, Data)).
155
156%% @spec ok(Expr, Desc) -> Result
157%%       Expr = true | false
158%%       Desc = string()
159%%       Result = true | false
160%% @doc Assert that a statement is true.
161ok(Expr, Desc) -> mk_tap(Expr == true, Desc).
162
163%% @spec not_ok(Expr, Desc) -> Result
164%%       Expr = true | false
165%%       Desc = string()
166%%       Result = true | false
167%% @doc Assert that a statement is false.
168not_ok(Expr, Desc) -> mk_tap(Expr == false, Desc).
169
170%% @spec is(Got, Expected, Desc) -> Result
171%%       Got = any()
172%%       Expected = any()
173%%       Desc = string()
174%%       Result = true | false
175%% @doc Assert that two values are the same.
176is(Got, Expected, Desc) ->
177    case mk_tap(Got == Expected, Desc) of
178        false ->
179            etap_server ! {self(), diag, "    ---"},
180            etap_server ! {self(), diag, io_lib:format("    description: ~p", [Desc])},
181            etap_server ! {self(), diag, io_lib:format("    found:       ~p", [Got])},
182            etap_server ! {self(), diag, io_lib:format("    wanted:      ~p", [Expected])},
183            etap_server ! {self(), diag, "    ..."},
184            false;
185        true -> true
186    end.
187
188%% @spec isnt(Got, Expected, Desc) -> Result
189%%       Got = any()
190%%       Expected = any()
191%%       Desc = string()
192%%       Result = true | false
193%% @doc Assert that two values are not the same.
194isnt(Got, Expected, Desc) -> mk_tap(Got /= Expected, Desc).
195
196%% @spec is_greater(ValueA, ValueB, Desc) -> Result
197%%       ValueA = number()
198%%       ValueB = number()
199%%       Desc = string()
200%%       Result = true | false
201%% @doc Assert that an integer is greater than another.
202is_greater(ValueA, ValueB, Desc) when is_integer(ValueA), is_integer(ValueB) ->
203    mk_tap(ValueA > ValueB, Desc).
204
205%% @spec any(Got, Items, Desc) -> Result
206%%       Got = any()
207%%       Items = [any()]
208%%       Desc = string()
209%%       Result = true | false
210%% @doc Assert that an item is in a list.
211any(Got, Items, Desc) ->
212    is(lists:member(Got, Items), true, Desc).
213
214%% @spec none(Got, Items, Desc) -> Result
215%%       Got = any()
216%%       Items = [any()]
217%%       Desc = string()
218%%       Result = true | false
219%% @doc Assert that an item is not in a list.
220none(Got, Items, Desc) ->
221    is(lists:member(Got, Items), false, Desc).
222
223%% @spec fun_is(Fun, Expected, Desc) -> Result
224%%       Fun = function()
225%%       Expected = any()
226%%       Desc = string()
227%%       Result = true | false
228%% @doc Use an anonymous function to assert a pattern match.
229fun_is(Fun, Expected, Desc) when is_function(Fun) ->
230    is(Fun(Expected), true, Desc).
231
232%% @equiv skip(TestFun, "")
233skip(TestFun) when is_function(TestFun) ->
234    skip(TestFun, "").
235
236%% @spec skip(TestFun, Reason) -> ok
237%%       TestFun = function()
238%%       Reason = string()
239%% @doc Skip a test.
240skip(TestFun, Reason) when is_function(TestFun), is_list(Reason) ->
241    begin_skip(Reason),
242    catch TestFun(),
243    end_skip(),
244    ok.
245
246%% @spec skip(Q, TestFun, Reason) -> ok
247%%       Q = true | false | function()
248%%       TestFun = function()
249%%       Reason = string()
250%% @doc Skips a test conditionally. The first argument to this function can
251%% either be the 'true' or 'false' atoms or a function that returns 'true' or
252%% 'false'.
253skip(QFun, TestFun, Reason) when is_function(QFun), is_function(TestFun), is_list(Reason) ->
254    case QFun() of
255        true -> begin_skip(Reason), TestFun(), end_skip();
256        _ -> TestFun()
257    end,
258    ok;
259
260skip(Q, TestFun, Reason) when is_function(TestFun), is_list(Reason), Q == true ->
261    begin_skip(Reason),
262    TestFun(),
263    end_skip(),
264    ok;
265
266skip(_, TestFun, Reason) when is_function(TestFun), is_list(Reason) ->
267    TestFun(),
268    ok.
269
270%% @private
271begin_skip(Reason) ->
272    etap_server ! {self(), begin_skip, Reason}.
273
274%% @private
275end_skip() ->
276    etap_server ! {self(), end_skip}.
277
278% ---
279% Internal / Private functions
280
281%% @private
282%% @doc Start the etap_server process if it is not running already.
283ensure_test_server() ->
284    case whereis(etap_server) of
285        undefined ->
286            proc_lib:start(?MODULE, start_etap_server,[]);
287        _ ->
288            diag("The test server is already running.")
289    end.
290
291%% @private
292%% @doc Start the etap_server loop and register itself as the etap_server
293%% process.
294start_etap_server() ->
295    catch register(etap_server, self()),
296    proc_lib:init_ack(ok),
297    etap:test_server(#test_state{
298        planned = 0,
299        count = 0,
300        pass = 0,
301        fail = 0,
302        skip = 0,
303        skip_reason = ""
304    }).
305
306
307%% @private
308%% @doc The main etap_server receive/run loop. The etap_server receive loop
309%% responds to seven messages apperatining to failure or passing of tests.
310%% It is also used to initiate the testing process with the {_, plan, _}
311%% message that clears the current test state.
312test_server(State) ->
313    NewState = receive
314        {_From, plan, unknown} ->
315            io:format("# Current time local ~s~n", [datetime(erlang:localtime())]),
316            io:format("# Using etap version ~p~n", [ proplists:get_value(vsn, proplists:get_value(attributes, etap:module_info())) ]),
317            State#test_state{
318                planned = -1,
319                count = 0,
320                pass = 0,
321                fail = 0,
322                skip = 0,
323                skip_reason = ""
324            };
325        {_From, plan, N} ->
326            io:format("# Current time local ~s~n", [datetime(erlang:localtime())]),
327            io:format("# Using etap version ~p~n", [ proplists:get_value(vsn, proplists:get_value(attributes, etap:module_info())) ]),
328            io:format("1..~p~n", [N]),
329            State#test_state{
330                planned = N,
331                count = 0,
332                pass = 0,
333                fail = 0,
334                skip = 0,
335                skip_reason = ""
336            };
337        {_From, begin_skip, Reason} ->
338            State#test_state{
339                skip = 1,
340                skip_reason = Reason
341            };
342        {_From, end_skip} ->
343            State#test_state{
344                skip = 0,
345                skip_reason = ""
346            };
347        {_From, pass, Desc} ->
348            FullMessage = skip_diag(
349                " - " ++ Desc,
350                State#test_state.skip,
351                State#test_state.skip_reason
352            ),
353            io:format("ok ~p ~s~n", [State#test_state.count + 1, FullMessage]),
354            State#test_state{
355                count = State#test_state.count + 1,
356                pass = State#test_state.pass + 1
357            };
358
359        {_From, fail, Desc} ->
360            FullMessage = skip_diag(
361                " - " ++ Desc,
362                State#test_state.skip,
363                State#test_state.skip_reason
364            ),
365            io:format("not ok ~p ~s~n", [State#test_state.count + 1, FullMessage]),
366            State#test_state{
367                count = State#test_state.count + 1,
368                fail = State#test_state.fail + 1
369            };
370        {From, state, Ref} ->
371            From ! {Ref, State},
372            State;
373        {_From, diag, Message} ->
374            io:format("~s~n", [Message]),
375            State;
376        {From, count} ->
377            From ! State#test_state.count,
378            State;
379        {From, is_skip, Ref} ->
380            From ! {Ref, State#test_state.skip},
381            State;
382        done ->
383            exit(normal)
384    end,
385    test_server(NewState).
386
387%% @private
388%% @doc Process the result of a test and send it to the etap_server process.
389mk_tap(Result, Desc) ->
390    etap_server ! {self(), is_skip, Ref = make_ref()} ,
391    receive {Ref, IsSkip} ->ok end,
392    case [IsSkip, Result] of
393        [_, true] ->
394            etap_server ! {self(), pass, Desc},
395            true;
396        [1, _] ->
397            etap_server ! {self(), pass, Desc},
398            true;
399        _ ->
400            etap_server ! {self(), fail, Desc},
401            false
402    end.
403
404%% @private
405%% @doc Format a date/time string.
406datetime(DateTime) ->
407    {{Year, Month, Day}, {Hour, Min, Sec}} = DateTime,
408    io_lib:format("~4.10.0B-~2.10.0B-~2.10.0B ~2.10.0B:~2.10.0B:~2.10.0B", [Year, Month, Day, Hour, Min, Sec]).
409
410%% @private
411%% @doc Craft an output message taking skip/todo into consideration.
412skip_diag(Message, 0, _) ->
413    Message;
414skip_diag(_Message, 1, "") ->
415    " # SKIP";
416skip_diag(_Message, 1, Reason) ->
417    " # SKIP : " ++ Reason.
418