1#!/usr/bin/env escript
2%% -*- Mode: Erlang; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
3%%! -smp enable
4
5% Licensed under the Apache License, Version 2.0 (the "License"); you may not
6% use this file except in compliance with the License. You may obtain a copy of
7% the License at
8%
9%   http://www.apache.org/licenses/LICENSE-2.0
10%
11% Unless required by applicable law or agreed to in writing, software
12% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14% License for the specific language governing permissions and limitations under
15% the License.
16
17-define(JSON_ENCODE(V), ejson:encode(V)). % couch_db.hrl
18-define(MAX_WAIT_TIME, 900 * 1000).
19
20-include_lib("couch_set_view/include/couch_set_view.hrl").
21
22test_set_name() -> <<"couch_test_set_index_util_stats">>.
23num_set_partitions() -> 64.
24ddoc_id() -> <<"_design/test">>.
25num_docs_0() -> 78144.  % keep it a multiple of num_set_partitions()
26
27
28main(_) ->
29    test_util:init_code_path(),
30
31    etap:plan(24),
32    case (catch test()) of
33        ok ->
34            etap:end_tests();
35        Other ->
36            etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
37            etap:bail(Other)
38    end,
39    ok.
40
41
42test() ->
43    couch_set_view_test_util:start_server(test_set_name()),
44
45    couch_set_view_test_util:delete_set_dbs(test_set_name(), num_set_partitions()),
46    couch_set_view_test_util:create_set_dbs(test_set_name(), num_set_partitions()),
47
48    create_set(),
49
50    ValueGenFun1 = fun(I) -> I end,
51    update_documents(0, num_docs_0(), ValueGenFun1),
52
53    GroupPid = couch_set_view:get_group_pid(
54        mapreduce_view, test_set_name(), ddoc_id(), prod),
55
56    {ok, Stats0} = couch_set_view:get_utilization_stats(
57        mapreduce_view, test_set_name(), ddoc_id()),
58    etap:is(couch_util:get_value(updates, Stats0), 0,
59            "Right number of updates"),
60    etap:is(couch_util:get_value(updater_interruptions, Stats0), 0,
61            "Right number of updater interruptions"),
62    etap:is(couch_util:get_value(useful_indexing_time, Stats0), 0.0,
63            "Right useful indexing time"),
64    etap:is(couch_util:get_value(wasted_indexing_time, Stats0), 0.0,
65            "Right wasted indexing time"),
66    etap:is(couch_util:get_value(compactions, Stats0), 0,
67            "Right number of compactions"),
68    etap:is(couch_util:get_value(compactor_interruptions, Stats0), 0,
69            "Right number of compactor interruptions"),
70    etap:is(couch_util:get_value(compaction_time, Stats0), 0.0,
71            "Right compaction time"),
72
73    {ok, UpdaterPid1} = gen_server:call(GroupPid, {start_updater, []}, infinity),
74    Ref1 = erlang:monitor(process, UpdaterPid1),
75    etap:diag("Waiting for updater to finish"),
76    receive
77    {'DOWN', Ref1, _, _, {updater_finished, _}} ->
78        etap:diag("Updater finished");
79    {'DOWN', Ref1, _, _, Reason1} ->
80        etap:bail("Updater finished with unexpected reason: " ++ couch_util:to_list(Reason1))
81    after ?MAX_WAIT_TIME ->
82        etap:bail("Timeout waiting for updater to finish")
83    end,
84
85    {ok, CompactPid1} = couch_set_view_compactor:start_compact(
86        mapreduce_view, test_set_name(), ddoc_id(), main),
87    Ref2 = erlang:monitor(process, CompactPid1),
88    etap:diag("Waiting for main group compaction to finish"),
89    receive
90    {'DOWN', Ref2, _, _, normal} ->
91        ok;
92    {'DOWN', Ref2, _, _, noproc} ->
93        ok;
94    {'DOWN', Ref2, _, _, Reason2} ->
95        etap:bail("Failure compacting main group: " ++ couch_util:to_list(Reason2))
96    after ?MAX_WAIT_TIME ->
97        etap:bail("Timeout waiting for main group compaction to finish")
98    end,
99
100    update_documents(num_docs_0(), 64, ValueGenFun1),
101
102    {ok, UpdaterPid2} = gen_server:call(GroupPid, {start_updater, [pause]}, infinity),
103    Ref3 = erlang:monitor(process, UpdaterPid2),
104
105    etap:diag("Marking partition 0 for cleanup while updater is running"),
106    ok = couch_set_view:set_partition_states(
107        mapreduce_view, test_set_name(), ddoc_id(), [], [], [0]),
108
109    etap:diag("Waiting for updater to be shutdown"),
110    receive
111    {'DOWN', Ref3, _, _, {updater_error, shutdown}} ->
112        etap:diag("Updater shutdown")
113    after 5000 ->
114        etap:bail("Timeout waiting for updater to shutdown")
115    end,
116
117    {ok, UpdaterPid3} = gen_server:call(GroupPid, {start_updater, []}, infinity),
118    case is_pid(UpdaterPid3) of
119    true ->
120        Ref4 = erlang:monitor(process, UpdaterPid3),
121        etap:diag("Waiting for new updater to finish"),
122        receive
123        {'DOWN', Ref4, _, _, {updater_finished, _}} ->
124            etap:diag("Updater finished");
125        {'DOWN', Ref4, _, _, Reason4} ->
126            etap:bail("Updater finished with unexpected reason: " ++ couch_util:to_list(Reason4))
127        after ?MAX_WAIT_TIME ->
128            etap:bail("Timeout waiting for new updater to finish")
129        end;
130    false ->
131        ok
132    end,
133
134    {ok, CompactPid2} = couch_set_view_compactor:start_compact(
135        mapreduce_view, test_set_name(), ddoc_id(), main),
136    CompactPid2 ! pause,
137    Ref5 = erlang:monitor(process, CompactPid2),
138    etap:diag("Marking partition 1 for cleanup while compactor is running"),
139    ok = couch_set_view:set_partition_states(
140        mapreduce_view, test_set_name(), ddoc_id(), [], [], [1]),
141
142    etap:diag("Waiting for main group compaction to shutdown"),
143    receive
144    {'DOWN', Ref5, _, _, shutdown} ->
145        ok;
146    {'DOWN', Ref5, _, _, Reason5} ->
147        etap:bail("Failure compacting main group: " ++ couch_util:to_list(Reason5))
148    after 5000 ->
149        etap:bail("Timeout waiting for main group compaction to shutdown")
150    end,
151
152    {ok, Stats2} = couch_set_view:get_utilization_stats(
153        mapreduce_view, test_set_name(), ddoc_id()),
154    etap:is(couch_util:get_value(updates, Stats2), 2,
155            "Right number of updates"),
156    etap:is(couch_util:get_value(updater_interruptions, Stats2), 1,
157            "Right number of updater interruptions"),
158    etap:is(is_float(couch_util:get_value(useful_indexing_time, Stats2)), true,
159            "Useful indexing time is a float"),
160    etap:is(couch_util:get_value(useful_indexing_time, Stats2) > 0, true,
161            "Useful indexing time greater than zero"),
162    etap:is(is_float(couch_util:get_value(wasted_indexing_time, Stats2)), true,
163            "Wasted indexing time is a float"),
164    etap:is(couch_util:get_value(wasted_indexing_time, Stats2) > 0, true,
165            "Wasted indexing time greater than zero"),
166    etap:is(couch_util:get_value(compactions, Stats2), 1,
167            "Right number of compactions"),
168    etap:is(couch_util:get_value(compactor_interruptions, Stats2), 1,
169            "Right number of compactor interruptions"),
170    etap:is(is_float(couch_util:get_value(compaction_time, Stats2)), true,
171            "Compaction time is a float"),
172    etap:is(couch_util:get_value(compaction_time, Stats2) > 0, true,
173            "Compaction time is greater than zero"),
174
175    etap:diag("Reseting stats"),
176    ok = couch_set_view:reset_utilization_stats(
177        mapreduce_view, test_set_name(), ddoc_id()),
178    {ok, Stats3} = couch_set_view:get_utilization_stats(
179        mapreduce_view, test_set_name(), ddoc_id()),
180    etap:is(couch_util:get_value(updates, Stats3), 0,
181            "Right number of updates"),
182    etap:is(couch_util:get_value(updater_interruptions, Stats3), 0,
183            "Right number of updater interruptions"),
184    etap:is(couch_util:get_value(useful_indexing_time, Stats3), 0.0,
185            "Right useful indexing time"),
186    etap:is(couch_util:get_value(wasted_indexing_time, Stats3), 0.0,
187            "Right wasted indexing time"),
188    etap:is(couch_util:get_value(compactions, Stats3), 0,
189            "Right number of compactions"),
190    etap:is(couch_util:get_value(compactor_interruptions, Stats3), 0,
191            "Right number of compactor interruptions"),
192    etap:is(couch_util:get_value(compaction_time, Stats3), 0.0,
193            "Right compaction time"),
194
195    couch_set_view_test_util:delete_set_dbs(test_set_name(), num_set_partitions()),
196    couch_set_view_test_util:stop_server(),
197    ok.
198
199
200create_set() ->
201    couch_set_view_test_util:delete_set_dbs(test_set_name(), num_set_partitions()),
202    couch_set_view_test_util:create_set_dbs(test_set_name(), num_set_partitions()),
203    couch_set_view:cleanup_index_files(mapreduce_view, test_set_name()),
204    etap:diag("Creating the set databases (# of partitions: " ++
205        integer_to_list(num_set_partitions()) ++ ")"),
206    DDoc = {[
207        {<<"meta">>, {[{<<"id">>, ddoc_id()}]}},
208        {<<"json">>, {[
209        {<<"views">>, {[
210            {<<"view_1">>, {[
211                {<<"map">>, <<"function(doc, meta) { emit(meta.id, doc.value); }">>},
212                {<<"reduce">>, <<"_count">>}
213            ]}}
214        ]}}
215        ]}}
216    ]},
217    ok = couch_set_view_test_util:update_ddoc(test_set_name(), DDoc),
218    etap:diag("Configuring set view with partitions [0 .. 63] as active"),
219    Params = #set_view_params{
220        max_partitions = num_set_partitions(),
221        active_partitions = lists:seq(0, num_set_partitions() - 1),
222        passive_partitions = [],
223        use_replica_index = false
224    },
225    ok = couch_set_view:define_group(
226        mapreduce_view, test_set_name(), ddoc_id(), Params).
227
228
229update_documents(StartId, Count, ValueGenFun) ->
230    etap:diag("Updating " ++ integer_to_list(Count) ++ " documents"),
231    DocList0 = lists:map(
232        fun(I) ->
233            {I rem num_set_partitions(), {[
234                {<<"meta">>, {[{<<"id">>, doc_id(I)}]}},
235                {<<"json">>, {[
236                    {<<"value">>, ValueGenFun(I)}
237                ]}}
238            ]}}
239        end,
240        lists:seq(StartId, StartId + Count - 1)),
241    DocList = [Doc || {_, Doc} <- lists:keysort(1, DocList0)],
242    ok = couch_set_view_test_util:populate_set_sequentially(
243        test_set_name(),
244        lists:seq(0, num_set_partitions() - 1),
245        DocList).
246
247
248doc_id(I) ->
249    iolist_to_binary(io_lib:format("doc_~8..0b", [I])).
250