1#!/usr/bin/env escript
2%% -*- Mode: Erlang; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
3%%! -smp enable
4
5% Licensed under the Apache License, Version 2.0 (the "License"); you may not
6% use this file except in compliance with the License. You may obtain a copy of
7% the License at
8%
9%   http://www.apache.org/licenses/LICENSE-2.0
10%
11% Unless required by applicable law or agreed to in writing, software
12% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14% License for the specific language governing permissions and limitations under
15% the License.
16
17-include_lib("couch_set_view/include/couch_set_view.hrl").
18
19-define(JSON_ENCODE(V), ejson:encode(V)). % couch_db.hrl
20-define(MAX_WAIT_TIME, 900 * 1000).
21
22test_set_name() -> <<"couch_test_set_index_cleanups">>.
23num_set_partitions() -> 64.
24ddoc_id() -> <<"_design/test">>.
25num_docs() -> 21312.  % keep it a multiple of num_set_partitions()
26
27
28main(_) ->
29    test_util:init_code_path(),
30
31    etap:plan(770),
32    case (catch test()) of
33        ok ->
34            etap:end_tests();
35        Other ->
36            etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
37            etap:bail(Other)
38    end,
39    ok.
40
41
42test() ->
43    couch_set_view_test_util:start_server(test_set_name()),
44
45    create_set(),
46    add_documents(0, num_docs()),
47    GroupPid = couch_set_view:get_group_pid(
48        mapreduce_view, test_set_name(), ddoc_id(), prod),
49    erlang:put(group_pid, GroupPid),
50
51    % build index
52    _ = get_group_snapshot(),
53
54    ActivePartitions1 = lists:seq(0, 63),
55    ExpectedReduceValue1 = 3 * lists:sum(lists:seq(0, num_docs() - 1)),
56    FoldFun = fun(PartId, {ActivePartsAcc, RedValueAcc}) ->
57        ActivePartsAcc2 = ordsets:del_element(PartId, ActivePartsAcc),
58        RedValueAcc2 = RedValueAcc - (3 * lists:sum(
59            lists:seq(PartId, num_docs() - 1, num_set_partitions())
60        )),
61        ok = couch_set_view:set_partition_states(
62            mapreduce_view, test_set_name(), ddoc_id(), [], [], [PartId]),
63        wait_for_cleanup(),
64        verify_btrees(ActivePartsAcc2, RedValueAcc2),
65        {ActivePartsAcc2, RedValueAcc2}
66    end,
67
68    etap:diag("Starting phase 1 cleanup"),
69
70    {ActivePartitions2, ExpectedReduceValue2} = lists:foldl(
71        FoldFun,
72        {ActivePartitions1, ExpectedReduceValue1},
73        lists:seq(1, 63, 2)),
74
75    etap:diag("Phase 1 cleanup finished"),
76
77    etap:is(
78        ActivePartitions2,
79        lists:seq(0, 63, 2),
80        "Right list of active partitions after first cleanup phase"),
81
82    etap:diag("Starting phase 2 cleanup"),
83
84    {ActivePartitions3, _} = lists:foldl(
85        FoldFun,
86        {ActivePartitions2, ExpectedReduceValue2},
87        lists:reverse(lists:seq(0, 63, 2))),
88
89    etap:diag("Phase 2 cleanup finished"),
90
91    etap:is(
92        ActivePartitions3,
93        [],
94        "Right list of active partitions after second cleanup phase"),
95
96    couch_set_view_test_util:delete_set_dbs(test_set_name(), num_set_partitions()),
97    couch_set_view_test_util:stop_server(),
98    ok.
99
100
101get_group_snapshot() ->
102    GroupPid = couch_set_view:get_group_pid(
103        mapreduce_view, test_set_name(), ddoc_id(), prod),
104    {ok, Group, 0} = gen_server:call(
105        GroupPid, #set_view_group_req{stale = false, debug = true}, infinity),
106    Group.
107
108
109wait_for_cleanup() ->
110    {ok, Pid} = gen_server:call(erlang:get(group_pid), cleaner_pid, infinity),
111    CheckCleanupList = case Pid of
112    nil ->
113        true;
114    _ ->
115        Ref = erlang:monitor(process, Pid),
116        receive
117        {'DOWN', Ref, process, Pid, noproc} ->
118            true;
119        {'DOWN', Ref, process, Pid, {clean_group, _, _, _}} ->
120            false
121        after ?MAX_WAIT_TIME ->
122            etap:bail("Timeout waiting for index cleanup")
123        end
124    end,
125    case CheckCleanupList of
126    true ->
127        GroupInfo = get_group_info(),
128        case couch_util:get_value(cleanup_partitions, GroupInfo) of
129        [] ->
130            ok;
131        _ ->
132            etap:bail("Cleanup was not triggered")
133        end;
134    false ->
135        ok
136    end.
137
138
139get_group_info() ->
140    {ok, Info} = couch_set_view:get_group_info(
141        mapreduce_view, test_set_name(), ddoc_id(), prod),
142    Info.
143
144
145create_set() ->
146    couch_set_view_test_util:delete_set_dbs(test_set_name(), num_set_partitions()),
147    couch_set_view_test_util:create_set_dbs(test_set_name(), num_set_partitions()),
148    couch_set_view:cleanup_index_files(mapreduce_view, test_set_name()),
149    etap:diag("Creating the set databases (# of partitions: " ++
150        integer_to_list(num_set_partitions()) ++ ")"),
151    DDoc = {[
152        {<<"meta">>, {[{<<"id">>, ddoc_id()}]}},
153        {<<"json">>, {[
154        {<<"language">>, <<"javascript">>},
155        {<<"views">>, {[
156            {<<"view_1">>, {[
157                {<<"map">>, <<"function(doc, meta) { emit(meta.id, doc.value); }">>},
158                {<<"reduce">>, <<"_count">>}
159            ]}},
160            {<<"view_2">>, {[
161                {<<"map">>, <<"function(doc, meta) { emit(meta.id, doc.value * 3); }">>},
162                {<<"reduce">>, <<"_sum">>}
163            ]}}
164        ]}}
165        ]}}
166    ]},
167    ok = couch_set_view_test_util:update_ddoc(test_set_name(), DDoc),
168    etap:diag("Configuring set view with partitions [0 .. 63] as active"),
169    Params = #set_view_params{
170        max_partitions = num_set_partitions(),
171        active_partitions = lists:seq(0, 63),
172        passive_partitions = [],
173        use_replica_index = false
174    },
175    ok = couch_set_view:define_group(
176        mapreduce_view, test_set_name(), ddoc_id(), Params).
177
178
179add_documents(StartId, Count) ->
180    etap:diag("Adding " ++ integer_to_list(Count) ++ " new documents"),
181    DocList0 = lists:map(
182        fun(I) ->
183            {I rem num_set_partitions(), {[
184            {<<"meta">>, {[{<<"id">>, doc_id(I)}]}},
185            {<<"json">>, {[
186                {<<"value">>, I}
187            ]}}
188            ]}}
189        end,
190        lists:seq(StartId, StartId + Count - 1)),
191    DocList = [Doc || {_, Doc} <- lists:keysort(1, DocList0)],
192    ok = couch_set_view_test_util:populate_set_sequentially(
193        test_set_name(),
194        lists:seq(0, num_set_partitions() - 1),
195        DocList).
196
197
198doc_id(I) ->
199    iolist_to_binary(io_lib:format("doc_~8..0b", [I])).
200
201
202get_view(_ViewName, []) ->
203    undefined;
204get_view(ViewName, [SetView | Rest]) ->
205    RedFuns = (SetView#set_view.indexer)#mapreduce_view.reduce_funs,
206    case couch_util:get_value(ViewName, RedFuns) of
207    undefined ->
208        get_view(ViewName, Rest);
209    _ ->
210        SetView
211    end.
212
213
214verify_btrees([], _ExpectedView2Reduction) ->
215    Group = get_group_snapshot(),
216    #set_view_group{
217        id_btree = IdBtree,
218        views = Views,
219        index_header = #set_view_index_header{
220            seqs = HeaderUpdateSeqs,
221            abitmask = Abitmask,
222            pbitmask = Pbitmask,
223            cbitmask = Cbitmask
224        }
225    } = Group,
226    etap:is(2, length(Views), "2 view btrees in the group"),
227    View1 = get_view(<<"view_1">>, Views),
228    View2 = get_view(<<"view_2">>, Views),
229    etap:isnt(View1, View2, "Views 1 and 2 have different btrees"),
230    #set_view{
231        indexer = #mapreduce_view{
232            btree = View1Btree
233        }
234    } = View1,
235    #set_view{
236        indexer = #mapreduce_view{
237            btree = View2Btree
238        }
239    } = View2,
240
241    etap:is(
242        couch_set_view_test_util:full_reduce_id_btree(Group, IdBtree),
243        {ok, {0, 0}},
244        "Id Btree has the right reduce value"),
245    etap:is(
246        couch_set_view_test_util:full_reduce_view_btree(Group, View1Btree),
247        {ok, {0, [0], 0}},
248        "View1 Btree has the right reduce value"),
249    etap:is(
250        couch_set_view_test_util:full_reduce_view_btree(Group, View2Btree),
251        {ok, {0, [0], 0}},
252        "View2 Btree has the right reduce value"),
253
254    etap:is(HeaderUpdateSeqs, [], "Header has right update seqs list"),
255    etap:is(Abitmask, 0, "Header has right active bitmask"),
256    etap:is(Pbitmask, 0, "Header has right passive bitmask"),
257    etap:is(Cbitmask, 0, "Header has right cleanup bitmask"),
258
259    etap:diag("Verifying the Id Btree"),
260    {ok, _, IdBtreeFoldResult} = couch_set_view_test_util:fold_id_btree(
261        Group,
262        IdBtree,
263        fun(_Kv, _, I) ->
264            {ok, I + 1}
265        end,
266        0, []),
267    etap:is(IdBtreeFoldResult, 0, "Id Btree is empty"),
268
269    etap:diag("Verifying the View1 Btree"),
270    {ok, _, View1BtreeFoldResult} = couch_set_view_test_util:fold_view_btree(
271        Group,
272        View1Btree,
273        fun(_Kv, _, I) ->
274            {ok, I + 1}
275        end,
276        0, []),
277    etap:is(View1BtreeFoldResult, 0, "View1 Btree is empty"),
278
279    etap:diag("Verifying the View2 Btree"),
280    {ok, _, View2BtreeFoldResult} = couch_set_view_test_util:fold_view_btree(
281        Group,
282        View2Btree,
283        fun(_Kv, _, I) ->
284            {ok, I + 1}
285        end,
286        0, []),
287    etap:is(View2BtreeFoldResult, 0, "View2 Btree is empty");
288
289verify_btrees(ActiveParts, ExpectedView2Reduction) ->
290    Group = get_group_snapshot(),
291    #set_view_group{
292        id_btree = IdBtree,
293        views = Views,
294        index_header = #set_view_index_header{
295            seqs = HeaderUpdateSeqs,
296            abitmask = Abitmask,
297            pbitmask = Pbitmask,
298            cbitmask = Cbitmask
299        }
300    } = Group,
301    etap:is(2, length(Views), "2 view btrees in the group"),
302    View1 = get_view(<<"view_1">>, Views),
303    View2 = get_view(<<"view_2">>, Views),
304    etap:isnt(View1, View2, "Views 1 and 2 have different btrees"),
305    #set_view{
306        indexer = #mapreduce_view{
307            btree = View1Btree
308        }
309    } = View1,
310    #set_view{
311        indexer = #mapreduce_view{
312            btree = View2Btree
313        }
314    } = View2,
315    ExpectedBitmask = couch_set_view_util:build_bitmask(ActiveParts),
316    DbSeqs = couch_set_view_test_util:get_db_seqs(test_set_name(), ActiveParts),
317    ExpectedKVCount = (num_docs() div num_set_partitions()) * length(ActiveParts),
318
319    etap:is(
320        couch_set_view_test_util:full_reduce_id_btree(Group, IdBtree),
321        {ok, {ExpectedKVCount, ExpectedBitmask}},
322        "Id Btree has the right reduce value"),
323    etap:is(
324        couch_set_view_test_util:full_reduce_view_btree(Group, View1Btree),
325        {ok, {ExpectedKVCount, [ExpectedKVCount], ExpectedBitmask}},
326        "View1 Btree has the right reduce value"),
327    etap:is(
328        couch_set_view_test_util:full_reduce_view_btree(Group, View2Btree),
329        {ok, {ExpectedKVCount, [ExpectedView2Reduction], ExpectedBitmask}},
330        "View2 Btree has the right reduce value"),
331
332    etap:is(HeaderUpdateSeqs, DbSeqs, "Header has right update seqs list"),
333    etap:is(Abitmask, ExpectedBitmask, "Header has right active bitmask"),
334    etap:is(Pbitmask, 0, "Header has right passive bitmask"),
335    etap:is(Cbitmask, 0, "Header has right cleanup bitmask"),
336
337    etap:diag("Verifying the Id Btree"),
338    MaxPerPart = num_docs() div num_set_partitions(),
339    {ok, _, {_, _, _, IdBtreeFoldResult}} = couch_set_view_test_util:fold_id_btree(
340        Group,
341        IdBtree,
342        fun(Kv, _, {Parts, I0, C0, It}) ->
343            case C0 >= MaxPerPart of
344            true ->
345                [_ | RestParts] = Parts,
346                [P | _] = RestParts,
347                I = P,
348                C = 1;
349            false ->
350                RestParts = Parts,
351                [P | _] = RestParts,
352                I = I0,
353                C = C0 + 1
354            end,
355            true = (P < num_set_partitions()),
356            DocId = doc_id(I),
357            Value = [
358                {View2#set_view.id_num, doc_id(I)},
359                {View1#set_view.id_num, doc_id(I)}
360            ],
361            ExpectedKv = {<<P:16, DocId/binary>>, {P, Value}},
362            case ExpectedKv =:= Kv of
363            true ->
364                ok;
365            false ->
366                etap:bail("Id Btree has an unexpected KV at iteration " ++ integer_to_list(It))
367            end,
368            {ok, {RestParts, I + num_set_partitions(), C, It + 1}}
369        end,
370        {ActiveParts, hd(ActiveParts), 0, 0}, []),
371    etap:is(IdBtreeFoldResult, ExpectedKVCount,
372        "Id Btree has " ++ integer_to_list(ExpectedKVCount) ++ " entries"),
373
374    etap:diag("Verifying the View1 Btree"),
375    {ok, _, {_, View1BtreeFoldResult}} = couch_set_view_test_util:fold_view_btree(
376        Group,
377        View1Btree,
378        fun(Kv, _, {NextVal, I}) ->
379            PartId = NextVal rem num_set_partitions(),
380            DocId = doc_id(NextVal),
381            ExpectedKv = {{DocId, DocId}, {PartId, NextVal}},
382            case ExpectedKv =:= Kv of
383            true ->
384                ok;
385            false ->
386                etap:bail("View1 Btree has an unexpected KV at iteration " ++ integer_to_list(I))
387            end,
388            {ok, {next_val(NextVal, ActiveParts), I + 1}}
389        end,
390        {hd(ActiveParts), 0}, []),
391    etap:is(View1BtreeFoldResult, ExpectedKVCount,
392        "View1 Btree has " ++ integer_to_list(ExpectedKVCount) ++ " entries"),
393
394    etap:diag("Verifying the View2 Btree"),
395    {ok, _, {_, View2BtreeFoldResult}} = couch_set_view_test_util:fold_view_btree(
396        Group,
397        View2Btree,
398        fun(Kv, _, {NextVal, I}) ->
399            PartId = NextVal rem num_set_partitions(),
400            DocId = doc_id(NextVal),
401            ExpectedKv = {{DocId, DocId}, {PartId, NextVal * 3}},
402            case ExpectedKv =:= Kv of
403            true ->
404                ok;
405            false ->
406                etap:bail("View2 Btree has an unexpected KV at iteration " ++ integer_to_list(I))
407            end,
408            {ok, {next_val(NextVal, ActiveParts), I + 1}}
409        end,
410        {hd(ActiveParts), 0}, []),
411    etap:is(View2BtreeFoldResult, ExpectedKVCount,
412        "View2 Btree has " ++ integer_to_list(ExpectedKVCount) ++ " entries"),
413    ok.
414
415
416next_val(I, ActiveParts) ->
417    case ordsets:is_element((I + 1) rem num_set_partitions(), ActiveParts) of
418    true ->
419        I + 1;
420    false ->
421        next_val(I + 1, ActiveParts)
422    end.
423
424