1#!/usr/bin/env escript
2%% -*- Mode: Erlang; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
3%%! -smp enable
4
5% Licensed under the Apache License, Version 2.0 (the "License"); you may not
6% use this file except in compliance with the License. You may obtain a copy of
7% the License at
8%
9%   http://www.apache.org/licenses/LICENSE-2.0
10%
11% Unless required by applicable law or agreed to in writing, software
12% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14% License for the specific language governing permissions and limitations under
15% the License.
16
17-define(JSON_ENCODE(V), ejson:encode(V)). % couch_db.hrl
18-define(MAX_WAIT_TIME, 900 * 1000).
19
20-include_lib("couch_set_view/include/couch_set_view.hrl").
21
22test_set_name() -> <<"couch_test_set_index_compactor_cleanup">>.
23num_set_partitions() -> 64.
24ddoc_id() -> <<"_design/test">>.
25num_docs_0() -> 27712.  % keep it a multiple of num_set_partitions()
26
27
28main(_) ->
29    test_util:init_code_path(),
30
31    etap:plan(21),
32    case (catch test()) of
33        ok ->
34            etap:end_tests();
35        Other ->
36            etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
37            etap:bail(Other)
38    end,
39    ok.
40
41
42test() ->
43    couch_set_view_test_util:start_server(test_set_name()),
44
45    couch_set_view_test_util:delete_set_dbs(test_set_name(), num_set_partitions()),
46    couch_set_view_test_util:create_set_dbs(test_set_name(), num_set_partitions()),
47
48    create_set(),
49    GroupPid = couch_set_view:get_group_pid(
50        mapreduce_view, test_set_name(), ddoc_id(), prod),
51    ok = gen_server:call(GroupPid, {set_auto_cleanup, false}, infinity),
52
53    ValueGenFun1 = fun(I) -> I end,
54    update_documents(0, num_docs_0(), ValueGenFun1),
55
56    % build index
57    Group0 = get_group_snapshot(),
58
59    verify_btrees(ValueGenFun1, num_docs_0(), []),
60
61    % Mark some partitions for cleanup
62    etap:diag("Marking partitions for cleanup"),
63    CleanupPartitions = lists:seq(1, num_set_partitions() - 1, 2),
64    ok = couch_set_view:set_partition_states(
65        mapreduce_view, test_set_name(), ddoc_id(), [], [], CleanupPartitions),
66
67    [StatsBefore] = ets:lookup(Group0#set_view_group.stats_ets,
68        ?set_view_group_stats_key(Group0)),
69
70    {ok, CompactPid} = couch_set_view_compactor:start_compact(
71        mapreduce_view, test_set_name(), ddoc_id()),
72    Ref = erlang:monitor(process, CompactPid),
73    etap:diag("Waiting for main group compaction to finish"),
74    receive
75    {'DOWN', Ref, process, CompactPid, normal} ->
76        ok;
77    {'DOWN', Ref, process, CompactPid, Reason} ->
78        etap:bail("Failure compacting main group: " ++ couch_util:to_list(Reason))
79    after ?MAX_WAIT_TIME ->
80        etap:bail("Timeout waiting for main group compaction to finish")
81    end,
82
83    verify_btrees(ValueGenFun1, num_docs_0(), CleanupPartitions),
84
85    [StatsAfter] = ets:lookup(Group0#set_view_group.stats_ets,
86        ?set_view_group_stats_key(Group0)),
87
88    etap:is(StatsBefore#set_view_group_stats.cleanups, 0,
89            "# of cleanups before was 0"),
90    etap:is(StatsAfter#set_view_group_stats.cleanups, 1,
91            "# of cleanups after is 1"),
92    etap:is(StatsAfter#set_view_group_stats.compactions, 1,
93            "# of compactions is 1"),
94
95    couch_set_view_test_util:delete_set_dbs(test_set_name(), num_set_partitions()),
96    couch_set_view_test_util:stop_server(),
97    ok.
98
99
100get_group_snapshot() ->
101    GroupPid = couch_set_view:get_group_pid(
102        mapreduce_view, test_set_name(), ddoc_id(), prod),
103    {ok, Group, 0} = gen_server:call(
104        GroupPid, #set_view_group_req{stale = false, debug = true}, infinity),
105    Group.
106
107
108create_set() ->
109    couch_set_view_test_util:delete_set_dbs(test_set_name(), num_set_partitions()),
110    couch_set_view_test_util:create_set_dbs(test_set_name(), num_set_partitions()),
111    couch_set_view:cleanup_index_files(mapreduce_view, test_set_name()),
112    etap:diag("Creating the set databases (# of partitions: " ++
113        integer_to_list(num_set_partitions()) ++ ")"),
114    DDoc = {[
115        {<<"meta">>, {[{<<"id">>, ddoc_id()}]}},
116        {<<"json">>, {[
117        {<<"language">>, <<"javascript">>},
118        {<<"views">>, {[
119            {<<"view_1">>, {[
120                {<<"map">>, <<"function(doc, meta) { emit(meta.id, doc.value); }">>},
121                {<<"reduce">>, <<"_count">>}
122            ]}},
123            {<<"view_2">>, {[
124                {<<"map">>, <<"function(doc, meta) { emit(meta.id, meta.id); }">>},
125                {<<"reduce">>, <<"_count">>}
126            ]}}
127        ]}}
128        ]}}
129    ]},
130    ok = couch_set_view_test_util:update_ddoc(test_set_name(), DDoc),
131    etap:diag("Configuring set view with partitions [0 .. 63] as active"),
132    Params = #set_view_params{
133        max_partitions = num_set_partitions(),
134        active_partitions = lists:seq(0, 63),
135        passive_partitions = [],
136        use_replica_index = false
137    },
138    ok = couch_set_view:define_group(
139        mapreduce_view, test_set_name(), ddoc_id(), Params).
140
141
142update_documents(StartId, Count, ValueGenFun) ->
143    etap:diag("Updating " ++ integer_to_list(Count) ++ " documents"),
144    DocList0 = lists:map(
145        fun(I) ->
146            {I rem num_set_partitions(), {[
147                {<<"meta">>, {[{<<"id">>, doc_id(I)}]}},
148                {<<"json">>, {[
149                    {<<"value">>, ValueGenFun(I)}
150                ]}}
151            ]}}
152        end,
153        lists:seq(StartId, StartId + Count - 1)),
154    DocList = [Doc || {_, Doc} <- lists:keysort(1, DocList0)],
155    ok = couch_set_view_test_util:populate_set_sequentially(
156        test_set_name(),
157        lists:seq(0, num_set_partitions() - 1),
158        DocList).
159
160
161doc_id(I) ->
162    iolist_to_binary(io_lib:format("doc_~8..0b", [I])).
163
164
165verify_btrees(ValueGenFun, NumDocs, CleanupParts) ->
166    Group = get_group_snapshot(),
167    #set_view_group{
168        id_btree = IdBtree,
169        views = [View1, View2],
170        index_header = #set_view_index_header{
171            seqs = HeaderUpdateSeqs,
172            abitmask = Abitmask,
173            pbitmask = Pbitmask,
174            cbitmask = Cbitmask
175        }
176    } = Group,
177    #set_view{
178        indexer = #mapreduce_view{
179            btree = View1Btree
180        }
181    } = View1,
182    #set_view{
183        indexer = #mapreduce_view{
184            btree = View2Btree
185        }
186    } = View2,
187    ActiveParts = ordsets:subtract(lists:seq(0, num_set_partitions() - 1), CleanupParts),
188    ExpectedBitmask = couch_set_view_util:build_bitmask(ActiveParts),
189    DbSeqs = couch_set_view_test_util:get_db_seqs(test_set_name(), ActiveParts),
190    ExpectedKVCount = length([I || I <- lists:seq(0, NumDocs - 1),
191        ordsets:is_element((I rem num_set_partitions()), ActiveParts)]),
192
193    etap:is(
194        couch_set_view_test_util:full_reduce_id_btree(Group, IdBtree),
195        {ok, {ExpectedKVCount, ExpectedBitmask}},
196        "Id Btree has the right reduce value"),
197    etap:is(
198        couch_set_view_test_util:full_reduce_view_btree(Group, View1Btree),
199        {ok, {ExpectedKVCount, [ExpectedKVCount], ExpectedBitmask}},
200        "View1 Btree has the right reduce value"),
201
202    etap:is(HeaderUpdateSeqs, DbSeqs, "Header has right update seqs list"),
203    etap:is(Abitmask, ExpectedBitmask, "Header has right active bitmask"),
204    etap:is(Pbitmask, 0, "Header has right passive bitmask"),
205    etap:is(Cbitmask, 0, "Header has right cleanup bitmask"),
206
207    etap:diag("Verifying the Id Btree"),
208    MaxPerPart = NumDocs div num_set_partitions(),
209    {ok, _, {_, _, _, IdBtreeFoldResult}} = couch_set_view_test_util:fold_id_btree(
210        Group,
211        IdBtree,
212        fun(Kv, _, {Parts, I0, C0, It}) ->
213            case C0 >= MaxPerPart of
214            true ->
215                [_ | RestParts] = Parts,
216                [P | _] = RestParts,
217                I = P,
218                C = 1;
219            false ->
220                RestParts = Parts,
221                [P | _] = RestParts,
222                I = I0,
223                C = C0 + 1
224            end,
225            true = (P < num_set_partitions()),
226            DocId = doc_id(I),
227            Value = [{View1#set_view.id_num, DocId}, {View2#set_view.id_num, DocId}],
228            ExpectedKv = {<<P:16, DocId/binary>>, {P, Value}},
229            case ExpectedKv =:= Kv of
230            true ->
231                ok;
232            false ->
233                etap:bail("Id Btree has an unexpected KV at iteration " ++ integer_to_list(It))
234            end,
235            {ok, {RestParts, I + num_set_partitions(), C, It + 1}}
236        end,
237        {ActiveParts, hd(ActiveParts), 0, 0}, []),
238    etap:is(IdBtreeFoldResult, ExpectedKVCount,
239        "Id Btree has " ++ integer_to_list(ExpectedKVCount) ++ " entries"),
240
241    etap:diag("Verifying the View1 Btree"),
242    {ok, _, {_, View1BtreeFoldResult}} = couch_set_view_test_util:fold_view_btree(
243        Group,
244        View1Btree,
245        fun(Kv, _, {I, Count}) ->
246            PartId = I rem num_set_partitions(),
247            DocId = doc_id(I),
248            ExpectedKv = {{DocId, DocId}, {PartId, ValueGenFun(I)}},
249            case ExpectedKv =:= Kv of
250            true ->
251                ok;
252            false ->
253                etap:bail(io_lib:format("View1 Btree has an unexpected KV at iteration " ++ integer_to_list(I)
254                    ++ ": ExpectedKv ~w Kv ~w", [ExpectedKv, Kv]))
255            end,
256            {ok, {next_i(I, ActiveParts), Count + 1}}
257        end,
258        {hd(ActiveParts), 0}, []),
259    etap:is(View1BtreeFoldResult, ExpectedKVCount,
260        "View1 Btree has " ++ integer_to_list(ExpectedKVCount) ++ " entries"),
261
262    etap:diag("Verifying the View2 Btree"),
263    {ok, _, {_, View2BtreeFoldResult}} = couch_set_view_test_util:fold_view_btree(
264        Group,
265        View2Btree,
266        fun(Kv, _, {I, Count}) ->
267            PartId = I rem num_set_partitions(),
268            DocId = doc_id(I),
269            ExpectedKv = {{DocId, DocId}, {PartId, DocId}},
270            case ExpectedKv =:= Kv of
271            true ->
272                ok;
273            false ->
274                etap:bail("View2 Btree has an unexpected KV at iteration " ++ integer_to_list(I))
275            end,
276            {ok, {next_i(I, ActiveParts), Count + 1}}
277        end,
278        {hd(ActiveParts), 0}, []),
279    etap:is(View2BtreeFoldResult, ExpectedKVCount,
280        "View2 Btree has " ++ integer_to_list(ExpectedKVCount) ++ " entries"),
281    ok.
282
283
284next_i(I, ActiveParts) ->
285    case ordsets:is_element((I + 1) rem num_set_partitions(), ActiveParts) of
286    true ->
287        I + 1;
288    false ->
289        next_i(I + 1, ActiveParts)
290    end.
291