1#!/usr/bin/env escript
2%% -*- Mode: Erlang; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
3%%! -smp enable
4
5% Licensed under the Apache License, Version 2.0 (the "License"); you may not
6% use this file except in compliance with the License. You may obtain a copy of
7% the License at
8%
9%   http://www.apache.org/licenses/LICENSE-2.0
10%
11% Unless required by applicable law or agreed to in writing, software
12% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14% License for the specific language governing permissions and limitations under
15% the License.
16
17-include_lib("couch_set_view/include/couch_set_view.hrl").
18
19% Test motivated by MB-4518.
20
21-define(JSON_ENCODE(V), ejson:encode(V)). % couch_db.hrl
22-define(MAX_WAIT_TIME, 900 * 1000).
23
24% from couch_db.hrl
25-define(MIN_STR, <<>>).
26-define(MAX_STR, <<255>>).
27
28-record(view_query_args, {
29    start_key,
30    end_key,
31    start_docid = ?MIN_STR,
32    end_docid = ?MAX_STR,
33    direction = fwd,
34    inclusive_end = true,
35    limit = 10000000000,
36    skip = 0,
37    group_level = 0,
38    view_type = nil,
39    include_docs = false,
40    conflicts = false,
41    stale = false,
42    multi_get = false,
43    callback = nil,
44    list = nil,
45    run_reduce = true,
46    keys = nil,
47    view_name = nil,
48    debug = false,
49    filter = true,
50    type = main
51}).
52
53
54test_set_name() -> <<"couch_test_set_index_deletes_cleanup">>.
55num_set_partitions() -> 64.
56ddoc_id() -> <<"_design/test">>.
57initial_num_docs() -> 104192.  % must be multiple of num_set_partitions()
58
59
60main(_) ->
61    test_util:init_code_path(),
62
63    etap:plan(74),
64    case (catch test()) of
65        ok ->
66            etap:end_tests();
67        Other ->
68            etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
69            etap:bail(Other)
70    end,
71    ok.
72
73
74test() ->
75    couch_set_view_test_util:start_server(test_set_name()),
76
77    create_set(),
78    add_documents(0, initial_num_docs()),
79
80    {QueryResult1, Group1} = query_reduce_view(false),
81    etap:is(
82        QueryResult1,
83        initial_num_docs(),
84        "Reduce view has value " ++ couch_util:to_list(initial_num_docs())),
85    verify_btrees_1(Group1),
86
87    compact_view_group(),
88
89    {QueryResult2, Group2} = query_reduce_view(false),
90    etap:is(
91        QueryResult2,
92        initial_num_docs(),
93        "Reduce view has value " ++ couch_util:to_list(initial_num_docs())),
94    verify_btrees_1(Group2),
95
96    etap:diag("Deleting all documents"),
97    delete_docs(0, initial_num_docs()),
98    etap:is(
99        couch_set_view_test_util:doc_count(test_set_name(), lists:seq(0, 63)),
100        0,
101        "All docs were deleted"),
102
103    etap:diag("Marking partitions [ 32 .. 63 ] for cleanup"),
104    ok = lists:foreach(
105        fun(I) ->
106            ok = couch_set_view:set_partition_states(
107                mapreduce_view, test_set_name(), ddoc_id(), [], [], [I])
108        end,
109        lists:seq(32, 63)),
110
111    etap:diag("Waiting for cleanup of partitions [ 32 .. 63 ]"),
112    MainGroupInfo = get_group_info(),
113    wait_for_cleanup(MainGroupInfo),
114    etap:diag("Cleanup finished"),
115
116    {QueryResult3, Group3} = query_reduce_view(false),
117    etap:is(
118        QueryResult3,
119        empty,
120        "Reduce view returned 0 rows"),
121    verify_btrees_2(Group3),
122
123    compact_view_group(),
124
125    {QueryResult4, Group4} = query_reduce_view(false),
126    etap:is(
127        QueryResult4,
128        empty,
129        "Reduce view returned 0 rows"),
130    verify_btrees_2(Group4),
131
132    etap:diag("Marking partitions [ 32 .. 63 ] as active"),
133    ok = lists:foreach(
134        fun(I) ->
135            ok = couch_set_view:set_partition_states(
136                mapreduce_view, test_set_name(), ddoc_id(), [I], [], [])
137        end,
138        lists:seq(32, 63)),
139
140    {QueryResult5, Group5} = query_reduce_view(false),
141    etap:is(
142        QueryResult5,
143        empty,
144        "Reduce view returned 0 rows"),
145    verify_btrees_3(Group5),
146
147    compact_view_group(),
148
149    {QueryResult6, Group6} = query_reduce_view(false),
150    etap:is(
151        QueryResult6,
152        empty,
153        "Reduce view returned 0 rows"),
154    verify_btrees_3(Group6),
155
156    etap:diag("Creating the same documents again"),
157    add_documents(0, initial_num_docs()),
158
159    {QueryResult7, Group7} = query_reduce_view(false),
160    etap:is(
161        QueryResult7,
162        initial_num_docs(),
163        "Reduce view has value " ++ couch_util:to_list(initial_num_docs())),
164    verify_btrees_1(Group7),
165
166    compact_view_group(),
167
168    {QueryResult8, Group8} = query_reduce_view(false),
169    etap:is(
170        QueryResult8,
171        initial_num_docs(),
172        "Reduce view has value " ++ couch_util:to_list(initial_num_docs())),
173    verify_btrees_1(Group8),
174
175    couch_set_view_test_util:delete_set_dbs(test_set_name(), num_set_partitions()),
176    couch_set_view_test_util:stop_server(),
177    ok.
178
179
180query_reduce_view(Stale) ->
181    etap:diag("Querying reduce view with ?group=true"),
182    {ok, View, Group, _} = couch_set_view:get_reduce_view(
183        test_set_name(), ddoc_id(), <<"test">>,
184        #set_view_group_req{stale = Stale, debug = true}),
185    FoldFun = fun(Key, Red, Acc) -> {ok, [{Key, Red} | Acc]} end,
186    ViewArgs = #view_query_args{
187        run_reduce = true,
188        view_name = <<"test">>
189    },
190    {ok, Rows} = couch_set_view:fold_reduce(Group, View, FoldFun, [], ViewArgs),
191    couch_set_view:release_group(Group),
192    case Rows of
193    [{_Key, {json, RedValue}}] ->
194        {ejson:decode(RedValue), Group};
195    [] ->
196        {empty, Group}
197    end.
198
199
200wait_for_cleanup(GroupInfo) ->
201    etap:diag("Waiting for main index cleanup to finish"),
202    Pid = spawn(fun() ->
203        wait_for_cleanup_loop(GroupInfo)
204    end),
205    Ref = erlang:monitor(process, Pid),
206    receive
207    {'DOWN', Ref, process, Pid, normal} ->
208        ok;
209    {'DOWN', Ref, process, Pid, Reason} ->
210        etap:bail("Failure waiting for main index cleanup: " ++ couch_util:to_list(Reason))
211    after ?MAX_WAIT_TIME ->
212        etap:bail("Timeout waiting for main index cleanup")
213    end.
214
215
216wait_for_cleanup_loop(GroupInfo) ->
217    case couch_util:get_value(cleanup_partitions, GroupInfo) of
218    [] ->
219        {Stats} = couch_util:get_value(stats, GroupInfo),
220        Cleanups = couch_util:get_value(cleanups, Stats),
221        etap:is(
222            (is_integer(Cleanups) andalso (Cleanups > 0)),
223            true,
224            "Main group stats has at least 1 full cleanup");
225    _ ->
226        ok = timer:sleep(1000),
227        wait_for_cleanup_loop(get_group_info())
228    end.
229
230
231get_group_info() ->
232    {ok, Info} = couch_set_view:get_group_info(
233        mapreduce_view, test_set_name(), ddoc_id(), prod),
234    Info.
235
236
237delete_docs(StartId, NumDocs) ->
238    Dbs = dict:from_list(lists:map(
239        fun(I) ->
240            {ok, Db} = couch_set_view_test_util:open_set_db(test_set_name(), I),
241            {I, Db}
242        end,
243        lists:seq(0, 63))),
244    Docs = lists:foldl(
245        fun(I, Acc) ->
246            Doc = couch_doc:from_json_obj({[
247                {<<"meta">>, {[{<<"deleted">>, true}, {<<"id">>, doc_id(I)}]}},
248                {<<"json">>, {[]}}
249            ]}),
250            DocList = case orddict:find(I rem 64, Acc) of
251            {ok, L} ->
252                L;
253            error ->
254                []
255            end,
256            orddict:store(I rem 64, [Doc | DocList], Acc)
257        end,
258        orddict:new(), lists:seq(StartId, StartId + NumDocs - 1)),
259    [] = orddict:fold(
260        fun(I, DocList, Acc) ->
261            Db = dict:fetch(I, Dbs),
262            etap:diag("Deleting " ++ integer_to_list(length(DocList)) ++
263                " documents from partition " ++ integer_to_list(I)),
264            ok = couch_db:update_docs(Db, DocList, [sort_docs]),
265            Acc
266        end,
267        [], Docs),
268    ok = lists:foreach(fun({_, Db}) -> ok = couch_db:close(Db) end, dict:to_list(Dbs)).
269
270
271create_set() ->
272    couch_set_view_test_util:delete_set_dbs(test_set_name(), num_set_partitions()),
273    couch_set_view_test_util:create_set_dbs(test_set_name(), num_set_partitions()),
274    couch_set_view:cleanup_index_files(mapreduce_view, test_set_name()),
275    etap:diag("Creating the set databases (# of partitions: " ++
276        integer_to_list(num_set_partitions()) ++ ")"),
277    DDoc = {[
278        {<<"meta">>, {[{<<"id">>, ddoc_id()}]}},
279        {<<"json">>, {[
280        {<<"language">>, <<"javascript">>},
281        {<<"views">>, {[
282            {<<"test">>, {[
283                {<<"map">>, <<"function(doc, meta) { emit(meta.id, doc.value); }">>},
284                {<<"reduce">>, <<"_count">>}
285            ]}}
286        ]}}
287        ]}}
288    ]},
289    ok = couch_set_view_test_util:update_ddoc(test_set_name(), DDoc),
290    etap:diag("Configuring set view with partitions [0 .. 63] as active"),
291    Params = #set_view_params{
292        max_partitions = num_set_partitions(),
293        active_partitions = lists:seq(0, 63),
294        passive_partitions = [],
295        use_replica_index = false
296    },
297    ok = couch_set_view:define_group(
298        mapreduce_view, test_set_name(), ddoc_id(), Params).
299
300
301add_documents(StartId, Count) ->
302    etap:diag("Adding " ++ integer_to_list(Count) ++ " new documents"),
303    DocList0 = lists:map(
304        fun(I) ->
305            {I rem num_set_partitions(), {[
306                {<<"meta">>, {[{<<"id">>, doc_id(I)}]}},
307                {<<"json">>, {[
308                    {<<"value">>, I}
309                ]}}
310            ]}}
311        end,
312        lists:seq(StartId, StartId + Count - 1)),
313    DocList = [Doc || {_, Doc} <- lists:keysort(1, DocList0)],
314    ok = couch_set_view_test_util:populate_set_sequentially(
315        test_set_name(),
316        lists:seq(0, num_set_partitions() - 1),
317        DocList).
318
319
320doc_id(I) ->
321    iolist_to_binary(io_lib:format("doc_~8..0b", [I])).
322
323
324compact_view_group() ->
325    {ok, CompactPid} = couch_set_view_compactor:start_compact(
326        mapreduce_view, test_set_name(), ddoc_id(), main),
327    Ref = erlang:monitor(process, CompactPid),
328    etap:diag("Waiting for view group compaction to finish"),
329    receive
330    {'DOWN', Ref, process, CompactPid, normal} ->
331        ok;
332    {'DOWN', Ref, process, CompactPid, noproc} ->
333        ok;
334    {'DOWN', Ref, process, CompactPid, Reason} ->
335        etap:bail("Failure compacting main group: " ++ couch_util:to_list(Reason))
336    after ?MAX_WAIT_TIME ->
337        etap:bail("Timeout waiting for main group compaction to finish")
338    end.
339
340
341verify_btrees_1(Group) ->
342    #set_view_group{
343        id_btree = IdBtree,
344        views = [View0],
345        index_header = #set_view_index_header{
346            seqs = HeaderUpdateSeqs,
347            abitmask = Abitmask,
348            pbitmask = Pbitmask,
349            cbitmask = Cbitmask
350        }
351    } = Group,
352    #set_view{
353        id_num = 0,
354        indexer = #mapreduce_view{
355            btree = View0Btree
356        }
357    } = View0,
358    etap:diag("Verifying view group btrees"),
359    ExpectedBitmask = couch_set_view_util:build_bitmask(lists:seq(0, 63)),
360    DbSeqs = couch_set_view_test_util:get_db_seqs(test_set_name(), lists:seq(0, 63)),
361
362    etap:is(
363        couch_set_view_test_util:full_reduce_id_btree(Group, IdBtree),
364        {ok, {initial_num_docs(), ExpectedBitmask}},
365        "Id Btree has the right reduce value"),
366    etap:is(
367        couch_set_view_test_util:full_reduce_view_btree(Group, View0Btree),
368        {ok, {initial_num_docs(), [initial_num_docs()], ExpectedBitmask}},
369        "View0 Btree has the right reduce value"),
370
371    etap:is(HeaderUpdateSeqs, DbSeqs, "Header has right update seqs list"),
372    etap:is(Abitmask, ExpectedBitmask, "Header has right active bitmask"),
373    etap:is(Pbitmask, 0, "Header has right passive bitmask"),
374    etap:is(Cbitmask, 0, "Header has right cleanup bitmask"),
375
376    etap:diag("Verifying the Id Btree"),
377    MaxPerPart = initial_num_docs() div num_set_partitions(),
378    {ok, _, {_, _, _, IdBtreeFoldResult}} = couch_btree:fold(
379        IdBtree,
380        fun(Kv, _, {P0, I0, C0, It}) ->
381            case C0 >= MaxPerPart of
382            true ->
383                P = P0 + 1,
384                I = P,
385                C = 1;
386            false ->
387                P = P0,
388                I = I0,
389                C = C0 + 1
390            end,
391            true = (P < num_set_partitions()),
392            V = ?JSON_ENCODE(doc_id(I)),
393            [ExpectedKv] = mapreduce_view:convert_back_index_kvs_to_binary(
394                [{doc_id(I), {P, [{0, [V]}]}}], []),
395            case ExpectedKv =:= Kv of
396            true ->
397                ok;
398            false ->
399                etap:bail("Id Btree has an unexpected KV at iteration " ++ integer_to_list(It))
400            end,
401            {ok, {P, I + num_set_partitions(), C, It + 1}}
402        end,
403        {0, 0, 0, 0}, []),
404    etap:is(IdBtreeFoldResult, initial_num_docs(),
405        "Id Btree has " ++ integer_to_list(initial_num_docs()) ++ " entries"),
406    etap:diag("Verifying the View0 Btree"),
407    {ok, _, View0BtreeFoldResult} = couch_btree:fold(
408        View0Btree,
409        fun(Kv, _, Acc) ->
410            V = ?JSON_ENCODE(Acc),
411            ExpectedKeyDocId = mapreduce_view:encode_key_docid(
412                ?JSON_ENCODE(doc_id(Acc)), doc_id(Acc)),
413            ExpectedKv = {ExpectedKeyDocId, <<(Acc rem 64):16, (size(V)):24, V/binary>>},
414            case ExpectedKv =:= Kv of
415            true ->
416                ok;
417            false ->
418                etap:bail("View0 Btree has an unexpected KV at iteration " ++ integer_to_list(Acc))
419            end,
420            {ok, Acc + 1}
421        end,
422        0, []),
423    etap:is(View0BtreeFoldResult, initial_num_docs(),
424        "View0 Btree has " ++ integer_to_list(initial_num_docs()) ++ " entries").
425
426
427verify_btrees_2(Group) ->
428    #set_view_group{
429        id_btree = IdBtree,
430        views = [View0],
431        index_header = #set_view_index_header{
432            seqs = HeaderUpdateSeqs,
433            abitmask = Abitmask,
434            pbitmask = Pbitmask,
435            cbitmask = Cbitmask
436        }
437    } = Group,
438    #set_view{
439        id_num = 0,
440        indexer = #mapreduce_view{
441            btree = View0Btree
442        }
443    } = View0,
444    etap:diag("Verifying view group btrees"),
445    ExpectedABitmask = couch_set_view_util:build_bitmask(lists:seq(0, 31)),
446    DbSeqs = couch_set_view_test_util:get_db_seqs(test_set_name(), lists:seq(0, 31)),
447
448    etap:is(
449        couch_set_view_test_util:full_reduce_id_btree(Group, IdBtree),
450        {ok, {0, 0}},
451        "Id Btree has the right reduce value"),
452    etap:is(
453        couch_set_view_test_util:full_reduce_view_btree(Group, View0Btree),
454        {ok, {0, [0], 0}},
455        "View0 Btree has the right reduce value"),
456
457    etap:is(HeaderUpdateSeqs, DbSeqs, "Header has right update seqs list"),
458    etap:is(Abitmask, ExpectedABitmask, "Header has right active bitmask"),
459    etap:is(Pbitmask, 0, "Header has right passive bitmask"),
460    etap:is(Cbitmask, 0, "Header has right cleanup bitmask"),
461
462    etap:diag("Verifying the Id Btree"),
463    {ok, _, IdBtreeFoldResult} = couch_set_view_test_util:fold_id_btree(
464        Group,
465        IdBtree,
466        fun(_Kv, _, Acc) ->
467            {ok, Acc + 1}
468        end,
469        0, []),
470    etap:is(IdBtreeFoldResult, 0, "Id Btree is empty"),
471    etap:diag("Verifying the View0 Btree"),
472    {ok, _, View0BtreeFoldResult} = couch_set_view_test_util:fold_view_btree(
473        Group,
474        View0Btree,
475        fun(_Kv, _, Acc) ->
476            {ok, Acc + 1}
477        end,
478        0, []),
479    etap:is(View0BtreeFoldResult, 0, "View0 Btree is empty").
480
481
482verify_btrees_3(Group) ->
483    #set_view_group{
484        id_btree = IdBtree,
485        views = [View0],
486        index_header = #set_view_index_header{
487            seqs = HeaderUpdateSeqs,
488            abitmask = Abitmask,
489            pbitmask = Pbitmask,
490            cbitmask = Cbitmask
491        }
492    } = Group,
493    #set_view{
494        id_num = 0,
495        indexer = #mapreduce_view{
496            btree = View0Btree
497        }
498    } = View0,
499    etap:diag("Verifying view group btrees"),
500    ExpectedABitmask = couch_set_view_util:build_bitmask(lists:seq(0, 63)),
501    DbSeqs = couch_set_view_test_util:get_db_seqs(test_set_name(), lists:seq(0, 63)),
502
503    etap:is(
504        couch_set_view_test_util:full_reduce_id_btree(Group, IdBtree),
505        {ok, {0, 0}},
506        "Id Btree has the right reduce value"),
507    etap:is(
508        couch_set_view_test_util:full_reduce_view_btree(Group, View0Btree),
509        {ok, {0, [0], 0}},
510        "View0 Btree has the right reduce value"),
511
512    etap:is(HeaderUpdateSeqs, DbSeqs, "Header has right update seqs list"),
513    etap:is(Abitmask, ExpectedABitmask, "Header has right active bitmask"),
514    etap:is(Pbitmask, 0, "Header has right passive bitmask"),
515    etap:is(Cbitmask, 0, "Header has right cleanup bitmask"),
516
517    etap:diag("Verifying the Id Btree"),
518    {ok, _, IdBtreeFoldResult} = couch_set_view_test_util:fold_id_btree(
519        Group,
520        IdBtree,
521        fun(_Kv, _, Acc) ->
522            {ok, Acc + 1}
523        end,
524        0, []),
525    etap:is(IdBtreeFoldResult, 0, "Id Btree is empty"),
526    etap:diag("Verifying the View0 Btree"),
527    {ok, _, View0BtreeFoldResult} = couch_set_view_test_util:fold_view_btree(
528        Group,
529        View0Btree,
530        fun(_Kv, _, Acc) ->
531            {ok, Acc + 1}
532        end,
533        0, []),
534    etap:is(View0BtreeFoldResult, 0, "View0 Btree is empty").
535