1#!/usr/bin/env escript
2%% -*- Mode: Erlang; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
3%%! -smp enable
4
5% Licensed under the Apache License, Version 2.0 (the "License"); you may not
6% use this file except in compliance with the License. You may obtain a copy of
7% the License at
8%
9%   http://www.apache.org/licenses/LICENSE-2.0
10%
11% Unless required by applicable law or agreed to in writing, software
12% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14% License for the specific language governing permissions and limitations under
15% the License.
16
17-include_lib("couch_set_view/include/couch_set_view.hrl").
18
19-define(MAX_WAIT_TIME, 900 * 1000).
20
21-define(etap_match(Got, Expected, Desc),
22        etap:fun_is(fun(XXXXXX) ->
23            case XXXXXX of Expected -> true; _ -> false end
24        end, Got, Desc)).
25
26test_set_name() -> <<"couch_test_set_index_multiple_reductions">>.
27num_set_partitions() -> 64.
28ddoc_id() -> <<"_design/test">>.
29initial_num_docs() -> 70400.  % must be multiple of num_set_partitions()
30
31
32% from couch_db.hrl
33-define(MIN_STR, <<>>).
34-define(MAX_STR, <<255>>).
35
36-record(view_query_args, {
37    start_key,
38    end_key,
39    start_docid = ?MIN_STR,
40    end_docid = ?MAX_STR,
41    direction = fwd,
42    inclusive_end = true,
43    limit = 10000000000,
44    skip = 0,
45    group_level = 0,
46    view_type = nil,
47    include_docs = false,
48    conflicts = false,
49    stale = false,
50    multi_get = false,
51    callback = nil,
52    list = nil,
53    run_reduce = true,
54    keys = nil,
55    view_name = nil,
56    debug = false,
57    filter = true,
58    type = main
59}).
60
61
62main(_) ->
63    test_util:init_code_path(),
64
65    etap:plan(34),
66    case (catch test()) of
67        ok ->
68            etap:end_tests();
69        Other ->
70            etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
71            etap:bail(Other)
72    end,
73    ok.
74
75
76test() ->
77    couch_set_view_test_util:start_server(test_set_name()),
78
79    ValueGenFun1 = fun(I) -> I end,
80    create_set(),
81    update_documents(0, initial_num_docs(), ValueGenFun1),
82    trigger_update(),
83
84    QueryResult1 = query_reduce_view(<<"test2">>, doc_id(1000), doc_id(5000)),
85    etap:is(QueryResult1, <<"4001">>, "Query result is \"4001\""),
86
87    QueryResult2 = query_reduce_view(<<"test5">>, [doc_id(1000),1], [doc_id(5000),1]),
88    etap:is(QueryResult2, 4001, "Query result is 4001"),
89
90    QueryResult3 = query_reduce_view(<<"test2">>, doc_id(5000), doc_id(15000)),
91    etap:is(QueryResult3, <<"10001">>, "Query result is \"10001\""),
92
93    QueryResult4 = query_reduce_view(<<"test4">>, [doc_id(2000),1], [doc_id(3500),1]),
94    etap:is(QueryResult4, -1501, "Query result is -1501"),
95
96    verify_btrees_1(ValueGenFun1),
97    compact_view_group(),
98    verify_btrees_1(ValueGenFun1),
99
100    QueryResult11 = query_reduce_view(<<"test2">>, doc_id(1000), doc_id(5000)),
101    etap:is(QueryResult11, <<"4001">>, "Query result is \"4001\""),
102
103    QueryResult22 = query_reduce_view(<<"test5">>, [doc_id(1000),1], [doc_id(5000),1]),
104    etap:is(QueryResult22, 4001, "Query result is 4001"),
105
106    QueryResult33 = query_reduce_view(<<"test2">>, doc_id(5000), doc_id(15000)),
107    etap:is(QueryResult33, <<"10001">>, "Query result is \"10001\""),
108
109    QueryResult44 = query_reduce_view(<<"test4">>, [doc_id(2000),1], [doc_id(3500),1]),
110    etap:is(QueryResult44, -1501, "Query result is -1501"),
111
112    couch_set_view_test_util:delete_set_dbs(test_set_name(), num_set_partitions()),
113    couch_set_view_test_util:stop_server(),
114    ok.
115
116
117create_set() ->
118    couch_set_view_test_util:delete_set_dbs(test_set_name(), num_set_partitions()),
119    couch_set_view_test_util:create_set_dbs(test_set_name(), num_set_partitions()),
120    couch_set_view:cleanup_index_files(mapreduce_view, test_set_name()),
121    etap:diag("Creating the set databases (# of partitions: " ++
122        integer_to_list(num_set_partitions()) ++ ")"),
123    DDoc = {[
124        {<<"meta">>, {[{<<"id">>, ddoc_id()}]}},
125        {<<"json">>, {[
126        {<<"language">>, <<"javascript">>},
127        {<<"views">>, {[
128            {<<"test">>, {[
129                {<<"map">>, <<"function(doc, meta) { emit(meta.id, doc.value); }">>},
130                {<<"reduce">>, <<"_count">>}
131            ]}},
132            {<<"test2">>, {[
133                {<<"map">>, <<"function(doc, meta) { emit(meta.id, doc.value); }">>},
134                {<<"reduce">>, <<"function(key, values, rereduce) {"
135                                 "  if (rereduce) {"
136                                 "     var s = 0;"
137                                 "     for (var i = 0; i < values.length; ++i) {"
138                                 "         s += Number(values[i]);"
139                                 "     }"
140                                 "     return String(s);"
141                                 "  }"
142                                 "  return String(values.length);"
143                                 "}">>}
144            ]}},
145            {<<"test3">>, {[
146                {<<"map">>, <<"function(doc, meta) { emit([meta.id,1], doc.value); }">>},
147                {<<"reduce">>, <<"_sum">>}
148            ]}},
149            {<<"test4">>, {[
150                {<<"map">>, <<"function(doc, meta) { emit([meta.id,1], doc.value); }">>},
151                {<<"reduce">>, <<"function(key, values, rereduce) {"
152                                 "  if (rereduce) return sum(values);"
153                                 "  return -values.length;"
154                                 "}">>}
155            ]}},
156            {<<"test5">>, {[
157                {<<"map">>, <<"function(doc, meta) { emit([meta.id,1], doc.value); }">>},
158                {<<"reduce">>, <<"_count">>}
159            ]}}
160        ]}}
161        ]}}
162    ]},
163    ok = couch_set_view_test_util:update_ddoc(test_set_name(), DDoc),
164    etap:diag("Configuring set view with partitions [0 .. 63] as active"),
165    Params = #set_view_params{
166        max_partitions = num_set_partitions(),
167        active_partitions = lists:seq(0, 63),
168        passive_partitions = [],
169        use_replica_index = false
170    },
171    ok = couch_set_view:define_group(
172        mapreduce_view, test_set_name(), ddoc_id(), Params).
173
174
175update_documents(StartId, Count, ValueGenFun) ->
176    etap:diag("Updating " ++ integer_to_list(Count) ++ " documents"),
177    DocList0 = lists:map(
178        fun(I) ->
179            {I rem num_set_partitions(), {[
180                {<<"meta">>, {[{<<"id">>, doc_id(I)}]}},
181                {<<"json">>, {[
182                    {<<"value">>, ValueGenFun(I)}
183                ]}}
184            ]}}
185        end,
186        lists:seq(StartId, StartId + Count - 1)),
187    DocList = [Doc || {_, Doc} <- lists:keysort(1, DocList0)],
188    ok = couch_set_view_test_util:populate_set_sequentially(
189        test_set_name(),
190        lists:seq(0, num_set_partitions() - 1),
191        DocList).
192
193
194doc_id(I) ->
195    iolist_to_binary(io_lib:format("doc_~8..0b", [I])).
196
197
198compact_view_group() ->
199    {ok, CompactPid} = couch_set_view_compactor:start_compact(
200        mapreduce_view, test_set_name(), ddoc_id(), main),
201    Ref = erlang:monitor(process, CompactPid),
202    etap:diag("Waiting for view group compaction to finish"),
203    receive
204    {'DOWN', Ref, process, CompactPid, normal} ->
205        ok;
206    {'DOWN', Ref, process, CompactPid, noproc} ->
207        ok;
208    {'DOWN', Ref, process, CompactPid, Reason} ->
209        etap:bail("Failure compacting main group: " ++ couch_util:to_list(Reason))
210    after ?MAX_WAIT_TIME ->
211        etap:bail("Timeout waiting for main group compaction to finish")
212    end.
213
214
215trigger_update() ->
216    GroupPid = couch_set_view:get_group_pid(
217        mapreduce_view, test_set_name(), ddoc_id(), prod),
218    {ok, UpdaterPid} = gen_server:call(GroupPid, {start_updater, []}, infinity),
219    Ref = erlang:monitor(process, UpdaterPid),
220    etap:diag("Waiting for updater to finish"),
221    receive
222    {'DOWN', Ref, _, _, {updater_finished, _}} ->
223        etap:diag("Updater finished");
224    {'DOWN', Ref, _, _, Reason} ->
225        etap:bail("Updater finished with unexpected reason: " ++ couch_util:to_list(Reason))
226    after ?MAX_WAIT_TIME ->
227        etap:bail("Timeout waiting for updater to finish")
228    end.
229
230
231query_reduce_view(ViewName, StartKey, EndKey) ->
232    etap:diag("Querying reduce view " ++ binary_to_list(ViewName)),
233    {ok, View, Group, _} = couch_set_view:get_reduce_view(
234        test_set_name(), ddoc_id(), ViewName,
235        #set_view_group_req{stale = false, debug = true}),
236    FoldFun = fun(Key, Red, Acc) -> {ok, [{Key, Red} | Acc]} end,
237    ViewArgs = #view_query_args{
238        run_reduce = true,
239        view_name = ViewName,
240        start_key = StartKey,
241        end_key = EndKey
242    },
243    {ok, Rows} = couch_set_view:fold_reduce(Group, View, FoldFun, [], ViewArgs),
244    couch_set_view:release_group(Group),
245    case Rows of
246    [{_Key, {json, RedValue}}] ->
247        ejson:decode(RedValue);
248    [] ->
249        empty
250    end.
251
252
253verify_btrees_1(ValueGenFun) ->
254    GroupPid = couch_set_view:get_group_pid(
255        mapreduce_view, test_set_name(), ddoc_id(), prod),
256    {ok, Group} = gen_server:call(GroupPid, request_group, infinity),
257    #set_view_group{
258        id_btree = IdBtree,
259        views = Views,
260        index_header = #set_view_index_header{
261            seqs = HeaderUpdateSeqs,
262            abitmask = Abitmask,
263            pbitmask = Pbitmask,
264            cbitmask = Cbitmask
265        }
266    } = Group,
267    etap:is(length(Views), 2, "2 view record in view group"),
268    [View1, View2] = Views,
269    ?etap_match((View1#set_view.indexer)#mapreduce_view.reduce_funs,
270                [{<<"test5">>, _}, {<<"test4">>, _}, {<<"test3">>, _}],
271                "First view record has views 5 downto 3"),
272    ?etap_match((View2#set_view.indexer)#mapreduce_view.reduce_funs,
273                [{<<"test2">>, _}, {<<"test">>, _}],
274                "Second view record has views 2 downto 1"),
275    #set_view{
276        id_num = 0,
277        indexer = #mapreduce_view{
278            btree = View1Btree
279        }
280    } = View1,
281    #set_view{
282        id_num = 1,
283        indexer = #mapreduce_view{
284            btree = View2Btree
285        }
286    } = View2,
287
288    etap:diag("Verifying view group btrees"),
289    ExpectedBitmask = couch_set_view_util:build_bitmask(lists:seq(0, 63)),
290    DbSeqs = couch_set_view_test_util:get_db_seqs(test_set_name(), lists:seq(0, 63)),
291    ExpectedReductionViewBtree1 = [
292        initial_num_docs(),
293        -initial_num_docs(),
294        lists:sum([ValueGenFun(I) || I <- lists:seq(0, initial_num_docs() - 1)])
295    ],
296    ExpectedReductionViewBtree2 = [
297        list_to_binary(integer_to_list(initial_num_docs())),
298        initial_num_docs()
299    ],
300
301    etap:is(
302        couch_set_view_test_util:full_reduce_id_btree(Group, IdBtree),
303        {ok, {initial_num_docs(), ExpectedBitmask}},
304        "Id Btree has the right reduce value"),
305    etap:is(
306        couch_set_view_test_util:full_reduce_view_btree(Group, View1Btree),
307        {ok, {initial_num_docs(), ExpectedReductionViewBtree1, ExpectedBitmask}},
308        "View1 Btree has the right reduce value"),
309    etap:is(
310        couch_set_view_test_util:full_reduce_view_btree(Group, View2Btree),
311        {ok, {initial_num_docs(), ExpectedReductionViewBtree2, ExpectedBitmask}},
312        "View2 Btree has the right reduce value"),
313
314    etap:is(HeaderUpdateSeqs, DbSeqs, "Header has right update seqs list"),
315    etap:is(Abitmask, ExpectedBitmask, "Header has right active bitmask"),
316    etap:is(Pbitmask, 0, "Header has right passive bitmask"),
317    etap:is(Cbitmask, 0, "Header has right cleanup bitmask"),
318
319    etap:diag("Verifying the Id Btree"),
320    MaxPerPart = initial_num_docs() div num_set_partitions(),
321    {ok, _, {_, _, _, IdBtreeFoldResult}} = couch_set_view_test_util:fold_id_btree(
322        Group,
323        IdBtree,
324        fun(Kv, _, {P0, I0, C0, It}) ->
325            case C0 >= MaxPerPart of
326            true ->
327                P = P0 + 1,
328                I = P,
329                C = 1;
330            false ->
331                P = P0,
332                I = I0,
333                C = C0 + 1
334            end,
335            true = (P < num_set_partitions()),
336            ExpectedKv = {
337                <<P:16, (doc_id(I))/binary>>,
338                {P, [{0, [doc_id(I), 1]}, {1, doc_id(I)}]}
339            },
340            case ExpectedKv =:= Kv of
341            true ->
342                ok;
343            false ->
344                etap:bail("Id Btree has an unexpected KV at iteration " ++ integer_to_list(It))
345            end,
346            {ok, {P, I + num_set_partitions(), C, It + 1}}
347        end,
348        {0, 0, 0, 0}, []),
349    etap:is(IdBtreeFoldResult, initial_num_docs(),
350        "Id Btree has " ++ integer_to_list(initial_num_docs()) ++ " entries"),
351    etap:diag("Verifying the View1 Btree"),
352    {ok, _, View1BtreeFoldResult} = couch_set_view_test_util:fold_view_btree(
353        Group,
354        View1Btree,
355        fun(Kv, _, Acc) ->
356            PartId = Acc rem 64,
357            ExpectedKv = {{[doc_id(Acc), 1], doc_id(Acc)}, {PartId, ValueGenFun(Acc)}},
358            case ExpectedKv =:= Kv of
359            true ->
360                ok;
361            false ->
362                etap:bail("View1 Btree has an unexpected KV at iteration " ++ integer_to_list(Acc))
363            end,
364            {ok, Acc + 1}
365        end,
366        0, []),
367    etap:is(View1BtreeFoldResult, initial_num_docs(),
368        "View1 Btree has " ++ integer_to_list(initial_num_docs()) ++ " entries"),
369    etap:diag("Verifying the View2 Btree"),
370    {ok, _, View2BtreeFoldResult} = couch_set_view_test_util:fold_view_btree(
371        Group,
372        View2Btree,
373        fun(Kv, _, Acc) ->
374            PartId = Acc rem 64,
375            ExpectedKv = {{doc_id(Acc), doc_id(Acc)}, {PartId, ValueGenFun(Acc)}},
376            case ExpectedKv =:= Kv of
377            true ->
378                ok;
379            false ->
380                etap:bail("View2 Btree has an unexpected KV at iteration " ++ integer_to_list(Acc))
381            end,
382            {ok, Acc + 1}
383        end,
384        0, []),
385    etap:is(View2BtreeFoldResult, initial_num_docs(),
386        "View2 Btree has " ++ integer_to_list(initial_num_docs()) ++ " entries").
387