1#!/usr/bin/env escript
2%% -*- Mode: Erlang; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
3%%! -smp enable
4
5% Licensed under the Apache License, Version 2.0 (the "License"); you may not
6% use this file except in compliance with the License. You may obtain a copy of
7% the License at
8%
9%   http://www.apache.org/licenses/LICENSE-2.0
10%
11% Unless required by applicable law or agreed to in writing, software
12% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14% License for the specific language governing permissions and limitations under
15% the License.
16
17-include_lib("couch_set_view/include/couch_set_view.hrl").
18
19-define(JSON_ENCODE(V), ejson:encode(V)). % couch_db.hrl
20-define(MAX_WAIT_TIME, 600 * 1000).
21
22test_set_name() -> <<"couch_test_set_pending_transition">>.
23num_set_partitions() -> 64.
24ddoc_id() -> <<"_design/test">>.
25num_docs() -> 20288.  % keep it a multiple of num_set_partitions()
26
27-record(user_ctx, {
28    name = null,
29    roles = [],
30    handler
31}).
32
33admin_user_ctx() ->
34    {user_ctx, #user_ctx{roles = [<<"_admin">>]}}.
35
36
37main(_) ->
38    test_util:init_code_path(),
39
40    etap:plan(73),
41    case (catch test()) of
42        ok ->
43            etap:end_tests();
44        Other ->
45            etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
46            etap:bail(Other)
47    end,
48    ok.
49
50
51test() ->
52    couch_set_view_test_util:start_server(test_set_name()),
53
54    create_set(),
55    ValueGenFun1 = fun(I) -> I end,
56    update_documents(0, num_docs(), ValueGenFun1),
57
58    GroupPid = couch_set_view:get_group_pid(
59        mapreduce_view, test_set_name(), ddoc_id(), prod),
60    ok = gen_server:call(GroupPid, {set_auto_cleanup, false}, infinity),
61
62    % build index
63    _ = get_group_snapshot(),
64
65    verify_btrees_1(ValueGenFun1),
66
67    etap:diag("Marking all odd partitions for cleanup"),
68    ok = couch_set_view:set_partition_states(
69        mapreduce_view, test_set_name(), ddoc_id(), [], [],
70        lists:seq(1, num_set_partitions() - 1, 2)),
71
72    verify_btrees_2(ValueGenFun1),
73
74    etap:diag("Marking partition 1 as active and all even partitions, "
75              "except partition 0, for cleanup"),
76    ok = couch_set_view:set_partition_states(
77        mapreduce_view, test_set_name(), ddoc_id(),
78        [1], [], lists:seq(2, num_set_partitions() - 1, 2)),
79
80    verify_btrees_3(ValueGenFun1),
81
82    test_unindexable_partitions(),
83
84    test_pending_transition_changes(),
85
86    lists:foreach(fun(PartId) ->
87        etap:diag("Deleting partition " ++ integer_to_list(PartId) ++
88            ", currently marked for cleanup in the pending transition"),
89        ok = couch_set_view_test_util:delete_set_db(test_set_name(), PartId)
90    end, lists:seq(2, num_set_partitions() - 1, 2)),
91    ok = timer:sleep(5000),
92    etap:is(is_process_alive(GroupPid), true, "Group process didn't die"),
93
94    % Recreate database 1, populate new contents, verify that neither old
95    % contents nor new contents are in the index after a stale=false request.
96    etap:diag("Recreating partition 1 database, currenly marked as active in the"
97              " pending transition - shouldn't cause the group process to die"),
98    ok = couch_set_view:set_partition_states(
99           mapreduce_view, test_set_name(), ddoc_id(), [], [], [1]),
100    recreate_db(1, 9000009),
101    ok = timer:sleep(6000),
102    etap:is(is_process_alive(GroupPid), true, "Group process didn't die"),
103
104    {ok, Db0} = open_db(0),
105    Doc = couch_doc:from_json_obj({[
106        {<<"meta">>, {[{<<"id">>, doc_id(9000010)}]}},
107        {<<"json">>, {[
108            {<<"value">>, 9000010}
109        ]}}
110    ]}),
111    ok = couch_db:update_doc(Db0, Doc, []),
112    ok = couch_db:close(Db0),
113
114    % update index - updater will trigger a cleanup and apply the pending transition
115    _ = get_group_snapshot(),
116
117    verify_btrees_4(ValueGenFun1),
118    compact_view_group(),
119    verify_btrees_4(ValueGenFun1),
120
121    test_monitor_pending_partition(),
122
123    couch_set_view_test_util:delete_set_dbs(test_set_name(), num_set_partitions() - 1),
124    couch_set_view_test_util:stop_server(),
125    ok.
126
127
128recreate_db(PartId, Value) ->
129    DbName = iolist_to_binary([test_set_name(), $/, integer_to_list(PartId)]),
130    ok = couch_server:delete(DbName, [admin_user_ctx()]),
131    ok = timer:sleep(300),
132    {ok, Db} = couch_db:create(DbName, [admin_user_ctx()]),
133    Doc = couch_doc:from_json_obj({[
134        {<<"meta">>, {[{<<"id">>, doc_id(Value)}]}},
135        {<<"json">>, {[
136            {<<"value">>, Value}
137        ]}}
138    ]}),
139    ok = couch_db:update_doc(Db, Doc, []),
140    ok = couch_db:close(Db).
141
142
143open_db(PartId) ->
144    DbName = iolist_to_binary([test_set_name(), $/, integer_to_list(PartId)]),
145    {ok, _} = couch_db:open_int(DbName, []).
146
147
148create_set() ->
149    couch_set_view_test_util:delete_set_dbs(test_set_name(), num_set_partitions()),
150    couch_set_view_test_util:create_set_dbs(test_set_name(), num_set_partitions()),
151    couch_set_view:cleanup_index_files(mapreduce_view, test_set_name()),
152    etap:diag("Creating the set databases (# of partitions: " ++
153        integer_to_list(num_set_partitions()) ++ ")"),
154    DDoc = {[
155        {<<"meta">>, {[{<<"id">>, ddoc_id()}]}},
156        {<<"json">>, {[
157        {<<"language">>, <<"javascript">>},
158        {<<"views">>, {[
159            {<<"view_1">>, {[
160                {<<"map">>, <<"function(doc, meta) { emit(meta.id, doc.value); }">>},
161                {<<"reduce">>, <<"_count">>}
162            ]}}
163        ]}}
164        ]}}
165    ]},
166    ok = couch_set_view_test_util:update_ddoc(test_set_name(), DDoc),
167    etap:diag("Configuring set view with partitions [0 .. 63] as active"),
168    Params = #set_view_params{
169        max_partitions = num_set_partitions(),
170        active_partitions = lists:seq(0, 63),
171        passive_partitions = [],
172        use_replica_index = false
173    },
174    ok = couch_set_view:define_group(
175        mapreduce_view, test_set_name(), ddoc_id(), Params).
176
177
178update_documents(StartId, NumDocs, ValueGenFun) ->
179    etap:diag("About to update " ++ integer_to_list(NumDocs) ++ " documents"),
180    Dbs = dict:from_list(lists:map(
181        fun(I) ->
182            {ok, Db} = couch_set_view_test_util:open_set_db(test_set_name(), I),
183            {I, Db}
184        end,
185        lists:seq(0, num_set_partitions() - 1))),
186    Docs = lists:foldl(
187        fun(I, Acc) ->
188            Doc = couch_doc:from_json_obj({[
189                {<<"meta">>, {[{<<"id">>, doc_id(I)}]}},
190                {<<"json">>, {[
191                    {<<"value">>, ValueGenFun(I)}
192                ]}}
193            ]}),
194            DocList = case orddict:find(I rem num_set_partitions(), Acc) of
195            {ok, L} ->
196                L;
197            error ->
198                []
199            end,
200            orddict:store(I rem num_set_partitions(), [Doc | DocList], Acc)
201        end,
202        orddict:new(), lists:seq(StartId, StartId + NumDocs - 1)),
203    [] = orddict:fold(
204        fun(I, DocList, Acc) ->
205            Db = dict:fetch(I, Dbs),
206            ok = couch_db:update_docs(Db, DocList, [sort_docs]),
207            Acc
208        end,
209        [], Docs),
210    etap:diag("Updated " ++ integer_to_list(NumDocs) ++ " documents"),
211    ok = lists:foreach(fun({_, Db}) -> ok = couch_db:close(Db) end, dict:to_list(Dbs)).
212
213
214doc_id(I) ->
215    iolist_to_binary(io_lib:format("doc_~8..0b", [I])).
216
217
218verify_btrees_1(ValueGenFun) ->
219    Group = get_group_snapshot(),
220    etap:diag("Verifying btrees"),
221    #set_view_group{
222        id_btree = IdBtree,
223        views = [View1],
224        index_header = #set_view_index_header{
225            pending_transition = PendingTrans,
226            seqs = HeaderUpdateSeqs,
227            abitmask = Abitmask,
228            pbitmask = Pbitmask,
229            cbitmask = Cbitmask
230        }
231    } = Group,
232    #set_view{
233        indexer = #mapreduce_view{
234            btree = View1Btree
235        }
236    } = View1,
237    ActiveParts = lists:seq(0, num_set_partitions() - 1),
238    ExpectedBitmask = couch_set_view_util:build_bitmask(ActiveParts),
239    ExpectedABitmask = couch_set_view_util:build_bitmask(ActiveParts),
240    DbSeqs = couch_set_view_test_util:get_db_seqs(
241        test_set_name(), lists:seq(0, num_set_partitions() - 1)),
242    ExpectedKVCount = num_docs(),
243    ExpectedBtreeViewReduction = num_docs(),
244
245    etap:is(
246        couch_set_view_test_util:full_reduce_id_btree(Group, IdBtree),
247        {ok, {ExpectedKVCount, ExpectedBitmask}},
248        "Id Btree has the right reduce value"),
249    etap:is(
250        couch_set_view_test_util:full_reduce_view_btree(Group, View1Btree),
251        {ok, {ExpectedKVCount, [ExpectedBtreeViewReduction], ExpectedBitmask}},
252        "View1 Btree has the right reduce value"),
253
254    etap:is(HeaderUpdateSeqs, DbSeqs, "Header has right update seqs list"),
255    etap:is(Abitmask, ExpectedABitmask, "Header has right active bitmask"),
256    etap:is(Pbitmask, 0, "Header has right passive bitmask"),
257    etap:is(Cbitmask, 0, "Header has right cleanup bitmask"),
258    etap:is(PendingTrans, nil, "Header has nil pending transition"),
259
260    etap:diag("Verifying the Id Btree"),
261    MaxPerPart = num_docs() div num_set_partitions(),
262    {ok, _, {_, _, _, IdBtreeFoldResult}} = couch_set_view_test_util:fold_id_btree(
263        Group,
264        IdBtree,
265        fun(Kv, _, {P0, I0, C0, It}) ->
266            case C0 >= MaxPerPart of
267            true ->
268                P = P0 + 1,
269                I = P,
270                C = 1;
271            false ->
272                P = P0,
273                I = I0,
274                C = C0 + 1
275            end,
276            true = (P < num_set_partitions()),
277            DocId = doc_id(I),
278            Value = [{View1#set_view.id_num, DocId}],
279            ExpectedKv = {<<P:16, DocId/binary>>, {P, Value}},
280            case ExpectedKv =:= Kv of
281            true ->
282                ok;
283            false ->
284                etap:bail("Id Btree has an unexpected KV at iteration " ++ integer_to_list(It))
285            end,
286            {ok, {P, I + num_set_partitions(), C, It + 1}}
287        end,
288        {0, 0, 0, 0}, []),
289    etap:is(IdBtreeFoldResult, ExpectedKVCount,
290        "Id Btree has " ++ integer_to_list(ExpectedKVCount) ++ " entries"),
291
292    etap:diag("Verifying the View1 Btree"),
293    {ok, _, View1BtreeFoldResult} = couch_set_view_test_util:fold_view_btree(
294        Group,
295        View1Btree,
296        fun(Kv, _, I) ->
297            PartId = I rem num_set_partitions(),
298            DocId = doc_id(I),
299            ExpectedKv = {{DocId, DocId}, {PartId, ValueGenFun(I)}},
300            case ExpectedKv =:= Kv of
301            true ->
302                ok;
303            false ->
304                etap:bail("View1 Btree has an unexpected KV at iteration " ++ integer_to_list(I))
305            end,
306            {ok, I + 1}
307        end,
308        0, []),
309    etap:is(View1BtreeFoldResult, ExpectedKVCount,
310        "View1 Btree has " ++ integer_to_list(ExpectedKVCount) ++ " entries"),
311    ok.
312
313
314verify_btrees_2(ValueGenFun) ->
315    Group = get_group_snapshot(),
316    etap:diag("Verifying btrees"),
317    #set_view_group{
318        id_btree = IdBtree,
319        views = [View1],
320        index_header = #set_view_index_header{
321            pending_transition = PendingTrans,
322            seqs = HeaderUpdateSeqs,
323            abitmask = Abitmask,
324            pbitmask = Pbitmask,
325            cbitmask = Cbitmask
326        }
327    } = Group,
328    #set_view{
329        indexer = #mapreduce_view{
330            btree = View1Btree
331        }
332    } = View1,
333    ActiveParts = lists:seq(0, num_set_partitions() - 1, 2),
334    CleanupParts = lists:seq(1, num_set_partitions() - 1, 2),
335    ExpectedBitmask = couch_set_view_util:build_bitmask(lists:seq(0, num_set_partitions() - 1)),
336    ExpectedABitmask = couch_set_view_util:build_bitmask(ActiveParts),
337    ExpectedCBitmask = couch_set_view_util:build_bitmask(CleanupParts),
338    ExpectedDbSeqs = couch_set_view_test_util:get_db_seqs(test_set_name(), ActiveParts),
339    ExpectedKVCount = num_docs(),
340    ExpectedBtreeViewReduction = num_docs(),
341
342    etap:is(
343        couch_set_view_test_util:full_reduce_id_btree(Group, IdBtree),
344        {ok, {ExpectedKVCount, ExpectedBitmask}},
345        "Id Btree has the right reduce value"),
346    etap:is(
347        couch_set_view_test_util:full_reduce_view_btree(Group, View1Btree),
348        {ok, {ExpectedKVCount, [ExpectedBtreeViewReduction], ExpectedBitmask}},
349        "View1 Btree has the right reduce value"),
350
351    etap:is(HeaderUpdateSeqs, ExpectedDbSeqs, "Header has right update seqs list"),
352    etap:is(Abitmask, ExpectedABitmask, "Header has right active bitmask"),
353    etap:is(Pbitmask, 0, "Header has right passive bitmask"),
354    etap:is(Cbitmask, ExpectedCBitmask, "Header has right cleanup bitmask"),
355    etap:is(PendingTrans, nil, "Header has nil pending transition"),
356
357    etap:diag("Verifying the Id Btree"),
358    MaxPerPart = num_docs() div num_set_partitions(),
359    {ok, _, {_, _, _, IdBtreeFoldResult}} = couch_set_view_test_util:fold_id_btree(
360        Group,
361        IdBtree,
362        fun(Kv, _, {P0, I0, C0, It}) ->
363            case C0 >= MaxPerPart of
364            true ->
365                P = P0 + 1,
366                I = P,
367                C = 1;
368            false ->
369                P = P0,
370                I = I0,
371                C = C0 + 1
372            end,
373            true = (P < num_set_partitions()),
374            DocId = doc_id(I),
375            Value = [{View1#set_view.id_num, DocId}],
376            ExpectedKv = {<<P:16, DocId/binary>>, {P, Value}},
377            case ExpectedKv =:= Kv of
378            true ->
379                ok;
380            false ->
381                etap:bail("Id Btree has an unexpected KV at iteration " ++ integer_to_list(It))
382            end,
383            {ok, {P, I + num_set_partitions(), C, It + 1}}
384        end,
385        {0, 0, 0, 0}, []),
386    etap:is(IdBtreeFoldResult, ExpectedKVCount,
387        "Id Btree has " ++ integer_to_list(ExpectedKVCount) ++ " entries"),
388
389    etap:diag("Verifying the View1 Btree"),
390    {ok, _, View1BtreeFoldResult} = couch_set_view_test_util:fold_view_btree(
391        Group,
392        View1Btree,
393        fun(Kv, _, I) ->
394            PartId = I rem num_set_partitions(),
395            DocId = doc_id(I),
396            ExpectedKv = {{DocId, DocId}, {PartId, ValueGenFun(I)}},
397            case ExpectedKv =:= Kv of
398            true ->
399                ok;
400            false ->
401                etap:bail("View1 Btree has an unexpected KV at iteration " ++ integer_to_list(I))
402            end,
403            {ok, I + 1}
404        end,
405        0, []),
406    etap:is(View1BtreeFoldResult, ExpectedKVCount,
407        "View1 Btree has " ++ integer_to_list(ExpectedKVCount) ++ " entries"),
408    ok.
409
410
411verify_btrees_3(ValueGenFun) ->
412    Group = get_group_snapshot(),
413    etap:diag("Verifying btrees"),
414    #set_view_group{
415        id_btree = IdBtree,
416        views = [View1],
417        index_header = #set_view_index_header{
418            pending_transition = PendingTrans,
419            seqs = HeaderUpdateSeqs,
420            abitmask = Abitmask,
421            pbitmask = Pbitmask,
422            cbitmask = Cbitmask
423        }
424    } = Group,
425    #set_view{
426        indexer = #mapreduce_view{
427            btree = View1Btree
428        }
429    } = View1,
430    ActiveParts = [0],
431    CleanupParts = lists:seq(1, num_set_partitions() - 1),
432    ExpectedBitmask = couch_set_view_util:build_bitmask(lists:seq(0, num_set_partitions() - 1)),
433    ExpectedABitmask = couch_set_view_util:build_bitmask(ActiveParts),
434    ExpectedCBitmask = couch_set_view_util:build_bitmask(CleanupParts),
435    ExpectedDbSeqs = couch_set_view_test_util:get_db_seqs(test_set_name(), ActiveParts),
436    ExpectedKVCount = num_docs(),
437    ExpectedBtreeViewReduction = num_docs(),
438    ExpectedPendingTrans = #set_view_transition{
439        active = [1],
440        passive = [],
441        unindexable = []
442    },
443
444    etap:is(
445        couch_set_view_test_util:full_reduce_id_btree(Group, IdBtree),
446        {ok, {ExpectedKVCount, ExpectedBitmask}},
447        "Id Btree has the right reduce value"),
448    etap:is(
449        couch_set_view_test_util:full_reduce_view_btree(Group, View1Btree),
450        {ok, {ExpectedKVCount, [ExpectedBtreeViewReduction], ExpectedBitmask}},
451        "View1 Btree has the right reduce value"),
452
453    etap:is(HeaderUpdateSeqs, ExpectedDbSeqs, "Header has right update seqs list"),
454    etap:is(Abitmask, ExpectedABitmask, "Header has right active bitmask"),
455    etap:is(Pbitmask, 0, "Header has right passive bitmask"),
456    etap:is(Cbitmask, ExpectedCBitmask, "Header has right cleanup bitmask"),
457    etap:is(PendingTrans, ExpectedPendingTrans, "Header has expected pending transition"),
458
459    etap:diag("Verifying the Id Btree"),
460    MaxPerPart = num_docs() div num_set_partitions(),
461    {ok, _, {_, _, _, IdBtreeFoldResult}} = couch_set_view_test_util:fold_id_btree(
462        Group,
463        IdBtree,
464        fun(Kv, _, {P0, I0, C0, It}) ->
465            case C0 >= MaxPerPart of
466            true ->
467                P = P0 + 1,
468                I = P,
469                C = 1;
470            false ->
471                P = P0,
472                I = I0,
473                C = C0 + 1
474            end,
475            true = (P < num_set_partitions()),
476            DocId = doc_id(I),
477            Value = [{View1#set_view.id_num, DocId}],
478            ExpectedKv = {<<P:16, DocId/binary>>, {P, Value}},
479            case ExpectedKv =:= Kv of
480            true ->
481                ok;
482            false ->
483                etap:bail("Id Btree has an unexpected KV at iteration " ++ integer_to_list(It))
484            end,
485            {ok, {P, I + num_set_partitions(), C, It + 1}}
486        end,
487        {0, 0, 0, 0}, []),
488    etap:is(IdBtreeFoldResult, ExpectedKVCount,
489        "Id Btree has " ++ integer_to_list(ExpectedKVCount) ++ " entries"),
490
491    etap:diag("Verifying the View1 Btree"),
492    {ok, _, View1BtreeFoldResult} = couch_set_view_test_util:fold_view_btree(
493        Group,
494        View1Btree,
495        fun(Kv, _, I) ->
496            PartId = I rem num_set_partitions(),
497            DocId = doc_id(I),
498            ExpectedKv = {{DocId, DocId}, {PartId, ValueGenFun(I)}},
499            case ExpectedKv =:= Kv of
500            true ->
501                ok;
502            false ->
503                etap:bail("View1 Btree has an unexpected KV at iteration " ++ integer_to_list(I))
504            end,
505            {ok, I + 1}
506        end,
507        0, []),
508    etap:is(View1BtreeFoldResult, ExpectedKVCount,
509        "View1 Btree has " ++ integer_to_list(ExpectedKVCount) ++ " entries"),
510    ok.
511
512
513verify_btrees_4(ValueGenFun) ->
514    Group = get_group_snapshot(),
515    etap:diag("Verifying btrees"),
516    #set_view_group{
517        id_btree = IdBtree,
518        views = [View1],
519        index_header = #set_view_index_header{
520            pending_transition = PendingTrans,
521            seqs = HeaderUpdateSeqs,
522            abitmask = Abitmask,
523            pbitmask = Pbitmask,
524            cbitmask = Cbitmask
525        }
526    } = Group,
527    #set_view{
528        indexer = #mapreduce_view{
529            btree = View1Btree
530        }
531    } = View1,
532    ActiveParts = [0],
533    ExpectedBitmask = couch_set_view_util:build_bitmask(ActiveParts),
534    ExpectedABitmask = couch_set_view_util:build_bitmask(ActiveParts),
535    ExpectedDbSeqs = couch_set_view_test_util:get_db_seqs(test_set_name(), ActiveParts),
536    ExpectedKVCount = (num_docs() div num_set_partitions()) + 1,
537    ExpectedBtreeViewReduction = (num_docs() div num_set_partitions()) + 1,
538
539    etap:is(
540        couch_set_view_test_util:full_reduce_id_btree(Group, IdBtree),
541        {ok, {ExpectedKVCount, ExpectedBitmask}},
542        "Id Btree has the right reduce value"),
543    etap:is(
544        couch_set_view_test_util:full_reduce_view_btree(Group, View1Btree),
545        {ok, {ExpectedKVCount, [ExpectedBtreeViewReduction], ExpectedBitmask}},
546        "View1 Btree has the right reduce value"),
547
548    etap:is(HeaderUpdateSeqs, ExpectedDbSeqs, "Header has right update seqs list"),
549    etap:is(Abitmask, ExpectedABitmask, "Header has right active bitmask"),
550    etap:is(Pbitmask, 0, "Header has right passive bitmask"),
551    etap:is(Cbitmask, 0, "Header has right cleanup bitmask"),
552    etap:is(PendingTrans, nil, "Header has nil pending transition"),
553
554    etap:diag("Verifying the Id Btree"),
555    {ok, _, {_, IdBtreeFoldResult}} = couch_set_view_test_util:fold_id_btree(
556        Group,
557        IdBtree,
558        fun(Kv, _, {I, Count}) ->
559            PartId = 0,
560            case Count == (ExpectedKVCount - 1) of
561            true ->
562                DocId = doc_id(9000010);
563            false ->
564                DocId = doc_id(I)
565            end,
566            Value = [{View1#set_view.id_num, DocId}],
567            ExpectedKv = {<<PartId:16, DocId/binary>>, {PartId, Value}},
568            case ExpectedKv =:= Kv of
569            true ->
570                ok;
571            false ->
572                etap:bail("Id Btree has an unexpected KV at iteration " ++ integer_to_list(Count))
573            end,
574            {ok, {I + 64, Count + 1}}
575        end,
576        {0, 0}, []),
577    etap:is(IdBtreeFoldResult, ExpectedKVCount,
578        "Id Btree has " ++ integer_to_list(ExpectedKVCount) ++ " entries"),
579
580    etap:diag("Verifying the View1 Btree"),
581    {ok, _, {_, View1BtreeFoldResult}} = couch_set_view_test_util:fold_view_btree(
582        Group,
583        View1Btree,
584        fun(Kv, _, {I, Count}) ->
585            case Count == (ExpectedKVCount - 1) of
586            true ->
587                DocId = doc_id(9000010),
588                PartId = 0,
589                Value = 9000010;
590            false ->
591                DocId = doc_id(I),
592                PartId = I rem num_set_partitions(),
593                Value = ValueGenFun(I)
594            end,
595            ExpectedKv = {{DocId, DocId}, {PartId, Value}},
596            case ExpectedKv =:= Kv of
597            true ->
598                ok;
599            false ->
600                etap:bail("View1 Btree has an unexpected KV at iteration " ++ integer_to_list(Count))
601            end,
602            {ok, {I + 64, Count + 1}}
603        end,
604        {0, 0}, []),
605    etap:is(View1BtreeFoldResult, ExpectedKVCount,
606        "View1 Btree has " ++ integer_to_list(ExpectedKVCount) ++ " entries"),
607    ok.
608
609
610get_group_snapshot() ->
611    get_group_snapshot(false).
612
613get_group_snapshot(Staleness) ->
614    GroupPid = couch_set_view:get_group_pid(
615        mapreduce_view, test_set_name(), ddoc_id(), prod),
616    {ok, Group, 0} = gen_server:call(
617        GroupPid, #set_view_group_req{stale = Staleness, debug = true}, infinity),
618    Group.
619
620
621compact_view_group() ->
622    {ok, CompactPid} = couch_set_view_compactor:start_compact(
623        mapreduce_view, test_set_name(), ddoc_id(), main),
624    Ref = erlang:monitor(process, CompactPid),
625    etap:diag("Waiting for main view group compaction to finish"),
626    receive
627    {'DOWN', Ref, process, CompactPid, normal} ->
628        ok;
629    {'DOWN', Ref, process, CompactPid, noproc} ->
630        ok;
631    {'DOWN', Ref, process, CompactPid, Reason} ->
632        etap:bail("Failure compacting main view group: " ++ couch_util:to_list(Reason))
633    after ?MAX_WAIT_TIME ->
634        etap:bail("Timeout waiting for main view group compaction to finish")
635    end.
636
637
638test_unindexable_partitions() ->
639    % Verify that partitions in the pending transition can be marked
640    % as unindexable and indexable back again.
641    Group0 = get_group_snapshot(ok),
642    PrevPendingTrans = ?set_pending_transition(Group0),
643
644    NewActivePending = lists:seq(1, num_set_partitions() div 2, 2),
645    NewPassivePending = lists:seq(num_set_partitions() div 2, num_set_partitions() - 1, 2),
646    ok = couch_set_view:set_partition_states(
647        mapreduce_view, test_set_name(), ddoc_id(), NewActivePending,
648        NewPassivePending, []),
649
650    PendingActiveUnindexable = lists:sublist(NewActivePending, length(NewActivePending) div 2),
651    PendingPassiveUnindexable = lists:sublist(NewPassivePending, length(NewPassivePending) div 2),
652    Unindexable = ordsets:union(PendingActiveUnindexable, PendingPassiveUnindexable),
653    ok = couch_set_view:mark_partitions_unindexable(
654        mapreduce_view, test_set_name(), ddoc_id(), Unindexable),
655
656    etap:diag("Marking unindexable partitions to the state they're already in, is a no-op"),
657    ok = couch_set_view:set_partition_states(
658        mapreduce_view, test_set_name(), ddoc_id(),
659        PendingActiveUnindexable, PendingPassiveUnindexable, []),
660
661    Group1 = get_group_snapshot(ok),
662    PendingTrans = ?set_pending_transition(Group1),
663    etap:is(?pending_transition_unindexable(PendingTrans),
664            Unindexable,
665            "Right set of unindexable partitions in pending transition"),
666    etap:is(?pending_transition_active(PendingTrans),
667            NewActivePending,
668            "Right set of active partitions in pending transition"),
669    etap:is(?pending_transition_passive(PendingTrans),
670            NewPassivePending,
671            "Right set of passive partitions in pending transition"),
672    etap:is(?set_unindexable_seqs(Group1),
673            [],
674            "Right set of unindexable partitions"),
675
676    ok = couch_set_view:mark_partitions_indexable(
677        mapreduce_view, test_set_name(), ddoc_id(), Unindexable),
678
679    Group2 = get_group_snapshot(ok),
680    PendingTrans2 = ?set_pending_transition(Group2),
681    etap:is(?pending_transition_unindexable(PendingTrans2),
682            [],
683            "Empty set of unindexable partitions in pending transition"),
684    etap:is(?pending_transition_active(PendingTrans2),
685            NewActivePending,
686            "Right set of active partitions in pending transition"),
687    etap:is(?pending_transition_passive(PendingTrans2),
688            NewPassivePending,
689            "Right set of passive partitions in pending transition"),
690    etap:is(?set_unindexable_seqs(Group2),
691            [],
692            "Right set of unindexable partitions"),
693
694    % Restore to previous state
695    PrevActivePending = ?pending_transition_active(PrevPendingTrans),
696    PrevPassivePending = ?pending_transition_passive(PrevPendingTrans),
697    ok = couch_set_view:set_partition_states(
698        mapreduce_view, test_set_name(), ddoc_id(), [], [],
699        ordsets:union(NewActivePending, NewPassivePending)),
700    ok = couch_set_view:set_partition_states(
701        mapreduce_view, test_set_name(), ddoc_id(), PrevActivePending,
702        PrevPassivePending, []).
703
704
705test_monitor_pending_partition() ->
706    % Mark partition 0 for cleanup, recreate it (1 doc), add it to pending transition,
707    % ask to monitor partition 0, perform cleanup/update and check an update message
708    % is received after.
709    etap:diag("Marking partition 0 for cleanup"),
710    ok = couch_set_view:set_partition_states(
711        mapreduce_view, test_set_name(), ddoc_id(), [], [], [0]),
712
713    Group0 = get_group_snapshot(ok),
714    etap:is(?set_seqs(Group0), [], "Empty list of seqs in group snapshot"),
715    etap:is(?set_cbitmask(Group0), 1, "Partition 0 in cleanup bitmask"),
716
717    etap:diag("Recreating partition 0 database"),
718    recreate_db(0, 9000011),
719
720    etap:diag("Marking partition 0 as active while it's still in cleanup"),
721    ok = couch_set_view:set_partition_states(
722        mapreduce_view, test_set_name(), ddoc_id(), [0], [], []),
723
724    Group1 = get_group_snapshot(ok),
725    PendingTrans1 = ?set_pending_transition(Group1),
726    etap:is(?set_cbitmask(Group1), 1, "Partition 0 in cleanup bitmask"),
727    etap:is(?pending_transition_active(PendingTrans1), [0],
728            "Partition 0 in pending transition"),
729
730    Parent = self(),
731    {ListenerPid, ListenerRef} = spawn_monitor(fun() ->
732        etap:diag("Asking view group to monitor partition 0 (in pending transition)"),
733        Ref1 = couch_set_view:monitor_partition_update(
734            mapreduce_view, test_set_name(), ddoc_id(), 0),
735        Parent ! {self(), ok},
736        receive
737        {Ref1, Reason} ->
738            exit(Reason)
739        end
740    end),
741
742    receive
743    {ListenerPid, ok} ->
744        etap:diag("Received ack from listener child");
745    {'DOWN', ListenerRef, _, _, Reason} ->
746        etap:bail("Child terminated with reason: " ++ couch_util:to_list(Reason))
747    after 10000 ->
748        etap:bail("Timeout waiting for child listener ack")
749    end,
750
751    % Perform cleanup + apply pending transition + update + notify listener
752    GroupPid = couch_set_view:get_group_pid(
753        mapreduce_view, test_set_name(), ddoc_id(), prod),
754    {ok, CleanerPid} = gen_server:call(GroupPid, start_cleaner, infinity),
755    CleanerRef = erlang:monitor(process, CleanerPid),
756    receive
757    {'DOWN', CleanerRef, _, _, _} ->
758        ok
759    after 60000 ->
760        etap:bail("Timeout waiting for cleaner to finish")
761    end,
762
763    Group2 = get_group_snapshot(false),
764    #set_view_group{
765        id_btree = IdBtree,
766        views = [#set_view{indexer = #mapreduce_view{btree = View1Btree}}]
767    } = Group2,
768
769    etap:is(?set_cbitmask(Group2), 0, "Cleanup bitmask is 0"),
770    etap:is(?set_abitmask(Group2), 1, "Active bitmask is 1"),
771    etap:is(?set_pending_transition(Group2), nil, "Pending transition is nil"),
772
773    receive
774    {'DOWN', ListenerRef, _, _, updated} ->
775        etap:diag("Child got notified partition was updated in index");
776    {'DOWN', ListenerRef, _, _, Reason2} ->
777        etap:bail("Child terminated with reason: " ++ couch_util:to_list(Reason2))
778    after 30000 ->
779        etap:bail("Child didn't terminate after pending transition was applied")
780    end,
781
782    etap:diag("Verifying the Id Btree"),
783    {ok, _, IdBtreeFoldResult} = couch_set_view_test_util:fold_id_btree(
784        Group2,
785        IdBtree,
786        fun(Kv, _, Acc) -> {ok, [Kv | Acc]} end,
787        [], []),
788    etap:is(IdBtreeFoldResult, [{<<0:16, (doc_id(9000011))/binary>>, {0, [{0, doc_id(9000011)}]}}],
789            "Id Btree has 1 entry"),
790
791    etap:diag("Verifying the View1 Btree"),
792    {ok, _, View1BtreeFoldResult} = couch_set_view_test_util:fold_view_btree(
793        Group2,
794        View1Btree,
795        fun(Kv, _, Acc) -> {ok, [Kv | Acc]} end,
796        [], []),
797    etap:is(View1BtreeFoldResult, [{{doc_id(9000011), doc_id(9000011)}, {0, 9000011}}],
798            "View1 Btree has 1 entry"),
799    ok.
800
801
802test_pending_transition_changes() ->
803    Group0 = get_group_snapshot(ok),
804    PendingTrans0 = ?set_pending_transition(Group0),
805    etap:is(?pending_transition_active(PendingTrans0), [1],
806            "Partition 1 in pending transition active set"),
807    etap:is(?pending_transition_passive(PendingTrans0), [],
808            "Empty pending transition passive set"),
809    etap:is(?pending_transition_unindexable(PendingTrans0), [],
810            "Empty pending transition unindexable set"),
811
812    ok = couch_set_view:set_partition_states(
813        mapreduce_view, test_set_name(), ddoc_id(), [], [1], []),
814
815    Group1 = get_group_snapshot(ok),
816    PendingTrans1 = ?set_pending_transition(Group1),
817    etap:is(?pending_transition_active(PendingTrans1), [],
818            "Empty pending transition active set"),
819    etap:is(?pending_transition_passive(PendingTrans1), [1],
820            "Partition 1 in pending transition passive set"),
821    etap:is(?pending_transition_unindexable(PendingTrans1), [],
822            "Empty pending transition unindexable set"),
823
824    ok = couch_set_view:set_partition_states(
825        mapreduce_view, test_set_name(), ddoc_id(), [1], [], []),
826
827    Group2 = get_group_snapshot(ok),
828    PendingTrans2 = ?set_pending_transition(Group2),
829    etap:is(?pending_transition_active(PendingTrans2), [1],
830            "Partition 1 in pending transition active set"),
831    etap:is(?pending_transition_passive(PendingTrans2), [],
832            "Empty pending transition passive set"),
833    etap:is(?pending_transition_unindexable(PendingTrans2), [],
834            "Empty pending transition unindexable set"),
835    ok.
836