1#!/usr/bin/env escript
2%% -*- Mode: Erlang; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
3%%! -smp enable
4
5% Licensed under the Apache License, Version 2.0 (the "License"); you may not
6% use this file except in compliance with the License. You may obtain a copy of
7% the License at
8%
9%   http://www.apache.org/licenses/LICENSE-2.0
10%
11% Unless required by applicable law or agreed to in writing, software
12% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14% License for the specific language governing permissions and limitations under
15% the License.
16
17-define(MAX_WAIT_TIME, 600 * 1000).
18-define(i2l(I), integer_to_list(I)).
19
20-include_lib("couch_set_view/include/couch_set_view.hrl").
21
22test_set_name() -> <<"couch_test_set_index_main_compact">>.
23num_set_partitions() -> 64.
24ddoc_id() -> <<"_design/test">>.
25num_docs() -> 24128.
26
27
28main(_) ->
29    test_util:init_code_path(),
30
31    etap:plan(64),
32    case (catch test()) of
33        ok ->
34            etap:end_tests();
35        Other ->
36            etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
37            etap:bail(Other)
38    end,
39    ok.
40
41
42test() ->
43    couch_set_view_test_util:start_server(test_set_name()),
44
45    couch_set_view_test_util:delete_set_dbs(test_set_name(), num_set_partitions()),
46    couch_set_view_test_util:create_set_dbs(test_set_name(), num_set_partitions()),
47
48    populate_set(),
49
50    etap:diag("Verifying group snapshot before marking partitions [ 8 .. 31 ] for cleanup"),
51    #set_view_group{index_header = Header0} = get_group_snapshot(false),
52    etap:is(
53        [P || {P, _} <- Header0#set_view_index_header.seqs],
54        lists:seq(0, 31),
55        "Right list of partitions in the header's seq field"),
56    etap:is(
57        Header0#set_view_index_header.has_replica,
58        true,
59        "Header has replica support flag set to true"),
60    lists:foreach(
61        fun({PartId, Seq}) ->
62            DocCount = couch_set_view_test_util:doc_count(test_set_name(), [PartId]),
63            etap:is(Seq, DocCount, "Right update seq for partition " ++ ?i2l(PartId))
64        end,
65        Header0#set_view_index_header.seqs),
66
67    DiskSizeBefore = main_index_disk_size(),
68
69    verify_group_info_before_cleanup_request(),
70    GroupPid = couch_set_view:get_group_pid(
71        mapreduce_view, test_set_name(), ddoc_id(), prod),
72    ok = gen_server:call(GroupPid, {set_auto_cleanup, false}, infinity),
73    ok = couch_set_view:set_partition_states(
74        mapreduce_view, test_set_name(), ddoc_id(), [], [], lists:seq(8, 63)),
75    verify_group_info_after_cleanup_request(),
76
77    GroupBefore = get_group_snapshot(false),
78
79    etap:is(
80        couch_ref_counter:count(GroupBefore#set_view_group.ref_counter),
81        1,
82        "Main group's ref counter count is 1"),
83
84    etap:diag("Triggering main group compaction"),
85    {ok, CompactPid} = couch_set_view_compactor:start_compact(
86        mapreduce_view, test_set_name(), ddoc_id(), main),
87    Ref = erlang:monitor(process, CompactPid),
88    etap:diag("Waiting for main group compaction to finish"),
89    receive
90    {'DOWN', Ref, process, CompactPid, normal} ->
91        ok;
92    {'DOWN', Ref, process, CompactPid, noproc} ->
93        ok;
94    {'DOWN', Ref, process, CompactPid, Reason} ->
95        etap:bail("Failure compacting main group: " ++ couch_util:to_list(Reason))
96    after ?MAX_WAIT_TIME ->
97        etap:bail("Timeout waiting for main group compaction to finish")
98    end,
99
100    GroupAfter = get_group_snapshot(false),
101
102    etap:isnt(
103        GroupAfter#set_view_group.ref_counter,
104        GroupBefore#set_view_group.ref_counter,
105        "Different ref counter for main group after compaction"),
106    etap:isnt(
107        GroupAfter#set_view_group.fd,
108        GroupBefore#set_view_group.fd,
109        "Different fd for main group after compaction"),
110
111    etap:is(
112        couch_ref_counter:count(GroupAfter#set_view_group.ref_counter),
113        1,
114        "Main group's new ref counter count is 1 after compaction"),
115
116    etap:is(
117        is_process_alive(GroupBefore#set_view_group.ref_counter),
118        false,
119        "Old group ref counter is dead"),
120
121    etap:is(
122        is_process_alive(GroupBefore#set_view_group.fd),
123        false,
124        "Old group fd is dead"),
125
126    GroupInfo = get_main_group_info(),
127    {Stats} = couch_util:get_value(stats, GroupInfo),
128    etap:is(couch_util:get_value(compactions, Stats), 1, "Main group had 1 full compaction in stats"),
129    etap:is(couch_util:get_value(cleanups, Stats), 1, "Main group had 1 full cleanup in stats"),
130
131    verify_group_info_after_main_compact(),
132
133    DiskSizeAfter = main_index_disk_size(),
134    etap:is(DiskSizeAfter < DiskSizeBefore, true, "Index file size is smaller after compaction"),
135
136    etap:diag("Verifying group snapshot after main group compaction"),
137    #set_view_group{index_header = Header1} = get_group_snapshot(false),
138    etap:is(
139        [P || {P, _} <- Header1#set_view_index_header.seqs],
140        lists:seq(0, 7),
141        "Right list of partitions in the header's seq field"),
142    etap:is(
143        Header1#set_view_index_header.has_replica,
144        true,
145        "Header has replica support flag set to true"),
146    etap:is(
147        Header1#set_view_index_header.num_partitions,
148        Header0#set_view_index_header.num_partitions,
149        "Compaction preserved header field num_partitions"),
150    lists:foreach(
151        fun({PartId, Seq}) ->
152            DocCount = couch_set_view_test_util:doc_count(test_set_name(), [PartId]),
153            etap:is(Seq, DocCount, "Right update seq for partition " ++ ?i2l(PartId))
154        end,
155        Header1#set_view_index_header.seqs),
156
157    couch_set_view_test_util:delete_set_dbs(test_set_name(), num_set_partitions()),
158    couch_set_view_test_util:stop_server(),
159    ok.
160
161
162get_group_snapshot(StaleType) ->
163    {ok, Group} = couch_set_view:get_group(
164        mapreduce_view, test_set_name(), ddoc_id(),
165        #set_view_group_req{stale = StaleType, debug = true}),
166    couch_ref_counter:drop(Group#set_view_group.ref_counter),
167    Group.
168
169
170verify_group_info_before_cleanup_request() ->
171    etap:diag("Verifying main group info before marking partitions [ 8 .. 31 ] for cleanup"),
172    GroupInfo = get_main_group_info(),
173    etap:is(
174        couch_util:get_value(active_partitions, GroupInfo),
175        lists:seq(0, 31),
176        "Main group has [ 0 .. 31 ] as active partitions"),
177    etap:is(
178        couch_util:get_value(passive_partitions, GroupInfo),
179        [],
180        "Main group has [ ] as passive partitions"),
181    etap:is(
182        couch_util:get_value(cleanup_partitions, GroupInfo),
183        [],
184        "Main group has [ ] as cleanup partitions").
185
186
187verify_group_info_after_cleanup_request() ->
188    etap:diag("Verifying main group info after marking partitions [ 8 .. 31 ] for cleanup"),
189    GroupInfo = get_main_group_info(),
190    etap:is(
191        couch_util:get_value(active_partitions, GroupInfo),
192        lists:seq(0, 7),
193        "Main group has [ 0 .. 7 ] as active partitions"),
194    etap:is(
195        couch_util:get_value(passive_partitions, GroupInfo),
196        [],
197        "Main group has [ ] as passive partitions"),
198    CleanupParts = couch_util:get_value(cleanup_partitions, GroupInfo),
199    etap:is(
200        length(CleanupParts) > 0,
201        true,
202        "Main group has non-empty set of cleanup partitions"),
203    etap:is(
204        ordsets:intersection(CleanupParts, lists:seq(0, 7) ++ lists:seq(32, 63)),
205        [],
206        "Main group doesn't have any cleanup partition with ID in [ 0 .. 7, 32 .. 63 ]").
207
208
209verify_group_info_after_main_compact() ->
210    etap:diag("Verifying main group info after compaction"),
211    GroupInfo = get_main_group_info(),
212    etap:is(
213        couch_util:get_value(active_partitions, GroupInfo),
214        lists:seq(0, 7),
215        "Main group has [ 0 .. 7 ] as active partitions"),
216    etap:is(
217        couch_util:get_value(passive_partitions, GroupInfo),
218        [],
219        "Main group has [ ] as passive partitions"),
220    etap:is(
221        couch_util:get_value(cleanup_partitions, GroupInfo),
222        [],
223        "Main group has [ ] as cleanup partitions").
224
225
226get_main_group_info() ->
227    {ok, MainInfo} = couch_set_view:get_group_info(
228        mapreduce_view, test_set_name(), ddoc_id(), prod),
229    MainInfo.
230
231
232main_index_disk_size() ->
233    Info = get_main_group_info(),
234    Size = couch_util:get_value(disk_size, Info),
235    true = is_integer(Size),
236    true = (Size >= 0),
237    Size.
238
239
240populate_set() ->
241    couch_set_view:cleanup_index_files(mapreduce_view, test_set_name()),
242    etap:diag("Populating the " ++ ?i2l(num_set_partitions()) ++
243        " databases with " ++ ?i2l(num_docs()) ++ " documents"),
244    DDoc = {[
245        {<<"meta">>, {[{<<"id">>, ddoc_id()}]}},
246        {<<"json">>, {[
247        {<<"language">>, <<"javascript">>},
248        {<<"views">>, {[
249            {<<"test">>, {[
250                {<<"map">>, <<"function(doc, meta) { emit(meta.id, null); }">>},
251                {<<"reduce">>, <<"_count">>}
252            ]}}
253        ]}}
254        ]}}
255    ]},
256    ok = couch_set_view_test_util:update_ddoc(test_set_name(), DDoc),
257    DocList = lists:map(
258        fun(I) ->
259            {[
260                {<<"meta">>, {[{<<"id">>, iolist_to_binary(["doc", ?i2l(I)])}]}},
261                {<<"json">>, {[
262                    {<<"value">>, I}
263                ]}}
264            ]}
265        end,
266        lists:seq(1, num_docs())),
267    ok = couch_set_view_test_util:populate_set_sequentially(
268        test_set_name(),
269        lists:seq(0, num_set_partitions() - 1),
270        DocList),
271    etap:diag("Configuring set view with partitions [0 .. 31] as active"),
272    Params = #set_view_params{
273        max_partitions = num_set_partitions(),
274        active_partitions = lists:seq(0, 31),
275        passive_partitions = [],
276        use_replica_index = true
277    },
278    ok = couch_set_view:define_group(
279        mapreduce_view, test_set_name(), ddoc_id(), Params).
280