1#!/usr/bin/env escript
2%% -*- Mode: Erlang; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
3%%! -smp enable
4
5% Licensed under the Apache License, Version 2.0 (the "License"); you may not
6% use this file except in compliance with the License. You may obtain a copy of
7% the License at
8%
9%   http://www.apache.org/licenses/LICENSE-2.0
10%
11% Unless required by applicable law or agreed to in writing, software
12% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14% License for the specific language governing permissions and limitations under
15% the License.
16
17-define(MAX_WAIT_TIME, 600 * 1000).
18-define(i2l(I), integer_to_list(I)).
19
20-include_lib("couch_set_view/include/couch_set_view.hrl").
21
22test_set_name() -> <<"couch_test_spatial_index_main_compact">>.
23num_set_partitions() -> 64.
24ddoc_id() -> <<"_design/test">>.
25num_docs() -> 24128.
26
27
28main(_) ->
29    test_util:init_code_path(),
30
31    etap:plan(64),
32    case (catch test()) of
33        ok ->
34            etap:end_tests();
35        Other ->
36            etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
37            etap:bail(Other)
38    end,
39    %init:stop(),
40    %receive after infinity -> ok end,
41    ok.
42
43
44test() ->
45    spatial_test_util:start_server(test_set_name()),
46
47    couch_set_view_test_util:delete_set_dbs(
48        test_set_name(), num_set_partitions()),
49    couch_set_view_test_util:create_set_dbs(
50        test_set_name(), num_set_partitions()),
51
52    populate_set(),
53
54    etap:diag("Verifying group snapshot before marking partitions [ 8 .. 31 ] "
55        " for cleanup"),
56    #set_view_group{index_header = Header0} = get_group_snapshot(false),
57    etap:is(
58        [P || {P, _} <- Header0#set_view_index_header.seqs],
59        lists:seq(0, 31),
60        "Right list of partitions in the header's seq field"),
61    etap:is(
62        Header0#set_view_index_header.has_replica,
63        true,
64        "Header has replica support flag set to true"),
65    lists:foreach(
66        fun({PartId, Seq}) ->
67            DocCount = couch_set_view_test_util:doc_count(
68                test_set_name(), [PartId]),
69            etap:is(Seq, DocCount,
70                "Right update seq for partition " ++ ?i2l(PartId))
71        end,
72        Header0#set_view_index_header.seqs),
73
74    DiskSizeBefore = main_index_disk_size(),
75
76    verify_group_info_before_cleanup_request(),
77    GroupPid = couch_set_view:get_group_pid(
78        spatial_view, test_set_name(), ddoc_id(), prod),
79    ok = gen_server:call(GroupPid, {set_auto_cleanup, false}, infinity),
80    ok = couch_set_view:set_partition_states(
81        spatial_view, test_set_name(), ddoc_id(), [], [], lists:seq(8, 63)),
82    verify_group_info_after_cleanup_request(),
83
84    GroupBefore = get_group_snapshot(false),
85
86    etap:is(
87        couch_ref_counter:count(GroupBefore#set_view_group.ref_counter),
88        1,
89        "Main group's ref counter count is 1"),
90
91    etap:diag("Triggering main group compaction"),
92    {ok, CompactPid} = couch_set_view_compactor:start_compact(
93        spatial_view, test_set_name(), ddoc_id(), main),
94    Ref = erlang:monitor(process, CompactPid),
95    etap:diag("Waiting for main group compaction to finish"),
96    receive
97    {'DOWN', Ref, process, CompactPid, normal} ->
98        ok;
99    {'DOWN', Ref, process, CompactPid, noproc} ->
100        ok;
101    {'DOWN', Ref, process, CompactPid, Reason} ->
102        etap:bail("Failure compacting main group: " ++
103            couch_util:to_list(Reason))
104    after ?MAX_WAIT_TIME ->
105        etap:bail("Timeout waiting for main group compaction to finish")
106    end,
107
108    GroupAfter = get_group_snapshot(false),
109
110    etap:isnt(
111        GroupAfter#set_view_group.ref_counter,
112        GroupBefore#set_view_group.ref_counter,
113        "Different ref counter for main group after compaction"),
114    etap:isnt(
115        GroupAfter#set_view_group.fd,
116        GroupBefore#set_view_group.fd,
117        "Different fd for main group after compaction"),
118
119    etap:is(
120        couch_ref_counter:count(GroupAfter#set_view_group.ref_counter),
121        1,
122        "Main group's new ref counter count is 1 after compaction"),
123
124    etap:is(
125        is_process_alive(GroupBefore#set_view_group.ref_counter),
126        false,
127        "Old group ref counter is dead"),
128
129    etap:is(
130        is_process_alive(GroupBefore#set_view_group.fd),
131        false,
132        "Old group fd is dead"),
133
134    GroupInfo = get_main_group_info(),
135    {Stats} = couch_util:get_value(stats, GroupInfo),
136    etap:is(couch_util:get_value(compactions, Stats), 1,
137        "Main group had 1 full compaction in stats"),
138    etap:is(couch_util:get_value(cleanups, Stats), 1,
139        "Main group had 1 full cleanup in stats"),
140
141    verify_group_info_after_main_compact(),
142
143    DiskSizeAfter = main_index_disk_size(),
144    etap:is(DiskSizeAfter < DiskSizeBefore, true,
145        "Index file size is smaller after compaction"),
146
147    etap:diag("Verifying group snapshot after main group compaction"),
148    #set_view_group{index_header = Header1} = get_group_snapshot(false),
149    etap:is(
150        [P || {P, _} <- Header1#set_view_index_header.seqs],
151        lists:seq(0, 7),
152        "Right list of partitions in the header's seq field"),
153    etap:is(
154        Header1#set_view_index_header.has_replica,
155        true,
156        "Header has replica support flag set to true"),
157    etap:is(
158        Header1#set_view_index_header.num_partitions,
159        Header0#set_view_index_header.num_partitions,
160        "Compaction preserved header field num_partitions"),
161    lists:foreach(
162        fun({PartId, Seq}) ->
163            DocCount = couch_set_view_test_util:doc_count(
164                test_set_name(), [PartId]),
165            etap:is(Seq, DocCount,
166                "Right update seq for partition " ++ ?i2l(PartId))
167        end,
168        Header1#set_view_index_header.seqs),
169
170    couch_set_view_test_util:delete_set_dbs(
171        test_set_name(), num_set_partitions()),
172    ok = timer:sleep(1000),
173    couch_set_view_test_util:stop_server(),
174    ok.
175
176
177get_group_snapshot(StaleType) ->
178    {ok, Group} = couch_set_view:get_group(
179        spatial_view, test_set_name(), ddoc_id(),
180        #set_view_group_req{stale = StaleType, debug = true}),
181    couch_ref_counter:drop(Group#set_view_group.ref_counter),
182    Group.
183
184
185verify_group_info_before_cleanup_request() ->
186    etap:diag("Verifying main group info before marking partitions "
187          "[ 8 .. 31 ] for cleanup"),
188    GroupInfo = get_main_group_info(),
189    etap:is(
190        couch_util:get_value(active_partitions, GroupInfo),
191        lists:seq(0, 31),
192        "Main group has [ 0 .. 31 ] as active partitions"),
193    etap:is(
194        couch_util:get_value(passive_partitions, GroupInfo),
195        [],
196        "Main group has [ ] as passive partitions"),
197    etap:is(
198        couch_util:get_value(cleanup_partitions, GroupInfo),
199        [],
200        "Main group has [ ] as cleanup partitions").
201
202
203verify_group_info_after_cleanup_request() ->
204    etap:diag("Verifying main group info after marking partitions [ 8 .. 31 ] "
205          "for cleanup"),
206    GroupInfo = get_main_group_info(),
207    etap:is(
208        couch_util:get_value(active_partitions, GroupInfo),
209        lists:seq(0, 7),
210        "Main group has [ 0 .. 7 ] as active partitions"),
211    etap:is(
212        couch_util:get_value(passive_partitions, GroupInfo),
213        [],
214        "Main group has [ ] as passive partitions"),
215    CleanupParts = couch_util:get_value(cleanup_partitions, GroupInfo),
216    etap:is(
217        length(CleanupParts) > 0,
218        true,
219        "Main group has non-empty set of cleanup partitions"),
220    etap:is(
221        ordsets:intersection(CleanupParts,
222            lists:seq(0, 7) ++ lists:seq(32, 63)),
223        [],
224        "Main group doesn't have any cleanup partition with ID in"
225        " [ 0 .. 7, 32 .. 63 ]").
226
227
228verify_group_info_after_main_compact() ->
229    etap:diag("Verifying main group info after compaction"),
230    GroupInfo = get_main_group_info(),
231    etap:is(
232        couch_util:get_value(active_partitions, GroupInfo),
233        lists:seq(0, 7),
234        "Main group has [ 0 .. 7 ] as active partitions"),
235    etap:is(
236        couch_util:get_value(passive_partitions, GroupInfo),
237        [],
238        "Main group has [ ] as passive partitions"),
239    etap:is(
240        couch_util:get_value(cleanup_partitions, GroupInfo),
241        [],
242        "Main group has [ ] as cleanup partitions").
243
244
245get_main_group_info() ->
246    {ok, MainInfo} = couch_set_view:get_group_info(
247        spatial_view, test_set_name(), ddoc_id(), prod),
248    MainInfo.
249
250
251main_index_disk_size() ->
252    Info = get_main_group_info(),
253    Size = couch_util:get_value(disk_size, Info),
254    true = is_integer(Size),
255    true = (Size >= 0),
256    Size.
257
258
259create_docs(From, To) ->
260    rand:seed(exrop, {91, 1, 11}),
261    lists:map(
262        fun(I) ->
263            RandomMin = rand:uniform(2000),
264            RandomMax = RandomMin + rand:uniform(167),
265            RandomMin2 = rand:uniform(1769),
266            RandomMax2 = RandomMin2 + rand:uniform(132),
267            DocId = iolist_to_binary(["doc", integer_to_list(I)]),
268            {[
269              {<<"meta">>, {[{<<"id">>, DocId}]}},
270              {<<"json">>, {[
271                             {<<"value">>, I},
272                             {<<"min">>, RandomMin},
273                             {<<"max">>, RandomMax},
274                             {<<"min2">>, RandomMin2},
275                             {<<"max2">>, RandomMax2}
276                            ]}}
277            ]}
278        end,
279        lists:seq(From, To)).
280
281
282populate_set() ->
283    couch_set_view:cleanup_index_files(spatial_view, test_set_name()),
284    etap:diag("Populating the " ++ ?i2l(num_set_partitions()) ++
285        " databases with " ++ ?i2l(num_docs()) ++ " documents"),
286    DDoc = {[
287        {<<"meta">>, {[{<<"id">>, ddoc_id()}]}},
288        {<<"json">>, {[
289            {<<"spatial">>, {[
290                {<<"test">>, <<"function(doc, meta) { "
291                    "emit([[doc.min, doc.max], [doc.min2, doc.max2]], "
292                    "'val'+doc.value); }">>}
293            ]}}
294        ]}}
295    ]},
296    ok = couch_set_view_test_util:update_ddoc(test_set_name(), DDoc),
297    DocList = create_docs(1, num_docs()),
298    ok = couch_set_view_test_util:populate_set_sequentially(
299        test_set_name(),
300        lists:seq(0, num_set_partitions() - 1),
301        DocList),
302    etap:diag("Configuring set view with partitions [0 .. 31] as active"),
303    Params = #set_view_params{
304        max_partitions = num_set_partitions(),
305        active_partitions = lists:seq(0, 31),
306        passive_partitions = [],
307        use_replica_index = true
308    },
309    ok = couch_set_view:define_group(
310           spatial_view, test_set_name(), ddoc_id(), Params).
311
312