1#!/usr/bin/env escript
2%% -*- Mode: Erlang; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
3%%! -smp enable
4
5% Licensed under the Apache License, Version 2.0 (the "License"); you may not
6% use this file except in compliance with the License. You may obtain a copy of
7% the License at
8%
9%   http://www.apache.org/licenses/LICENSE-2.0
10%
11% Unless required by applicable law or agreed to in writing, software
12% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14% License for the specific language governing permissions and limitations under
15% the License.
16
17-define(MAX_WAIT_TIME, 600 * 1000).
18
19-include_lib("couch_set_view/include/couch_set_view.hrl").
20
21test_set_name() -> <<"couch_test_spatial_index_replica_compact">>.
22num_set_partitions() -> 64.
23ddoc_id() -> <<"_design/test">>.
24num_docs() -> 24128.
25
26
27main(_) ->
28    test_util:init_code_path(),
29
30    etap:plan(26),
31    case (catch test()) of
32        ok ->
33            etap:end_tests();
34        Other ->
35            etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
36            etap:bail(Other)
37    end,
38    %init:stop(),
39    %receive after infinity -> ok end,
40    ok.
41
42
43test() ->
44    spatial_test_util:start_server(test_set_name()),
45
46    couch_set_view_test_util:delete_set_dbs(
47        test_set_name(), num_set_partitions()),
48    couch_set_view_test_util:create_set_dbs(
49        test_set_name(), num_set_partitions()),
50
51    populate_set(),
52
53    etap:diag("Marking partitions [ 8 .. 63 ] as replicas"),
54    ok = couch_set_view:add_replica_partitions(
55        spatial_view, test_set_name(), ddoc_id(), lists:seq(8, 63)),
56
57    verify_group_info_before_replica_removal(),
58    wait_for_replica_full_update(),
59    verify_group_info_before_replica_removal(),
60
61    etap:diag("Removing partitions [ 8 .. 63 ] from replica set"),
62    ok = couch_set_view:remove_replica_partitions(
63        spatial_view, test_set_name(), ddoc_id(), lists:seq(8, 63)),
64    verify_group_info_after_replica_removal(),
65
66    DiskSizeBefore = replica_index_disk_size(),
67
68    {MainGroupBefore, RepGroupBefore} = get_group_snapshots(),
69
70    etap:diag("Trigerring replica group compaction"),
71    {ok, CompactPid} = couch_set_view_compactor:start_compact(
72        spatial_view, test_set_name(), ddoc_id(), replica),
73    Ref = erlang:monitor(process, CompactPid),
74    etap:diag("Waiting for replica group compaction to finish"),
75    receive
76    {'DOWN', Ref, process, CompactPid, normal} ->
77        ok;
78    {'DOWN', Ref, process, CompactPid, noproc} ->
79        ok;
80    {'DOWN', Ref, process, CompactPid, Reason} ->
81        etap:bail("Failure compacting replica group: " ++
82            couch_util:to_list(Reason))
83    after ?MAX_WAIT_TIME ->
84        etap:bail("Timeout waiting for replica group compaction to finish")
85    end,
86
87    {MainGroupAfter, RepGroupAfter} = get_group_snapshots(),
88
89    etap:is(
90        MainGroupAfter#set_view_group.ref_counter,
91        MainGroupBefore#set_view_group.ref_counter,
92        "Same ref counter for main group after replica compaction"),
93    etap:is(
94        MainGroupAfter#set_view_group.fd,
95        MainGroupBefore#set_view_group.fd,
96        "Same fd for main group after replica compaction"),
97
98    etap:is(
99        is_process_alive(MainGroupBefore#set_view_group.ref_counter),
100        true,
101        "Main group's ref counter still alive"),
102    etap:is(
103        is_process_alive(MainGroupBefore#set_view_group.fd),
104        true,
105        "Main group's fd still alive"),
106
107    etap:is(
108        couch_ref_counter:count(MainGroupAfter#set_view_group.ref_counter),
109        1,
110        "Main group's ref counter count is 1"),
111
112    etap:isnt(
113        RepGroupAfter#set_view_group.ref_counter,
114        RepGroupBefore#set_view_group.ref_counter,
115        "Different ref counter for replica group after replica compaction"),
116    etap:isnt(
117        RepGroupAfter#set_view_group.fd,
118        RepGroupBefore#set_view_group.fd,
119        "Different fd for replica group after replica compaction"),
120
121    etap:is(
122        is_process_alive(RepGroupBefore#set_view_group.ref_counter),
123        false,
124        "Old replica group ref counter is dead"),
125
126    etap:is(
127        is_process_alive(RepGroupBefore#set_view_group.fd),
128        false,
129        "Old replica group fd is dead"),
130
131    etap:is(
132        couch_ref_counter:count(RepGroupAfter#set_view_group.ref_counter),
133        1,
134        "Replica group's new ref counter count is 1"),
135
136    RepGroupInfo = get_replica_group_info(),
137    {Stats} = couch_util:get_value(stats, RepGroupInfo),
138    etap:is(couch_util:get_value(compactions, Stats), 1,
139        "Replica had 1 full compaction in stats"),
140    etap:is(couch_util:get_value(cleanups, Stats), 1,
141        "Replica had 1 full cleanup in stats"),
142    verify_group_info_after_replica_compact(),
143
144    DiskSizeAfter = replica_index_disk_size(),
145    etap:is(DiskSizeAfter < DiskSizeBefore, true,
146        "Index file size is smaller after compaction"),
147
148    couch_set_view_test_util:delete_set_dbs(
149        test_set_name(), num_set_partitions()),
150    ok = timer:sleep(1000),
151    couch_set_view_test_util:stop_server(),
152    ok.
153
154
155get_group_snapshots() ->
156    GroupPid = couch_set_view:get_group_pid(
157        spatial_view, test_set_name(), ddoc_id(), prod),
158    {ok, MainGroup, 0} = gen_server:call(
159        GroupPid,
160        #set_view_group_req{stale = false, debug = true},
161        infinity),
162    {ok, RepGroup, 0} = gen_server:call(
163        MainGroup#set_view_group.replica_pid,
164        #set_view_group_req{stale = false, debug = true},
165        infinity),
166    couch_ref_counter:drop(MainGroup#set_view_group.ref_counter),
167    couch_ref_counter:drop(RepGroup#set_view_group.ref_counter),
168    {MainGroup, RepGroup}.
169
170
171verify_group_info_before_replica_removal() ->
172    etap:diag(
173        "Verifying replica group info before removing replica partitions"),
174    RepGroupInfo = get_replica_group_info(),
175    etap:is(
176        couch_util:get_value(active_partitions, RepGroupInfo),
177        [],
178        "Replica group has [ ] as active partitions"),
179    etap:is(
180        couch_util:get_value(passive_partitions, RepGroupInfo),
181        lists:seq(8, 63),
182        "Replica group has [ 8 .. 63 ] as passive partitions"),
183    etap:is(
184        couch_util:get_value(cleanup_partitions, RepGroupInfo),
185        [],
186        "Replica group has [ ] as cleanup partitions").
187
188
189verify_group_info_after_replica_removal() ->
190    etap:diag(
191        "Verifying replica group info after removing replica partitions"),
192    RepGroupInfo = get_replica_group_info(),
193    etap:is(
194        couch_util:get_value(active_partitions, RepGroupInfo),
195        [],
196        "Replica group has [ ] as active partitions"),
197    etap:is(
198        couch_util:get_value(passive_partitions, RepGroupInfo),
199        [],
200        "Replica group has [ ] as passive partitions"),
201    CleanupParts = couch_util:get_value(cleanup_partitions, RepGroupInfo),
202    {Stats} = couch_util:get_value(stats, RepGroupInfo),
203    CleanupHist = couch_util:get_value(cleanup_history, Stats),
204    case length(CleanupHist) > 0 of
205    true ->
206        etap:is(
207            length(CleanupParts),
208            0,
209            "Replica group has a right value for cleanup partitions");
210    false ->
211        etap:is(
212            length(CleanupParts) > 0,
213            true,
214           "Replica group has a right value for cleanup partitions")
215    end,
216    etap:is(
217        ordsets:intersection(CleanupParts, lists:seq(0, 7)),
218        [],
219        "Replica group doesn't have any cleanup partition with ID in "
220        "[ 0 .. 7 ]").
221
222
223verify_group_info_after_replica_compact() ->
224    etap:diag("Verifying replica group info after compaction"),
225    RepGroupInfo = get_replica_group_info(),
226    etap:is(
227        couch_util:get_value(active_partitions, RepGroupInfo),
228        [],
229        "Replica group has [ ] as active partitions"),
230    etap:is(
231        couch_util:get_value(passive_partitions, RepGroupInfo),
232        [],
233        "Replica group has [ ] as passive partitions"),
234    etap:is(
235        couch_util:get_value(cleanup_partitions, RepGroupInfo),
236        [],
237        "Replica group has [ ] as cleanup partitions").
238
239
240wait_for_replica_full_update() ->
241    etap:diag("Waiting for a full replica group update"),
242    UpdateCountBefore = get_replica_updates_count(),
243    MainGroupPid = couch_set_view:get_group_pid(
244        spatial_view, test_set_name(), ddoc_id(), prod),
245    {ok, ReplicaGroupPid} = gen_server:call(
246        MainGroupPid, replica_pid, infinity),
247    {ok, UpPid} = gen_server:call(
248        ReplicaGroupPid, {start_updater, []}, infinity),
249    case is_pid(UpPid) of
250    true ->
251        ok;
252    false ->
253        etap:bail("Updater was not triggered")
254    end,
255    Ref = erlang:monitor(process, UpPid),
256    receive
257    {'DOWN', Ref, process, UpPid, {updater_finished, _}} ->
258        ok;
259    {'DOWN', Ref, process, UpPid, noproc} ->
260        ok;
261    {'DOWN', Ref, process, UpPid, Reason} ->
262        etap:bail(
263            "Failure updating replica group: " ++ couch_util:to_list(Reason))
264    after ?MAX_WAIT_TIME ->
265        etap:bail("Timeout waiting for replica group update")
266    end,
267    UpdateCountAfter = get_replica_updates_count(),
268    case UpdateCountAfter == (UpdateCountBefore + 1) of
269    true ->
270        ok;
271    false ->
272        etap:bail("Updater was not triggered")
273    end.
274
275
276get_replica_updates_count() ->
277    get_replica_updates_count(get_replica_group_info()).
278
279
280get_replica_updates_count(RepGroupInfo) ->
281    {Stats} = couch_util:get_value(stats, RepGroupInfo),
282    Updates = couch_util:get_value(full_updates, Stats),
283    true = is_integer(Updates),
284    Updates.
285
286
287get_replica_group_info() ->
288    {ok, MainInfo} = couch_set_view:get_group_info(
289        spatial_view, test_set_name(), ddoc_id(), prod),
290    {RepInfo} = couch_util:get_value(replica_group_info, MainInfo),
291    RepInfo.
292
293
294replica_index_disk_size() ->
295    Info = get_replica_group_info(),
296    Size = couch_util:get_value(disk_size, Info),
297    true = is_integer(Size),
298    true = (Size >= 0),
299    Size.
300
301
302create_docs(From, To) ->
303    rand:seed(exrop, {91, 1, 11}),
304    lists:map(
305        fun(I) ->
306            RandomMin = rand:uniform(2000),
307            RandomMax = RandomMin + rand:uniform(167),
308            RandomMin2 = rand:uniform(1769),
309            RandomMax2 = RandomMin2 + rand:uniform(132),
310            DocId = iolist_to_binary(["doc", integer_to_list(I)]),
311            {[
312              {<<"meta">>, {[{<<"id">>, DocId}]}},
313              {<<"json">>, {[
314                             {<<"value">>, I},
315                             {<<"min">>, RandomMin},
316                             {<<"max">>, RandomMax},
317                             {<<"min2">>, RandomMin2},
318                             {<<"max2">>, RandomMax2}
319                            ]}}
320            ]}
321        end,
322        lists:seq(From, To)).
323
324
325populate_set() ->
326    couch_set_view:cleanup_index_files(spatial_view, test_set_name()),
327    etap:diag("Populating the " ++ integer_to_list(num_set_partitions()) ++
328        " databases with " ++ integer_to_list(num_docs()) ++ " documents"),
329    DDoc = {[
330        {<<"meta">>, {[{<<"id">>, ddoc_id()}]}},
331        {<<"json">>, {[
332            {<<"spatial">>, {[
333                {<<"test">>, <<"function(doc, meta) { "
334                    "emit([[doc.min, doc.max], [doc.min2, doc.max2]], "
335                    "'val'+doc.value); }">>}
336            ]}}
337        ]}}
338    ]},
339    ok = couch_set_view_test_util:update_ddoc(test_set_name(), DDoc),
340    DocList = create_docs(1, num_docs()),
341    ok = couch_set_view_test_util:populate_set_sequentially(
342        test_set_name(),
343        lists:seq(0, num_set_partitions() - 1),
344        DocList),
345    etap:diag("Configuring set view with partitions [0 .. 7] as active"),
346    Params = #set_view_params{
347        max_partitions = num_set_partitions(),
348        active_partitions = lists:seq(0, 7),
349        passive_partitions = [],
350        use_replica_index = true
351    },
352    ok = couch_set_view:define_group(
353           spatial_view, test_set_name(), ddoc_id(), Params).
354