1#!/usr/bin/env escript
2%% -*- erlang -*-
3%%! -pa ./src/couchdb -sasl errlog_type error -noshell -smp enable
4
5% Licensed under the Apache License, Version 2.0 (the "License"); you may not
6% use this file except in compliance with the License. You may obtain a copy of
7% the License at
8%
9%   http://www.apache.org/licenses/LICENSE-2.0
10%
11% Unless required by applicable law or agreed to in writing, software
12% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14% License for the specific language governing permissions and limitations under
15% the License.
16
17filename() -> "./test/etap/temp.023".
18
19
20main(_) ->
21    test_util:init_code_path(),
22    etap:plan(27),
23    case (catch test()) of
24        ok ->
25            etap:end_tests();
26        Other ->
27            etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
28            etap:bail()
29    end,
30    ok.
31
32
33test() ->
34    couch_file_write_guard:sup_start_link(),
35    no_purged_items_test(),
36    all_purged_items_test(),
37    partial_purges_test(),
38    partial_purges_test_2(),
39    partial_purges_test_with_stop(),
40    add_remove_and_purge_test(),
41    ok.
42
43
44no_purged_items_test() ->
45    ReduceFun = fun
46        (reduce, KVs) -> length(KVs);
47        (rereduce, Reds) -> lists:sum(Reds)
48    end,
49
50    {ok, Fd} = couch_file:open(filename(), [create, overwrite]),
51    {ok, Btree} = couch_btree:open(nil, Fd, [
52        {reduce, ReduceFun},
53        {kv_chunk_threshold, 6 * 1024},
54        {kp_chunk_threshold, 6 * 1024}
55    ]),
56
57    N = 211341,
58    KVs = [{I, I} || I <- lists:seq(1, N)],
59    {ok, Btree2} = couch_btree:add_remove(Btree, KVs, []),
60    ok = couch_file:flush(Fd),
61
62    {ok, Red} = couch_btree:full_reduce(Btree2),
63    etap:is(Red, N, "Initial reduce value equals N"),
64
65    PurgeFun = fun
66        (value, _, Acc) ->
67            {keep, Acc};
68        (branch, _, Acc) ->
69            {keep, Acc}
70    end,
71    {ok, Btree3, Acc1} = couch_btree:guided_purge(Btree2, PurgeFun, []),
72    ok = couch_file:flush(Fd),
73    etap:is(Acc1, [], "guided_purge returned right accumulator"),
74    {ok, Red2} = couch_btree:full_reduce(Btree3),
75    etap:is(Red2, N, "Reduce value after guided purge equals N"),
76
77    FoldFun = fun(KV, _, Acc) ->
78        {ok, [KV | Acc]}
79    end,
80    {ok, _, KVs2} = couch_btree:foldl(Btree3, FoldFun, []),
81    etap:is(lists:reverse(KVs2), KVs, "Btree has same values after guided purge"),
82
83    couch_file:close(Fd).
84
85
86all_purged_items_test() ->
87    ReduceFun = fun
88        (reduce, KVs) -> length(KVs);
89        (rereduce, Reds) -> lists:sum(Reds)
90    end,
91
92    {ok, Fd} = couch_file:open(filename(), [create, overwrite]),
93    {ok, Btree} = couch_btree:open(nil, Fd, [
94        {reduce, ReduceFun},
95        {kv_chunk_threshold, 6 * 1024},
96        {kp_chunk_threshold, 6 * 1024}
97    ]),
98
99    N = 211341,
100    KVs = [{I, I} || I <- lists:seq(1, N)],
101    {ok, Btree2} = couch_btree:add_remove(Btree, KVs, []),
102    ok = couch_file:flush(Fd),
103
104    {ok, Red} = couch_btree:full_reduce(Btree2),
105    etap:is(Red, N, "Initial reduce value equals N"),
106
107    PurgeFun = fun
108        (value, _, {V, B}) ->
109            {purge, {V + 1, B}};
110        (branch, R, {V, B}) ->
111            {purge, {V, B + R}}
112    end,
113    {ok, Btree3, Acc1} = couch_btree:guided_purge(Btree2, PurgeFun, {0, 0}),
114    ok = couch_file:flush(Fd),
115    etap:is(Acc1, {0, N}, "guided_purge returned right accumulator - {0, N}"),
116    {ok, Red2} = couch_btree:full_reduce(Btree3),
117    etap:is(Red2, 0, "Reduce value after guided purge equals 0"),
118
119    FoldFun = fun(KV, _, Acc) ->
120        {ok, [KV | Acc]}
121    end,
122    {ok, _, KVs2} = couch_btree:foldl(Btree3, FoldFun, []),
123    etap:is(lists:reverse(KVs2), [], "Btree is empty after guided purge"),
124
125    couch_file:close(Fd).
126
127
128partial_purges_test() ->
129    ReduceFun = fun
130        (reduce, KVs) ->
131            even_odd_count(KVs);
132        (rereduce, Reds) ->
133            Even = lists:sum([E || {E, _} <- Reds]),
134            Odd = lists:sum([O || {_, O} <- Reds]),
135            {Even, Odd}
136    end,
137
138    {ok, Fd} = couch_file:open(filename(), [create, overwrite]),
139    {ok, Btree} = couch_btree:open(nil, Fd, [
140        {reduce, ReduceFun},
141        {kv_chunk_threshold, 6 * 1024},
142        {kp_chunk_threshold, 6 * 1024}
143    ]),
144
145    N = 211341,
146    KVs = [{I, I} || I <- lists:seq(1, N)],
147    {NumEven, NumOdds} = even_odd_count(KVs),
148
149    {ok, Btree2} = couch_btree:add_remove(Btree, KVs, []),
150    ok = couch_file:flush(Fd),
151
152    {ok, Red} = couch_btree:full_reduce(Btree2),
153    etap:is(Red, {NumEven, NumOdds}, "Initial reduce value equals {NumEven, NumOdd}"),
154
155    PurgeFun = fun
156        (value, {K, K}, Count) ->
157            case (K rem 2) of
158            0 ->
159                {keep, Count};
160            _ ->
161                {purge, Count + 1}
162            end;
163        (branch, {0, _OdCount}, Count) ->
164            {purge, Count};
165        (branch, {_, 0}, Count) ->
166            {keep, Count};
167        (branch, {EvCount, _OdCount}, Count) when EvCount > 0 ->
168            {partial_purge, Count}
169    end,
170    {ok, Btree3, Acc1} = couch_btree:guided_purge(Btree2, PurgeFun, 0),
171    ok = couch_file:flush(Fd),
172    etap:is(Acc1, NumOdds, "guided_purge returned right accumulator - NumOdds}"),
173    {ok, Red2} = couch_btree:full_reduce(Btree3),
174    etap:is(Red2, {NumEven, 0}, "Reduce value after guided purge equals {NumEven, 0}"),
175
176    FoldFun = fun(KV, _, Acc) ->
177        {ok, [KV | Acc]}
178    end,
179    {ok, _, KVs2} = couch_btree:foldl(Btree3, FoldFun, []),
180    lists:foreach(
181        fun({K, K}) ->
182            case (K rem 2) of
183            0 ->
184                ok;
185            _ ->
186                etap:bail("Got odd value in btree after guided purge: " ++ integer_to_list(K))
187            end
188        end,
189        KVs2),
190    etap:diag("Btree has no odd values after guided purge"),
191    etap:is(lists:reverse([K || {K, _} <- KVs2]), lists:sort([K || {K, _} <- KVs2]), "Btree keys are sorted"),
192
193    couch_file:close(Fd).
194
195
196partial_purges_test_2() ->
197    ReduceFun = fun
198        (reduce, KVs) ->
199            {length(KVs), ordsets:from_list([P || {_K, {P, _I}} <- KVs])};
200        (rereduce, Reds) ->
201            {lists:sum([C || {C, _} <- Reds]), ordsets:union([S || {_, S} <- Reds])}
202    end,
203
204    {ok, Fd} = couch_file:open(filename(), [create, overwrite]),
205    {ok, Btree} = couch_btree:open(nil, Fd, [
206        {reduce, ReduceFun},
207        {kv_chunk_threshold, 6 * 1024},
208        {kp_chunk_threshold, 6 * 1024}
209    ]),
210
211    N = 320000,
212    KVs = [{iolist_to_binary(io_lib:format("doc_~6..0b", [I])), {I rem 64, I}} || I <- lists:seq(1, N)],
213
214    {ok, Btree2} = couch_btree:add_remove(Btree, KVs, []),
215    ok = couch_file:flush(Fd),
216
217    {ok, Red} = couch_btree:full_reduce(Btree2),
218    etap:is(Red, {N, lists:seq(0, 63)}, "Initial reduce value equals {N, lists:seq(0, 63)}"),
219
220    PurgeFun = fun
221        (value, {_K, {P, _V}}, Count) ->
222            case P >= 32 of
223            true ->
224                {purge, Count + 1};
225            false ->
226                {keep, Count}
227            end;
228        (branch, {C, S}, Count) ->
229            RemSet = lists:seq(32, 63),
230            case ordsets:intersection(S, RemSet) of
231            [] ->
232                {keep, Count};
233            S ->
234                {purge, Count + C};
235            _ ->
236                {partial_purge, Count}
237            end
238    end,
239    {ok, Btree3, Acc1} = couch_btree:guided_purge(Btree2, PurgeFun, 0),
240    ok = couch_file:flush(Fd),
241    etap:is(Acc1, N div 2, "guided_purge returned right accumulator - N div 2"),
242    {ok, Red2} = couch_btree:full_reduce(Btree3),
243    etap:is(Red2, {N div 2, lists:seq(0, 31)}, "Reduce value after guided purge equals {N div 2, lists:seq(0, 31)}"),
244
245    FoldFun = fun(KV, _, Acc) ->
246        {ok, [KV | Acc]}
247    end,
248    {ok, _, KVs2} = couch_btree:foldl(Btree3, FoldFun, []),
249    lists:foreach(
250        fun({_K, {P, V}}) ->
251            case ordsets:is_element(P, lists:seq(0, 31)) of
252            true ->
253                ok;
254            false ->
255                etap:bail("Got value outside the range [0 .. 31]: " ++ integer_to_list(V))
256            end
257        end,
258        KVs2),
259    etap:diag("Btree has no values outside the range [0 .. 31]"),
260    etap:is(lists:reverse([K || {K, _} <- KVs2]), lists:sort([K || {K, _} <- KVs2]), "Btree keys are sorted"),
261
262    couch_file:close(Fd).
263
264
265partial_purges_test_with_stop() ->
266    ReduceFun = fun
267        (reduce, KVs) ->
268            even_odd_count(KVs);
269        (rereduce, Reds) ->
270            Even = lists:sum([E || {E, _} <- Reds]),
271            Odd = lists:sum([O || {_, O} <- Reds]),
272            {Even, Odd}
273    end,
274
275    {ok, Fd} = couch_file:open(filename(), [create, overwrite]),
276    {ok, Btree} = couch_btree:open(nil, Fd, [
277        {reduce, ReduceFun},
278        {kv_chunk_threshold, 6 * 1024},
279        {kp_chunk_threshold, 6 * 1024}
280    ]),
281
282    N = 211341,
283    KVs = [{I, I} || I <- lists:seq(1, N)],
284    {NumEven, NumOdds} = even_odd_count(KVs),
285
286    {ok, Btree2} = couch_btree:add_remove(Btree, KVs, []),
287    ok = couch_file:flush(Fd),
288
289    {ok, Red} = couch_btree:full_reduce(Btree2),
290    etap:is(Red, {NumEven, NumOdds}, "Initial reduce value equals {NumEven, NumOdd}"),
291
292    PurgeFun = fun
293        (_, _, Count) when Count >= 4 ->
294            {stop, Count};
295        (value, {K, K}, Count) ->
296            case (K rem 2) of
297            0 ->
298                {keep, Count};
299            _ ->
300                {purge, Count + 1}
301            end;
302        (branch, {0, _OdCount}, Count) ->
303            {purge, Count};
304        (branch, {_, 0}, Count) ->
305            {keep, Count};
306        (branch, {EvCount, _OdCount}, Count) when EvCount > 0 ->
307            {partial_purge, Count}
308    end,
309    {ok, Btree3, Acc1} = couch_btree:guided_purge(Btree2, PurgeFun, 0),
310    ok = couch_file:flush(Fd),
311    etap:is(Acc1, 4, "guided_purge returned right accumulator - 4}"),
312    {ok, Red2} = couch_btree:full_reduce(Btree3),
313    etap:is(Red2, {NumEven, NumOdds - 4}, "Reduce value after guided purge equals {NumEven, NumOdds - 4}"),
314
315    FoldFun = fun(KV, _, Acc) ->
316        {ok, [KV | Acc]}
317    end,
318    {ok, _, KVs2} = couch_btree:foldl(Btree3, FoldFun, []),
319    lists:foreach(
320        fun({K, K}) ->
321            case lists:member(K, [1, 3, 5, 7]) of
322            false ->
323                ok;
324            true ->
325                etap:bail("Got odd value <= 7 in btree after guided purge: " ++ integer_to_list(K))
326            end
327        end,
328        KVs2),
329    etap:diag("Btree has no odd values after guided purge"),
330    etap:is(lists:reverse([K || {K, _} <- KVs2]), lists:sort([K || {K, _} <- KVs2]), "Btree keys are sorted"),
331
332    couch_file:close(Fd).
333
334
335add_remove_and_purge_test() ->
336    ReduceFun = fun
337        (reduce, KVs) ->
338            even_odd_count(KVs);
339        (rereduce, Reds) ->
340            Even = lists:sum([E || {E, _} <- Reds]),
341            Odd = lists:sum([O || {_, O} <- Reds]),
342            {Even, Odd}
343    end,
344
345    {ok, Fd} = couch_file:open(filename(), [create, overwrite]),
346    {ok, Btree} = couch_btree:open(nil, Fd, [
347        {reduce, ReduceFun},
348        {kv_chunk_threshold, 6 * 1024},
349        {kp_chunk_threshold, 6 * 1024}
350    ]),
351
352    N = 211341,
353    KVs = [{I, I} || I <- lists:seq(1, N)],
354    {NumEven, NumOdds} = even_odd_count(KVs),
355
356    {ok, Btree2} = couch_btree:add_remove(Btree, KVs, []),
357    ok = couch_file:flush(Fd),
358
359    {ok, Red} = couch_btree:full_reduce(Btree2),
360    etap:is(Red, {NumEven, NumOdds}, "Initial reduce value equals {NumEven, NumOdd}"),
361
362    PurgeFun = fun
363        (value, {K, K}, Count) ->
364            case (K rem 2) of
365            0 ->
366                {keep, Count};
367            _ ->
368                {purge, Count + 1}
369            end;
370        (branch, {0, _OddCount}, Count) ->
371            {purge, Count};
372        (branch, {_, 0}, Count) ->
373            {keep, Count};
374        (branch, {EvCount, _OddCount}, Count) when EvCount > 0 ->
375            {partial_purge, Count}
376    end,
377    {ok, PurgeAcc, Btree3} = couch_btree:add_remove(
378        Btree2, [{2, -1}, {14006, -1}, {500000, -1}], [4, 200000, 10], PurgeFun, 0),
379    ok = couch_file:flush(Fd),
380    etap:is(PurgeAcc, NumOdds, "couch_btree:add_remove/5 returned right accumulator - NumOdds}"),
381    {ok, Red2} = couch_btree:full_reduce(Btree3),
382    etap:is(Red2, {NumEven - 2, 0}, "Reduce value after guided purge equals {NumEven - 2, 0}"),
383
384    FoldFun = fun(KV, _, Acc) ->
385        {ok, [KV | Acc]}
386    end,
387    {ok, _, KVs2} = couch_btree:foldl(Btree3, FoldFun, []),
388    lists:foreach(
389        fun({K, V}) ->
390            case (K rem 2) =:= 0 of
391            true ->
392                ok;
393            false ->
394                etap:bail("Got odd value in btree after purge: " ++ integer_to_list(K))
395            end,
396            case lists:member(K, [2, 14006, 500000]) of
397            true when V =:= -1 ->
398                ok;
399            true ->
400                etap:bail("Key " ++ integer_to_list(K) ++ " has wrong value (expected -1)");
401            false ->
402                case K =:= V of
403                true ->
404                    ok;
405                false ->
406                    etap:bail("Key " ++ integer_to_list(K) ++ " has wrong value")
407                end
408            end
409        end,
410        KVs2),
411    etap:diag("Btree has no odd values after couch_btree:add_remove/6 call"),
412    etap:is(couch_util:get_value(4, KVs2), undefined, "Key 2 not in tree anymore"),
413    etap:is(couch_util:get_value(10, KVs2), undefined, "Key 10 not in tree anymore"),
414    etap:is(couch_util:get_value(200000, KVs2), undefined, "Key 200000 not in tree anymore"),
415    etap:is(lists:reverse([K || {K, _} <- KVs2]), lists:sort([K || {K, _} <- KVs2]), "Btree keys are sorted"),
416
417    couch_file:close(Fd).
418
419
420even_odd_count(KVs) ->
421    Even = lists:sum([1 || {E, _} <- KVs, (E rem 2) =:= 0]),
422    Odd = lists:sum([1 || {O, _} <- KVs, (O rem 2) =/= 0]),
423    {Even, Odd}.
424