1#!/usr/bin/env escript
2%% -*- Mode: Erlang; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
3%%! -smp enable
4
5% Licensed under the Apache License, Version 2.0 (the "License"); you may not
6% use this file except in compliance with the License. You may obtain a copy of
7% the License at
8%
9%   http://www.apache.org/licenses/LICENSE-2.0
10%
11% Unless required by applicable law or agreed to in writing, software
12% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14% License for the specific language governing permissions and limitations under
15% the License.
16
17
18-include_lib("couch_set_view/include/couch_set_view.hrl").
19
20test_set_name() -> <<"couch_test_set_index_cleanup">>.
21num_set_partitions() -> 4.
22ddoc_id() -> <<"_design/test">>.
23ddoc_id_copy() -> <<"_design/test_copy">>.
24num_docs() -> 1000.
25
26
27main(_) ->
28    test_util:init_code_path(),
29
30    etap:plan(73),
31    case (catch test()) of
32        ok ->
33            etap:end_tests();
34        Other ->
35            etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
36            etap:bail(Other)
37    end,
38    ok.
39
40
41test() ->
42    couch_set_view_test_util:start_server(test_set_name()),
43
44    couch_set_view_test_util:delete_set_dbs(test_set_name(), num_set_partitions()),
45    couch_set_view_test_util:create_set_dbs(test_set_name(), num_set_partitions()),
46
47    ok = populate_set(),
48
49    GroupPid = couch_set_view:get_group_pid(
50        mapreduce_view, test_set_name(), ddoc_id(), prod),
51    query_view(ddoc_id(), num_docs(), []),
52    etap:is(is_process_alive(GroupPid), true, "Group alive after query"),
53    GroupSig = get_group_sig(),
54    IndexFile = "main_" ++ binary_to_list(GroupSig) ++ ".view.1",
55
56    etap:is(all_index_files(), [IndexFile], "Index file found"),
57
58    RawGroupSig = get_raw_sig(ddoc_id()),
59    etap:is(ets:lookup(mapreduce_view:name_to_sig_ets(prod), test_set_name()),
60            [{test_set_name(), {ddoc_id(), RawGroupSig}}],
61            "Correct group entry in ?SET_VIEW_NAME_TO_SIG_ETS_PROD ets table"),
62    etap:is(ets:lookup(mapreduce_view:sig_to_pid_ets(prod),
63                {test_set_name(), RawGroupSig}),
64            [{{test_set_name(), RawGroupSig}, GroupPid}],
65            "Correct group entry in ?SET_VIEW_SIG_TO_PID_ETS_PROD ets table"),
66
67    ok = update_ddoc(ddoc_id()),
68    ok = timer:sleep(1000),
69    NewGroupPid = couch_set_view:get_group_pid(
70        mapreduce_view, test_set_name(), ddoc_id(), prod),
71    etap:isnt(NewGroupPid, GroupPid, "Got new group after ddoc update"),
72
73    OldGroupMon = erlang:monitor(process, GroupPid),
74    receive
75    {'DOWN', OldGroupMon, _, _, _} ->
76        etap:diag("Old group shutdown after ddoc update")
77    after 30000 ->
78        etap:bail("Old group didn't shutdown after ddoc update")
79    end,
80
81    etap:is(is_process_alive(NewGroupPid), true, "New group alive before query"),
82    NewGroupSig = get_group_sig(),
83    etap:isnt(NewGroupSig, GroupSig, "New group has a different signature"),
84
85    RawNewGroupSig = get_raw_sig(ddoc_id()),
86    etap:is(ets:lookup(mapreduce_view:name_to_sig_ets(prod), test_set_name()),
87            [{test_set_name(), {ddoc_id(), RawNewGroupSig}}],
88            "Correct group entry in ?SET_VIEW_NAME_TO_SIG_ETS_PROD ets table"),
89    etap:is(ets:lookup(mapreduce_view:sig_to_pid_ets(prod),
90                {test_set_name(), RawNewGroupSig}),
91            [{{test_set_name(), RawNewGroupSig}, NewGroupPid}],
92            "Correct group entry in ?SET_VIEW_SIG_TO_PID_ETS_PROD ets table"),
93    etap:is(ets:lookup(mapreduce_view:sig_to_pid_ets(prod),
94                {test_set_name(), RawGroupSig}),
95            [],
96            "Old group entry not in ?SET_VIEW_SIG_TO_PID_ETS_PROD ets table anymore"),
97
98    NewIndexFile = "main_" ++ binary_to_list(NewGroupSig) ++ ".view.1",
99    AllIndexFiles = all_index_files(),
100    etap:is(lists:member(NewIndexFile, AllIndexFiles), true,
101        "New index file found"),
102    etap:is(lists:member(IndexFile, AllIndexFiles), false,
103        "Old index file deleted before cleanup because group was updated"),
104
105    couch_util:shutdown_sync(NewGroupPid),
106    % Let couch_set_view process group process EXIT message
107    ok = timer:sleep(1000),
108    NewGroupPid2 = couch_set_view:get_group_pid(
109        mapreduce_view, test_set_name(), ddoc_id(), prod),
110    etap:isnt(NewGroupPid2, NewGroupPid, "Got different group pid"),
111    AllIndexFiles2 = all_index_files(),
112    etap:is(lists:member(NewIndexFile, AllIndexFiles2), true,
113        "New index file found after group process restart"),
114
115    etap:diag("Performing view cleanup"),
116    couch_set_view:cleanup_index_files(mapreduce_view, test_set_name()),
117    NewAllIndexFiles = all_index_files(),
118    etap:is(lists:member(NewIndexFile, NewAllIndexFiles), true,
119        "New index file found after cleanup"),
120    etap:is(lists:member(IndexFile, NewAllIndexFiles), false,
121        "Old index file deleted after cleanup"),
122
123    query_view(ddoc_id(), 0, "stale=ok"),
124    etap:is(is_process_alive(NewGroupPid2), true,
125        "New group alive after query with ?stale=ok"),
126
127    query_view(ddoc_id(), num_docs(), []),
128    etap:is(is_process_alive(NewGroupPid2), true,
129        "New group alive after query without ?stale=ok"),
130
131    etap:diag("Creating ddoc copy with different _id"),
132    ok = create_ddoc_copy(ddoc_id_copy()),
133    ok = timer:sleep(1000),
134
135    RawNewGroupCopySig = get_raw_sig(ddoc_id_copy()),
136    etap:is(RawNewGroupCopySig, RawNewGroupSig, "Group copy has same signature"),
137    etap:is(ets:lookup(mapreduce_view:name_to_sig_ets(prod), test_set_name()),
138            [{test_set_name(), {ddoc_id(), RawNewGroupSig}},
139             {test_set_name(), {ddoc_id_copy(), RawNewGroupCopySig}}],
140            "Correct group entries in ?SET_VIEW_NAME_TO_SIG_ETS_PROD ets table"),
141    etap:is(ets:lookup(mapreduce_view:sig_to_pid_ets(prod),
142                {test_set_name(), RawNewGroupSig}),
143            [{{test_set_name(), RawNewGroupSig}, NewGroupPid2}],
144            "Correct group entry in ?SET_VIEW_SIG_TO_PID_ETS_PROD ets table"),
145
146    etap:diag("Deleting original ddoc"),
147    ok = couch_set_view_test_util:delete_ddoc(test_set_name(), ddoc_id()),
148
149    GroupMon = erlang:monitor(process, NewGroupPid2),
150    receive
151    {'DOWN', GroupMon, _, _, _} ->
152        etap:diag("New group shutdown after ddoc deleted")
153    after 30000 ->
154        etap:bail("New group didn't shutdown after ddoc was deleted")
155    end,
156    % Let couch_set_view have some time to process the group's down message
157    ok = timer:sleep(1500),
158
159    AllIndexFiles3 = all_index_files(),
160    etap:is(lists:member(NewIndexFile, AllIndexFiles3), true,
161        "Index file found after deleting original ddoc"),
162
163    etap:is(ets:lookup(mapreduce_view:name_to_sig_ets(prod), test_set_name()),
164            [],
165            "No group entry in ?SET_VIEW_NAME_TO_SIG_ETS_PROD ets table"),
166    etap:is(ets:lookup(mapreduce_view:sig_to_pid_ets(prod),
167                {test_set_name(), RawNewGroupSig}),
168            [],
169            "No group entry in ?SET_VIEW_SIG_TO_PID_ETS_PROD ets table"),
170
171    NewGroupPid3 = couch_set_view:get_group_pid(
172        mapreduce_view, test_set_name(), ddoc_id_copy(), prod),
173    etap:isnt(NewGroupPid3, NewGroupPid2, "New group pid after deleting original ddoc"),
174    query_view(ddoc_id_copy(), num_docs(), "stale=ok"),
175    etap:diag("Got empty results after deleting original ddoc and querying "
176              "ddoc copy with ?stale=ok"),
177    etap:is(is_process_alive(NewGroupPid3), true,
178        "New group copy alive after query with ?stale=ok"),
179
180    etap:is(ets:lookup(mapreduce_view:name_to_sig_ets(prod), test_set_name()),
181            [{test_set_name(), {ddoc_id_copy(), RawNewGroupSig}}],
182            "New group entry in ?SET_VIEW_NAME_TO_SIG_ETS_PROD ets table"),
183    etap:is(ets:lookup(mapreduce_view:sig_to_pid_ets(prod),
184                {test_set_name(), RawNewGroupSig}),
185            [{{test_set_name(), RawNewGroupSig}, NewGroupPid3}],
186            "New group entry in ?SET_VIEW_SIG_TO_PID_ETS_PROD ets table"),
187
188    etap:diag("Deleting ddoc copy"),
189    ok = couch_set_view_test_util:delete_ddoc(test_set_name(), ddoc_id_copy()),
190
191    etap:diag("Performing view cleanup"),
192    couch_set_view:cleanup_index_files(mapreduce_view, test_set_name()),
193    etap:is(all_index_files(), [], "0 index files after ddoc deleted and cleanup"),
194
195    % Let couch_set_view have some time to process the group's down message
196    ok = timer:sleep(1500),
197
198    etap:is(ets:lookup(mapreduce_view:name_to_sig_ets(prod), test_set_name()),
199            [],
200            "No group entry in ?SET_VIEW_NAME_TO_SIG_ETS_PROD ets table"),
201    etap:is(ets:lookup(mapreduce_view:sig_to_pid_ets(prod),
202                {test_set_name(), RawNewGroupSig}),
203            [],
204            "No group entry in ?SET_VIEW_SIG_TO_PID_ETS_PROD ets table"),
205
206    test_recreate_ddoc_with_copy(),
207
208    couch_set_view_test_util:delete_set_dbs(test_set_name(), num_set_partitions()),
209
210    couch_set_view_test_util:stop_server(),
211    ok.
212
213
214test_recreate_ddoc_with_copy() ->
215    etap:diag("Recreating design doc with a copy"),
216
217    update_ddoc(ddoc_id()),
218    GroupPid = couch_set_view:get_group_pid(
219        mapreduce_view, test_set_name(), ddoc_id(), prod),
220    RawGroupSig = get_raw_sig(ddoc_id()),
221
222    etap:is(ets:lookup(mapreduce_view:name_to_sig_ets(prod), test_set_name()),
223            [{test_set_name(), {ddoc_id(), RawGroupSig}}],
224            "Correct group entry in ?SET_VIEW_NAME_TO_SIG_ETS_PROD ets table"),
225    etap:is(ets:lookup(mapreduce_view:sig_to_pid_ets(prod),
226                {test_set_name(), RawGroupSig}),
227            [{{test_set_name(), RawGroupSig}, GroupPid}],
228            "Correct group entry in ?SET_VIEW_SIG_TO_PID_ETS_PROD ets table"),
229
230    query_view(ddoc_id(), num_docs(), []),
231
232    etap:diag("Creating ddoc copy"),
233    ok = create_ddoc_copy(ddoc_id_copy()),
234
235    GroupPidCopy = couch_set_view:get_group_pid(
236        mapreduce_view, test_set_name(), ddoc_id_copy(), prod),
237    etap:is(GroupPidCopy, GroupPid, "DDoc copy has same group pid"),
238    RawGroupSigCopy = get_raw_sig(ddoc_id_copy()),
239    etap:is(RawGroupSigCopy, RawGroupSig, "DDoc copy has same signature"),
240
241    query_view(ddoc_id_copy(), num_docs(), "stale=ok"),
242
243    etap:is(ets:lookup(mapreduce_view:name_to_sig_ets(prod), test_set_name()),
244            [{test_set_name(), {ddoc_id(), RawGroupSig}},
245             {test_set_name(), {ddoc_id_copy(), RawGroupSig}}],
246            "Correct group entries in ?SET_VIEW_NAME_TO_SIG_ETS_PROD ets table"),
247    etap:is(ets:lookup(mapreduce_view:sig_to_pid_ets(prod),
248                {test_set_name(), RawGroupSig}),
249            [{{test_set_name(), RawGroupSig}, GroupPid}],
250            "Correct group entry in ?SET_VIEW_SIG_TO_PID_ETS_PROD ets table"),
251
252    etap:diag("Deleting ddoc copy"),
253    ok = couch_set_view_test_util:delete_ddoc(test_set_name(), ddoc_id_copy()),
254    ok = timer:sleep(1000),
255
256    etap:is(ets:lookup(mapreduce_view:name_to_sig_ets(prod), test_set_name()),
257            [],
258            "No group entry in ?SET_VIEW_NAME_TO_SIG_ETS_PROD ets table"),
259    etap:is(ets:lookup(mapreduce_view:sig_to_pid_ets(prod),
260                {test_set_name(), RawGroupSig}),
261            [],
262            "No group entry in ?SET_VIEW_SIG_TO_PID_ETS_PROD ets table"),
263
264    query_view(ddoc_id(), num_docs(), "stale=ok"),
265
266    etap:diag("Deleting original ddoc"),
267    ok = couch_set_view_test_util:delete_ddoc(test_set_name(), ddoc_id()),
268    % Let couch_set_view have some time to process the group's down message
269    ok = timer:sleep(1500),
270
271    etap:is(ets:lookup(mapreduce_view:name_to_sig_ets(prod), test_set_name()),
272            [],
273            "No group entry in ?SET_VIEW_NAME_TO_SIG_ETS_PROD ets table"),
274    etap:is(ets:lookup(mapreduce_view:sig_to_pid_ets(prod),
275                {test_set_name(), RawGroupSig}),
276            [],
277            "No group entry in ?SET_VIEW_SIG_TO_PID_ETS_PROD ets table"),
278
279    couch_set_view:cleanup_index_files(mapreduce_view, test_set_name()),
280
281    etap:diag("Creating original ddoc again"),
282    update_ddoc(ddoc_id()),
283    GroupPid2 = couch_set_view:get_group_pid(
284       mapreduce_view, test_set_name(), ddoc_id(), prod),
285    query_view(ddoc_id(), num_docs(), []),
286
287    etap:diag("Creating ddoc copy again"),
288    ok = create_ddoc_copy(ddoc_id_copy()),
289
290    etap:is(ets:lookup(mapreduce_view:name_to_sig_ets(prod), test_set_name()),
291            [{test_set_name(), {ddoc_id(), RawGroupSig}},
292             {test_set_name(), {ddoc_id_copy(), RawGroupSig}}],
293            "Correct group entries in ?SET_VIEW_NAME_TO_SIG_ETS_PROD ets table"),
294    etap:is(ets:lookup(mapreduce_view:sig_to_pid_ets(prod),
295                {test_set_name(), RawGroupSig}),
296            [{{test_set_name(), RawGroupSig}, GroupPid2}],
297            "Correct group entry in ?SET_VIEW_SIG_TO_PID_ETS_PROD ets table"),
298
299    etap:diag("Killing view group process"),
300    couch_util:shutdown_sync(GroupPid2),
301    ok = timer:sleep(1000),
302
303    etap:is(ets:lookup(mapreduce_view:name_to_sig_ets(prod), test_set_name()),
304            [],
305            "?SET_VIEW_NAME_TO_SIG_ETS_PROD ets table is empty"),
306    etap:is(ets:lookup(mapreduce_view:sig_to_pid_ets(prod),
307                {test_set_name(), RawGroupSig}),
308            [],
309            "?SET_VIEW_SIG_TO_PID_ETS_PROD ets table is empty"),
310
311    etap:diag("Starting again view group process"),
312    GroupPid3 = couch_set_view:get_group_pid(
313       mapreduce_view, test_set_name(), ddoc_id(), prod),
314    etap:isnt(GroupPid3, GroupPid2, "Got a different view group pid"),
315
316    etap:is(ets:lookup(mapreduce_view:name_to_sig_ets(prod), test_set_name()),
317            [{test_set_name(), {ddoc_id(), RawGroupSig}},
318             {test_set_name(), {ddoc_id_copy(), RawGroupSig}}],
319            "Correct group entries in ?SET_VIEW_NAME_TO_SIG_ETS_PROD ets table"),
320    etap:is(ets:lookup(mapreduce_view:sig_to_pid_ets(prod),
321                {test_set_name(), RawGroupSig}),
322            [{{test_set_name(), RawGroupSig}, GroupPid3}],
323            "Correct group entry in ?SET_VIEW_SIG_TO_PID_ETS_PROD ets table"),
324
325    etap:diag("Deleting master database"),
326    couch_set_view_test_util:delete_set_db(test_set_name(), master),
327    ok = timer:sleep(1000),
328
329    etap:is(ets:lookup(mapreduce_view:name_to_sig_ets(prod), test_set_name()),
330            [],
331            "?SET_VIEW_NAME_TO_SIG_ETS_PROD ets table is empty"),
332    etap:is(ets:lookup(mapreduce_view:sig_to_pid_ets(prod),
333                {test_set_name(), RawGroupSig}),
334            [],
335            "?SET_VIEW_SIG_TO_PID_ETS_PROD ets table is empty"),
336
337    etap:is(ets:tab2list(set_view_by_ddoc_id_ets), [], "ddoc cache is empty"),
338
339    etap:diag("Recreating database set"),
340    couch_set_view_test_util:delete_set_dbs(test_set_name(), num_set_partitions()),
341    couch_set_view_test_util:create_set_dbs(test_set_name(), num_set_partitions()),
342
343    etap:diag("Adding design document again, but not opening its view group"),
344    ok = couch_set_view_test_util:update_ddoc(test_set_name(), ddoc(ddoc_id())),
345
346    etap:is(ets:lookup(mapreduce_view:name_to_sig_ets(prod), test_set_name()),
347            [{test_set_name(), {ddoc_id(), RawGroupSig}}],
348            "Correct alias in ?SET_VIEW_NAME_TO_SIG_ETS_PROD ets table"),
349    etap:is(ets:lookup(mapreduce_view:sig_to_pid_ets(prod),
350                {test_set_name(), RawGroupSig}),
351            [],
352            "?SET_VIEW_SIG_TO_PID_ETS_PROD ets table is empty"),
353
354    ViewManagerPid = couch_set_view_test_util:get_daemon_pid(set_view_manager),
355
356    etap:diag("Deleting master database"),
357    couch_set_view_test_util:delete_set_db(test_set_name(), master),
358    ok = timer:sleep(1000),
359
360    etap:is(is_process_alive(ViewManagerPid), true, "View manager didn't die"),
361    etap:is(ets:lookup(mapreduce_view:name_to_sig_ets(prod), test_set_name()),
362            [],
363            "?SET_VIEW_NAME_TO_SIG_ETS_PROD ets table is empty"),
364    etap:is(ets:lookup(mapreduce_view:sig_to_pid_ets(prod),
365                {test_set_name(), RawGroupSig}),
366            [],
367            "?SET_VIEW_SIG_TO_PID_ETS_PROD ets table is empty"),
368
369    etap:is(ets:tab2list(set_view_by_ddoc_id_ets), [], "ddoc cache is empty"),
370    ok.
371
372
373query_view(DDocId, ExpectedRowCount, QueryString) ->
374    {ok, {ViewResults}} = couch_set_view_test_util:query_view(
375        test_set_name(), DDocId, <<"test">>, QueryString),
376    etap:is(
377        length(couch_util:get_value(<<"rows">>, ViewResults)),
378        ExpectedRowCount,
379        "Got " ++ integer_to_list(ExpectedRowCount) ++ " view rows"),
380    SortedKeys =  couch_set_view_test_util:are_view_keys_sorted(
381        {ViewResults}, fun(A, B) -> A < B end),
382    etap:is(SortedKeys, true, "View result keys are sorted").
383
384
385populate_set() ->
386    DDoc = {[
387        {<<"meta">>, {[{<<"id">>, ddoc_id()}]}},
388        {<<"json">>, {[
389            {<<"language">>, <<"javascript">>},
390            {<<"views">>, {[
391                {<<"test">>, {[
392                    {<<"map">>, <<"function(doc, meta) { emit(doc.value, meta.id); }">>}
393                ]}}
394            ]}}
395        ]}}
396    ]},
397    ok = couch_set_view_test_util:update_ddoc(test_set_name(), DDoc),
398    DocList = lists:map(
399        fun(I) ->
400            {[
401                {<<"meta">>, {[{<<"id">>, iolist_to_binary(["doc", integer_to_list(I)])}]}},
402                {<<"json">>, {[
403                    {<<"value">>, I}
404                    ]}}
405            ]}
406        end,
407        lists:seq(1, num_docs())),
408    ok = couch_set_view_test_util:populate_set_alternated(
409        test_set_name(),
410        lists:seq(0, num_set_partitions() - 1),
411        DocList),
412    ok = couch_set_view_test_util:define_set_view(
413        test_set_name(),
414        ddoc_id(),
415        num_set_partitions(),
416        lists:seq(0, num_set_partitions() - 1),
417        []),
418    ok.
419
420
421ddoc(Id) ->
422    {[
423        {<<"meta">>, {[{<<"id">>, Id}]}},
424        {<<"json">>, {[
425            {<<"views">>, {[
426                {<<"test">>, {[
427                    {<<"map">>, <<"function(doc, meta) { emit(doc.value, null); }">>}
428                ]}}
429            ]}}
430        ]}}
431    ]}.
432
433
434update_ddoc(DDocId) ->
435    NewDDoc = ddoc(DDocId),
436    ok = couch_set_view_test_util:update_ddoc(test_set_name(), NewDDoc),
437    ok = couch_set_view_test_util:define_set_view(
438        test_set_name(),
439        ddoc_id(),
440        num_set_partitions(),
441        lists:seq(0, num_set_partitions() - 1),
442        []),
443    ok.
444
445
446create_ddoc_copy(CopyId) ->
447    DDoc = ddoc(CopyId),
448    ok = couch_set_view_test_util:update_ddoc(test_set_name(), DDoc).
449
450
451get_group_sig() ->
452    {ok, Info} = couch_set_view:get_group_info(
453       mapreduce_view, test_set_name(), ddoc_id(), prod),
454    couch_util:get_value(signature, Info).
455
456
457get_raw_sig(DDocId) ->
458    Pid = couch_set_view:get_group_pid(
459       mapreduce_view, test_set_name(), DDocId, prod),
460    {ok, Sig} = gen_server:call(Pid, get_sig, infinity),
461    Sig.
462
463
464all_index_files() ->
465    IndexDir = couch_set_view:set_index_dir(
466        couch_config:get("couchdb", "view_index_dir"), test_set_name(), prod),
467    filelib:fold_files(
468        IndexDir, ".*\\.view\\.[0-9]+$", false,
469        fun(N, A) -> [filename:basename(N) | A] end, []).
470