1/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/*
3 *     Copyright 2017 Couchbase, Inc
4 *
5 *   Licensed under the Apache License, Version 2.0 (the "License");
6 *   you may not use this file except in compliance with the License.
7 *   You may obtain a copy of the License at
8 *
9 *       http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *   Unless required by applicable law or agreed to in writing, software
12 *   distributed under the License is distributed on an "AS IS" BASIS,
13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *   See the License for the specific language governing permissions and
15 *   limitations under the License.
16 */
17
18#include "tests/module_tests/evp_store_single_threaded_test.h"
19
20class CollectionsEraserTest
21        : public SingleThreadedKVBucketTest,
22          public ::testing::WithParamInterface<std::string> {
23public:
24    void SetUp() override {
25        // Enable collections (which will enable namespace persistence).
26        config_string += "collections_prototype_enabled=true;";
27        config_string += GetParam();
28        SingleThreadedKVBucketTest::SetUp();
29        setVBucketStateAndRunPersistTask(vbid, vbucket_state_active);
30        vb = store->getVBucket(vbid);
31    }
32
33    void TearDown() override {
34        vb.reset();
35        SingleThreadedKVBucketTest::TearDown();
36    }
37
38    void runEraser() {
39        runCompaction();
40    }
41
42    bool isFullEviction() const {
43        return GetParam().find("item_eviction_policy=full_eviction") !=
44               std::string::npos;
45    }
46
47    VBucketPtr vb;
48};
49
50// Small numbers of items for easier debug
51TEST_P(CollectionsEraserTest, basic) {
52    // add a collection
53    vb->updateFromManifest({R"({"separator":":","uid":"0",)"
54                            R"("collections":[{"name":"$default", "uid":"0"},)"
55                            R"(               {"name":"dairy","uid":"1"}]})"});
56
57    flush_vbucket_to_disk(vbid, 1 /* 1 x system */);
58
59    // add some items
60    store_item(vbid, {"dairy:milk", DocNamespace::Collections}, "nice");
61    store_item(vbid, {"dairy:butter", DocNamespace::Collections}, "lovely");
62
63    flush_vbucket_to_disk(vbid, 2 /* 2 x items */);
64
65    EXPECT_EQ(2, vb->getNumItems());
66
67    // Evict one of the keys, we should still erase it
68    evict_key(vbid, {"dairy:butter", DocNamespace::Collections});
69
70    // delete the collection
71    vb->updateFromManifest(
72            {R"({"separator":":","uid":"0",)"
73             R"("collections":[{"name":"$default", "uid":"0"}]})"});
74
75    flush_vbucket_to_disk(vbid, 1 /* 1 x system */);
76
77    // Deleted, but still exists in the manifest
78    EXPECT_TRUE(vb->lockCollections().exists("dairy"));
79
80    runEraser();
81
82    EXPECT_EQ(0, vb->getNumItems());
83
84    EXPECT_FALSE(vb->lockCollections().exists("dairy"));
85}
86
87TEST_P(CollectionsEraserTest, basic_2_collections) {
88    // add a collection
89    vb->updateFromManifest({R"({"separator":":","uid":"0",)"
90                            R"("collections":[{"name":"$default", "uid":"0"},)"
91                            R"(               {"name":"fruit","uid":"1"},)"
92                            R"(               {"name":"dairy","uid":"1"}]})"});
93
94    flush_vbucket_to_disk(vbid, 2 /* 2 x system */);
95
96    // add some items
97    store_item(vbid, {"dairy:milk", DocNamespace::Collections}, "nice");
98    store_item(vbid, {"dairy:butter", DocNamespace::Collections}, "lovely");
99    store_item(vbid, {"fruit:apple", DocNamespace::Collections}, "nice");
100    store_item(vbid, {"fruit:apricot", DocNamespace::Collections}, "lovely");
101
102    flush_vbucket_to_disk(vbid, 4);
103
104    EXPECT_EQ(4, vb->getNumItems());
105
106    // delete the collections
107    vb->updateFromManifest(
108            {R"({"separator":":","uid":"0",)"
109             R"("collections":[{"name":"$default", "uid":"0"}]})"});
110
111    // Deleted, but still exists in the manifest
112    EXPECT_TRUE(vb->lockCollections().exists("dairy"));
113    EXPECT_TRUE(vb->lockCollections().exists("fruit"));
114
115    flush_vbucket_to_disk(vbid, 2 /* 2 x system */);
116
117    runEraser();
118
119    EXPECT_EQ(0, vb->getNumItems());
120
121    EXPECT_FALSE(vb->lockCollections().exists("dairy"));
122    EXPECT_FALSE(vb->lockCollections().exists("fruit"));
123}
124
125TEST_P(CollectionsEraserTest, basic_3_collections) {
126    // add a collection
127    vb->updateFromManifest({R"({"separator":":","uid":"0",)"
128                            R"("collections":[{"name":"$default", "uid":"0"},)"
129                            R"(               {"name":"fruit","uid":"1"},)"
130                            R"(               {"name":"dairy","uid":"1"}]})"});
131
132    flush_vbucket_to_disk(vbid, 2 /* 1x system */);
133
134    // add some items
135    store_item(vbid, {"dairy:milk", DocNamespace::Collections}, "nice");
136    store_item(vbid, {"dairy:butter", DocNamespace::Collections}, "lovely");
137    store_item(vbid, {"fruit:apple", DocNamespace::Collections}, "nice");
138    store_item(vbid, {"fruit:apricot", DocNamespace::Collections}, "lovely");
139
140    flush_vbucket_to_disk(vbid, 4 /* 2x items */);
141
142    EXPECT_EQ(4, vb->getNumItems());
143
144    // delete one of the 3 collections
145    vb->updateFromManifest({R"({"separator":":","uid":"0",)"
146                            R"("collections":[{"name":"$default", "uid":"0"},)"
147                            R"(               {"name":"dairy","uid":"1"}]})"});
148
149    // Deleted, but still exists in the manifest
150    EXPECT_TRUE(vb->lockCollections().exists("dairy"));
151    EXPECT_TRUE(vb->lockCollections().exists("fruit"));
152
153    flush_vbucket_to_disk(vbid, 1 /* 1 x system */);
154
155    runEraser();
156
157    EXPECT_EQ(2, vb->getNumItems());
158
159    EXPECT_TRUE(vb->lockCollections().exists("dairy"));
160    EXPECT_FALSE(vb->lockCollections().exists("fruit"));
161}
162
163TEST_P(CollectionsEraserTest, basic_4_collections) {
164    // add a collection
165    vb->updateFromManifest({R"({"separator":":","uid":"0",)"
166                            R"("collections":[{"name":"$default", "uid":"0"},)"
167                            R"(               {"name":"fruit","uid":"1"},)"
168                            R"(               {"name":"dairy","uid":"1"}]})"});
169
170    flush_vbucket_to_disk(vbid, 2 /* 1x system */);
171
172    // add some items
173    store_item(vbid, {"dairy:milk", DocNamespace::Collections}, "nice");
174    store_item(vbid, {"dairy:butter", DocNamespace::Collections}, "lovely");
175    store_item(vbid, {"fruit:apple", DocNamespace::Collections}, "nice");
176    store_item(vbid, {"fruit:apricot", DocNamespace::Collections}, "lovely");
177
178    flush_vbucket_to_disk(vbid, 4 /* 2x items */);
179
180    // delete the collection and re-add a new dairy
181    vb->updateFromManifest({R"({"separator":":","uid":"0",)"
182                            R"("collections":[{"name":"$default", "uid":"0"},)"
183                            R"(               {"name":"dairy","uid":"2"}]})"});
184
185    // Deleted, but still exists in the manifest
186    EXPECT_TRUE(vb->lockCollections().exists("dairy"));
187    EXPECT_TRUE(vb->lockCollections().exists("fruit"));
188
189    flush_vbucket_to_disk(vbid, 2 /* 1x system */);
190
191    runEraser();
192
193    EXPECT_EQ(0, vb->getNumItems());
194
195    EXPECT_TRUE(vb->lockCollections().exists("dairy"));
196    EXPECT_FALSE(vb->lockCollections().exists("fruit"));
197}
198
199TEST_P(CollectionsEraserTest, default_Destroy) {
200    // add some items
201    store_item(vbid, {"dairy:milk", DocNamespace::DefaultCollection}, "nice");
202    store_item(
203            vbid, {"dairy:butter", DocNamespace::DefaultCollection}, "lovely");
204    store_item(vbid, {"fruit:apple", DocNamespace::DefaultCollection}, "nice");
205    store_item(
206            vbid, {"fruit:apricot", DocNamespace::DefaultCollection}, "lovely");
207
208    flush_vbucket_to_disk(vbid, 4);
209
210    EXPECT_EQ(4, vb->getNumItems());
211
212    // delete the default collection
213    vb->updateFromManifest({R"({"separator":":","uid":"0",)"
214                            R"("collections":[]})"});
215
216    flush_vbucket_to_disk(vbid, 1 /* 1 x system */);
217
218    runEraser();
219
220    EXPECT_EQ(0, vb->getNumItems());
221
222    // Add default back - so we don't get collection unknown errors
223    vb->updateFromManifest(
224            {R"({"separator":":","uid":"0",)"
225             R"("collections":[{"name":"$default", "uid":"0"}]})"});
226
227    get_options_t options = static_cast<get_options_t>(
228            QUEUE_BG_FETCH | HONOR_STATES | TRACK_REFERENCE | DELETE_TEMP |
229            HIDE_LOCKED_CAS | TRACK_STATISTICS);
230
231    GetValue gv = store->get({"dairy:milk", DocNamespace::DefaultCollection},
232                             vbid,
233                             cookie,
234                             options);
235    EXPECT_EQ(ENGINE_KEY_ENOENT, gv.getStatus());
236}
237
238// Demonstrate MB_26455. Here we trigger collection erasing after a separator
239// change, the code processing the old items can no longer determine which
240// collection they belong too and fails to remove the items
241TEST_P(CollectionsEraserTest, MB_26455) {
242    const int items = 3;
243    std::vector<std::string> separators = {
244            "::", "*", "**", "***", "-=-=-=-=-="};
245
246    for (size_t n = 0; n < separators.size(); n++) {
247        // change sep
248        std::string manifest =
249                R"({"separator":")" + separators.at(n) +
250                R"(","uid":"0","collections":[{"name":"$default", "uid":"0"}]})";
251        vb->updateFromManifest({manifest});
252
253        // add fruit
254        manifest =
255                R"({"separator":")" + separators.at(n) +
256                R"(","uid":"0","collections":[{"name":"$default", "uid":"0"},)" +
257                R"({"name":"fruit", "uid":")" + std::to_string(n) +
258                R"("}]})";
259
260        vb->updateFromManifest({manifest});
261
262        // Mutate fruit
263        const int items = 3;
264        for (int ii = 0; ii < items; ii++) {
265            std::string key = "fruit" + separators.at(n) + std::to_string(ii);
266            store_item(vbid, {key, DocNamespace::Collections}, "value");
267        }
268
269        // expect change_separator + create_collection + items
270        flush_vbucket_to_disk(vbid, 2 + items);
271
272        // Drop fruit
273        manifest =
274                R"({"separator":")" + separators.at(n) +
275                R"(","uid":"0","collections":[{"name":"$default", "uid":"0"}]})";
276        vb->updateFromManifest({manifest});
277
278        flush_vbucket_to_disk(vbid, 1);
279    }
280
281    EXPECT_EQ(items * separators.size(), vb->getNumItems());
282
283    // Eraser will fail to delete keys of the original fruit as it will be using
284    // the new separator and will never split the key correctly.
285    runEraser();
286
287    // All items erased
288    EXPECT_EQ(0, vb->getNumItems());
289
290    // Expected items on disk (the separator change keys)
291    EXPECT_EQ(separators.size(),
292              store->getROUnderlying(vbid)->getItemCount(vbid));
293
294    // Eraser should of generated some deletes of the now defunct separator
295    // change keys
296    flush_vbucket_to_disk(vbid, separators.size() - 1);
297
298    // Expect 1 item on disk (the last separator change key)
299    EXPECT_EQ(1, store->getROUnderlying(vbid)->getItemCount(vbid));
300}
301
302struct PrintTestName {
303    std::string operator()(
304            const ::testing::TestParamInfo<std::string>& info) const {
305        if ("bucket_type=persistent;item_eviction_policy=value_only" ==
306            info.param) {
307            return "PersistentVE";
308        } else if (
309                "bucket_type=persistent;item_eviction_policy=full_eviction" ==
310                info.param) {
311            return "PersistentFE";
312        } else if ("bucket_type=ephemeral" == info.param) {
313            return "Ephemeral";
314        } else {
315            throw std::invalid_argument("PrintTestName::Unknown info.param:" +
316                                        info.param);
317        }
318    }
319};
320
321// @todo add ephemeral config
322INSTANTIATE_TEST_CASE_P(
323        CollectionsEraserTests,
324        CollectionsEraserTest,
325        ::testing::Values(
326                "bucket_type=persistent;item_eviction_policy=value_only",
327                "bucket_type=persistent;item_eviction_policy=full_eviction"),
328        PrintTestName());