1 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3  *     Copyright 2017 Couchbase, Inc
4  *
5  *   Licensed under the Apache License, Version 2.0 (the "License");
6  *   you may not use this file except in compliance with the License.
7  *   You may obtain a copy of the License at
8  *
9  *       http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *   Unless required by applicable law or agreed to in writing, software
12  *   distributed under the License is distributed on an "AS IS" BASIS,
13  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *   See the License for the specific language governing permissions and
15  *   limitations under the License.
16  */
17 
18 #include "tests/module_tests/evp_store_single_threaded_test.h"
19 
20 class CollectionsEraserTest
21         : public SingleThreadedKVBucketTest,
22           public ::testing::WithParamInterface<std::string> {
23 public:
24     void SetUp() override {
25         // Enable collections (which will enable namespace persistence).
26         config_string += "collections_prototype_enabled=true;";
27         config_string += GetParam();
28         SingleThreadedKVBucketTest::SetUp();
29         setVBucketStateAndRunPersistTask(vbid, vbucket_state_active);
30         vb = store->getVBucket(vbid);
31     }
32 
33     void TearDown() override {
34         vb.reset();
35         SingleThreadedKVBucketTest::TearDown();
36     }
37 
runEraser()38     void runEraser() {
39         runCompaction();
40     }
41 
isFullEviction() const42     bool isFullEviction() const {
43         return GetParam().find("item_eviction_policy=full_eviction") !=
44                std::string::npos;
45     }
46 
47     VBucketPtr vb;
48 };
49 
50 // Small numbers of items for easier debug
TEST_P(CollectionsEraserTest, basic)51 TEST_P(CollectionsEraserTest, basic) {
52     // add a collection
53     vb->updateFromManifest({R"({"separator":":","uid":"0",)"
54                             R"("collections":[{"name":"$default", "uid":"0"},)"
55                             R"(               {"name":"dairy","uid":"1"}]})"});
56 
57     flush_vbucket_to_disk(vbid, 1 /* 1 x system */);
58 
59     // add some items
60     store_item(vbid, {"dairy:milk", DocNamespace::Collections}, "nice");
61     store_item(vbid, {"dairy:butter", DocNamespace::Collections}, "lovely");
62 
63     flush_vbucket_to_disk(vbid, 2 /* 2 x items */);
64 
65     EXPECT_EQ(2, vb->getNumItems());
66 
67     // Evict one of the keys, we should still erase it
68     evict_key(vbid, {"dairy:butter", DocNamespace::Collections});
69 
70     // delete the collection
71     vb->updateFromManifest(
72             {R"({"separator":":","uid":"0",)"
73              R"("collections":[{"name":"$default", "uid":"0"}]})"});
74 
75     flush_vbucket_to_disk(vbid, 1 /* 1 x system */);
76 
77     // Deleted, but still exists in the manifest
78     EXPECT_TRUE(vb->lockCollections().exists("dairy"));
79 
80     runEraser();
81 
82     EXPECT_EQ(0, vb->getNumItems());
83 
84     EXPECT_FALSE(vb->lockCollections().exists("dairy"));
85 }
86 
TEST_P(CollectionsEraserTest, basic_2_collections)87 TEST_P(CollectionsEraserTest, basic_2_collections) {
88     // add a collection
89     vb->updateFromManifest({R"({"separator":":","uid":"0",)"
90                             R"("collections":[{"name":"$default", "uid":"0"},)"
91                             R"(               {"name":"fruit","uid":"1"},)"
92                             R"(               {"name":"dairy","uid":"1"}]})"});
93 
94     flush_vbucket_to_disk(vbid, 2 /* 2 x system */);
95 
96     // add some items
97     store_item(vbid, {"dairy:milk", DocNamespace::Collections}, "nice");
98     store_item(vbid, {"dairy:butter", DocNamespace::Collections}, "lovely");
99     store_item(vbid, {"fruit:apple", DocNamespace::Collections}, "nice");
100     store_item(vbid, {"fruit:apricot", DocNamespace::Collections}, "lovely");
101 
102     flush_vbucket_to_disk(vbid, 4);
103 
104     EXPECT_EQ(4, vb->getNumItems());
105 
106     // delete the collections
107     vb->updateFromManifest(
108             {R"({"separator":":","uid":"0",)"
109              R"("collections":[{"name":"$default", "uid":"0"}]})"});
110 
111     // Deleted, but still exists in the manifest
112     EXPECT_TRUE(vb->lockCollections().exists("dairy"));
113     EXPECT_TRUE(vb->lockCollections().exists("fruit"));
114 
115     flush_vbucket_to_disk(vbid, 2 /* 2 x system */);
116 
117     runEraser();
118 
119     EXPECT_EQ(0, vb->getNumItems());
120 
121     EXPECT_FALSE(vb->lockCollections().exists("dairy"));
122     EXPECT_FALSE(vb->lockCollections().exists("fruit"));
123 }
124 
TEST_P(CollectionsEraserTest, basic_3_collections)125 TEST_P(CollectionsEraserTest, basic_3_collections) {
126     // add a collection
127     vb->updateFromManifest({R"({"separator":":","uid":"0",)"
128                             R"("collections":[{"name":"$default", "uid":"0"},)"
129                             R"(               {"name":"fruit","uid":"1"},)"
130                             R"(               {"name":"dairy","uid":"1"}]})"});
131 
132     flush_vbucket_to_disk(vbid, 2 /* 1x system */);
133 
134     // add some items
135     store_item(vbid, {"dairy:milk", DocNamespace::Collections}, "nice");
136     store_item(vbid, {"dairy:butter", DocNamespace::Collections}, "lovely");
137     store_item(vbid, {"fruit:apple", DocNamespace::Collections}, "nice");
138     store_item(vbid, {"fruit:apricot", DocNamespace::Collections}, "lovely");
139 
140     flush_vbucket_to_disk(vbid, 4 /* 2x items */);
141 
142     EXPECT_EQ(4, vb->getNumItems());
143 
144     // delete one of the 3 collections
145     vb->updateFromManifest({R"({"separator":":","uid":"0",)"
146                             R"("collections":[{"name":"$default", "uid":"0"},)"
147                             R"(               {"name":"dairy","uid":"1"}]})"});
148 
149     // Deleted, but still exists in the manifest
150     EXPECT_TRUE(vb->lockCollections().exists("dairy"));
151     EXPECT_TRUE(vb->lockCollections().exists("fruit"));
152 
153     flush_vbucket_to_disk(vbid, 1 /* 1 x system */);
154 
155     runEraser();
156 
157     EXPECT_EQ(2, vb->getNumItems());
158 
159     EXPECT_TRUE(vb->lockCollections().exists("dairy"));
160     EXPECT_FALSE(vb->lockCollections().exists("fruit"));
161 }
162 
TEST_P(CollectionsEraserTest, basic_4_collections)163 TEST_P(CollectionsEraserTest, basic_4_collections) {
164     // add a collection
165     vb->updateFromManifest({R"({"separator":":","uid":"0",)"
166                             R"("collections":[{"name":"$default", "uid":"0"},)"
167                             R"(               {"name":"fruit","uid":"1"},)"
168                             R"(               {"name":"dairy","uid":"1"}]})"});
169 
170     flush_vbucket_to_disk(vbid, 2 /* 1x system */);
171 
172     // add some items
173     store_item(vbid, {"dairy:milk", DocNamespace::Collections}, "nice");
174     store_item(vbid, {"dairy:butter", DocNamespace::Collections}, "lovely");
175     store_item(vbid, {"fruit:apple", DocNamespace::Collections}, "nice");
176     store_item(vbid, {"fruit:apricot", DocNamespace::Collections}, "lovely");
177 
178     flush_vbucket_to_disk(vbid, 4 /* 2x items */);
179 
180     // delete the collection and re-add a new dairy
181     vb->updateFromManifest({R"({"separator":":","uid":"0",)"
182                             R"("collections":[{"name":"$default", "uid":"0"},)"
183                             R"(               {"name":"dairy","uid":"2"}]})"});
184 
185     // Deleted, but still exists in the manifest
186     EXPECT_TRUE(vb->lockCollections().exists("dairy"));
187     EXPECT_TRUE(vb->lockCollections().exists("fruit"));
188 
189     flush_vbucket_to_disk(vbid, 2 /* 1x system */);
190 
191     runEraser();
192 
193     EXPECT_EQ(0, vb->getNumItems());
194 
195     EXPECT_TRUE(vb->lockCollections().exists("dairy"));
196     EXPECT_FALSE(vb->lockCollections().exists("fruit"));
197 }
198 
TEST_P(CollectionsEraserTest, default_Destroy)199 TEST_P(CollectionsEraserTest, default_Destroy) {
200     // add some items
201     store_item(vbid, {"dairy:milk", DocNamespace::DefaultCollection}, "nice");
202     store_item(
203             vbid, {"dairy:butter", DocNamespace::DefaultCollection}, "lovely");
204     store_item(vbid, {"fruit:apple", DocNamespace::DefaultCollection}, "nice");
205     store_item(
206             vbid, {"fruit:apricot", DocNamespace::DefaultCollection}, "lovely");
207 
208     flush_vbucket_to_disk(vbid, 4);
209 
210     EXPECT_EQ(4, vb->getNumItems());
211 
212     // delete the default collection
213     vb->updateFromManifest({R"({"separator":":","uid":"0",)"
214                             R"("collections":[]})"});
215 
216     flush_vbucket_to_disk(vbid, 1 /* 1 x system */);
217 
218     runEraser();
219 
220     EXPECT_EQ(0, vb->getNumItems());
221 
222     // Add default back - so we don't get collection unknown errors
223     vb->updateFromManifest(
224             {R"({"separator":":","uid":"0",)"
225              R"("collections":[{"name":"$default", "uid":"0"}]})"});
226 
227     get_options_t options = static_cast<get_options_t>(
228             QUEUE_BG_FETCH | HONOR_STATES | TRACK_REFERENCE | DELETE_TEMP |
229             HIDE_LOCKED_CAS | TRACK_STATISTICS);
230 
231     GetValue gv = store->get({"dairy:milk", DocNamespace::DefaultCollection},
232                              vbid,
233                              cookie,
234                              options);
235     EXPECT_EQ(ENGINE_KEY_ENOENT, gv.getStatus());
236 }
237 
238 // Demonstrate MB_26455. Here we trigger collection erasing after a separator
239 // change, the code processing the old items can no longer determine which
240 // collection they belong too and fails to remove the items
TEST_P(CollectionsEraserTest, MB_26455)241 TEST_P(CollectionsEraserTest, MB_26455) {
242     const int items = 3;
243     std::vector<std::string> separators = {
244             "::", "*", "**", "***", "-=-=-=-=-="};
245 
246     for (size_t n = 0; n < separators.size(); n++) {
247         // change sep
248         std::string manifest =
249                 R"({"separator":")" + separators.at(n) +
250                 R"(","uid":"0","collections":[{"name":"$default", "uid":"0"}]})";
251         vb->updateFromManifest({manifest});
252 
253         // add fruit
254         manifest =
255                 R"({"separator":")" + separators.at(n) +
256                 R"(","uid":"0","collections":[{"name":"$default", "uid":"0"},)" +
257                 R"({"name":"fruit", "uid":")" + std::to_string(n) +
258                 R"("}]})";
259 
260         vb->updateFromManifest({manifest});
261 
262         // Mutate fruit
263         const int items = 3;
264         for (int ii = 0; ii < items; ii++) {
265             std::string key = "fruit" + separators.at(n) + std::to_string(ii);
266             store_item(vbid, {key, DocNamespace::Collections}, "value");
267         }
268 
269         // expect change_separator + create_collection + items
270         flush_vbucket_to_disk(vbid, 2 + items);
271 
272         // Drop fruit
273         manifest =
274                 R"({"separator":")" + separators.at(n) +
275                 R"(","uid":"0","collections":[{"name":"$default", "uid":"0"}]})";
276         vb->updateFromManifest({manifest});
277 
278         flush_vbucket_to_disk(vbid, 1);
279     }
280 
281     EXPECT_EQ(items * separators.size(), vb->getNumItems());
282 
283     // Eraser will fail to delete keys of the original fruit as it will be using
284     // the new separator and will never split the key correctly.
285     runEraser();
286 
287     // All items erased
288     EXPECT_EQ(0, vb->getNumItems());
289 
290     // Expected items on disk (the separator change keys)
291     EXPECT_EQ(separators.size(),
292               store->getROUnderlying(vbid)->getItemCount(vbid));
293 
294     // Eraser should of generated some deletes of the now defunct separator
295     // change keys
296     flush_vbucket_to_disk(vbid, separators.size() - 1);
297 
298     // Expect 1 item on disk (the last separator change key)
299     EXPECT_EQ(1, store->getROUnderlying(vbid)->getItemCount(vbid));
300 }
301 
302 struct PrintTestName {
303     std::string operator()(
304             const ::testing::TestParamInfo<std::string>& info) const {
305         if ("bucket_type=persistent;item_eviction_policy=value_only" ==
306             info.param) {
307             return "PersistentVE";
308         } else if (
309                 "bucket_type=persistent;item_eviction_policy=full_eviction" ==
310                 info.param) {
311             return "PersistentFE";
312         } else if ("bucket_type=ephemeral" == info.param) {
313             return "Ephemeral";
314         } else {
315             throw std::invalid_argument("PrintTestName::Unknown info.param:" +
316                                         info.param);
317         }
318     }
319 };
320 
321 // @todo add ephemeral config
322 INSTANTIATE_TEST_CASE_P(
323         CollectionsEraserTests,
324         CollectionsEraserTest,
325         ::testing::Values(
326                 "bucket_type=persistent;item_eviction_policy=value_only",
327                 "bucket_type=persistent;item_eviction_policy=full_eviction"),
328         PrintTestName());