1 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3 * Copyright 2017 Couchbase, Inc
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18 #include "tests/module_tests/evp_store_single_threaded_test.h"
19
20 class CollectionsEraserTest
21 : public SingleThreadedKVBucketTest,
22 public ::testing::WithParamInterface<std::string> {
23 public:
24 void SetUp() override {
25 // Enable collections (which will enable namespace persistence).
26 config_string += "collections_prototype_enabled=true;";
27 config_string += GetParam();
28 SingleThreadedKVBucketTest::SetUp();
29 setVBucketStateAndRunPersistTask(vbid, vbucket_state_active);
30 vb = store->getVBucket(vbid);
31 }
32
33 void TearDown() override {
34 vb.reset();
35 SingleThreadedKVBucketTest::TearDown();
36 }
37
runEraser()38 void runEraser() {
39 runCompaction();
40 }
41
isFullEviction() const42 bool isFullEviction() const {
43 return GetParam().find("item_eviction_policy=full_eviction") !=
44 std::string::npos;
45 }
46
47 VBucketPtr vb;
48 };
49
50 // Small numbers of items for easier debug
TEST_P(CollectionsEraserTest, basic)51 TEST_P(CollectionsEraserTest, basic) {
52 // add a collection
53 vb->updateFromManifest({R"({"separator":":","uid":"0",)"
54 R"("collections":[{"name":"$default", "uid":"0"},)"
55 R"( {"name":"dairy","uid":"1"}]})"});
56
57 flush_vbucket_to_disk(vbid, 1 /* 1 x system */);
58
59 // add some items
60 store_item(vbid, {"dairy:milk", DocNamespace::Collections}, "nice");
61 store_item(vbid, {"dairy:butter", DocNamespace::Collections}, "lovely");
62
63 flush_vbucket_to_disk(vbid, 2 /* 2 x items */);
64
65 EXPECT_EQ(2, vb->getNumItems());
66
67 // Evict one of the keys, we should still erase it
68 evict_key(vbid, {"dairy:butter", DocNamespace::Collections});
69
70 // delete the collection
71 vb->updateFromManifest(
72 {R"({"separator":":","uid":"0",)"
73 R"("collections":[{"name":"$default", "uid":"0"}]})"});
74
75 flush_vbucket_to_disk(vbid, 1 /* 1 x system */);
76
77 // Deleted, but still exists in the manifest
78 EXPECT_TRUE(vb->lockCollections().exists("dairy"));
79
80 runEraser();
81
82 EXPECT_EQ(0, vb->getNumItems());
83
84 EXPECT_FALSE(vb->lockCollections().exists("dairy"));
85 }
86
TEST_P(CollectionsEraserTest, basic_2_collections)87 TEST_P(CollectionsEraserTest, basic_2_collections) {
88 // add a collection
89 vb->updateFromManifest({R"({"separator":":","uid":"0",)"
90 R"("collections":[{"name":"$default", "uid":"0"},)"
91 R"( {"name":"fruit","uid":"1"},)"
92 R"( {"name":"dairy","uid":"1"}]})"});
93
94 flush_vbucket_to_disk(vbid, 2 /* 2 x system */);
95
96 // add some items
97 store_item(vbid, {"dairy:milk", DocNamespace::Collections}, "nice");
98 store_item(vbid, {"dairy:butter", DocNamespace::Collections}, "lovely");
99 store_item(vbid, {"fruit:apple", DocNamespace::Collections}, "nice");
100 store_item(vbid, {"fruit:apricot", DocNamespace::Collections}, "lovely");
101
102 flush_vbucket_to_disk(vbid, 4);
103
104 EXPECT_EQ(4, vb->getNumItems());
105
106 // delete the collections
107 vb->updateFromManifest(
108 {R"({"separator":":","uid":"0",)"
109 R"("collections":[{"name":"$default", "uid":"0"}]})"});
110
111 // Deleted, but still exists in the manifest
112 EXPECT_TRUE(vb->lockCollections().exists("dairy"));
113 EXPECT_TRUE(vb->lockCollections().exists("fruit"));
114
115 flush_vbucket_to_disk(vbid, 2 /* 2 x system */);
116
117 runEraser();
118
119 EXPECT_EQ(0, vb->getNumItems());
120
121 EXPECT_FALSE(vb->lockCollections().exists("dairy"));
122 EXPECT_FALSE(vb->lockCollections().exists("fruit"));
123 }
124
TEST_P(CollectionsEraserTest, basic_3_collections)125 TEST_P(CollectionsEraserTest, basic_3_collections) {
126 // add a collection
127 vb->updateFromManifest({R"({"separator":":","uid":"0",)"
128 R"("collections":[{"name":"$default", "uid":"0"},)"
129 R"( {"name":"fruit","uid":"1"},)"
130 R"( {"name":"dairy","uid":"1"}]})"});
131
132 flush_vbucket_to_disk(vbid, 2 /* 1x system */);
133
134 // add some items
135 store_item(vbid, {"dairy:milk", DocNamespace::Collections}, "nice");
136 store_item(vbid, {"dairy:butter", DocNamespace::Collections}, "lovely");
137 store_item(vbid, {"fruit:apple", DocNamespace::Collections}, "nice");
138 store_item(vbid, {"fruit:apricot", DocNamespace::Collections}, "lovely");
139
140 flush_vbucket_to_disk(vbid, 4 /* 2x items */);
141
142 EXPECT_EQ(4, vb->getNumItems());
143
144 // delete one of the 3 collections
145 vb->updateFromManifest({R"({"separator":":","uid":"0",)"
146 R"("collections":[{"name":"$default", "uid":"0"},)"
147 R"( {"name":"dairy","uid":"1"}]})"});
148
149 // Deleted, but still exists in the manifest
150 EXPECT_TRUE(vb->lockCollections().exists("dairy"));
151 EXPECT_TRUE(vb->lockCollections().exists("fruit"));
152
153 flush_vbucket_to_disk(vbid, 1 /* 1 x system */);
154
155 runEraser();
156
157 EXPECT_EQ(2, vb->getNumItems());
158
159 EXPECT_TRUE(vb->lockCollections().exists("dairy"));
160 EXPECT_FALSE(vb->lockCollections().exists("fruit"));
161 }
162
TEST_P(CollectionsEraserTest, basic_4_collections)163 TEST_P(CollectionsEraserTest, basic_4_collections) {
164 // add a collection
165 vb->updateFromManifest({R"({"separator":":","uid":"0",)"
166 R"("collections":[{"name":"$default", "uid":"0"},)"
167 R"( {"name":"fruit","uid":"1"},)"
168 R"( {"name":"dairy","uid":"1"}]})"});
169
170 flush_vbucket_to_disk(vbid, 2 /* 1x system */);
171
172 // add some items
173 store_item(vbid, {"dairy:milk", DocNamespace::Collections}, "nice");
174 store_item(vbid, {"dairy:butter", DocNamespace::Collections}, "lovely");
175 store_item(vbid, {"fruit:apple", DocNamespace::Collections}, "nice");
176 store_item(vbid, {"fruit:apricot", DocNamespace::Collections}, "lovely");
177
178 flush_vbucket_to_disk(vbid, 4 /* 2x items */);
179
180 // delete the collection and re-add a new dairy
181 vb->updateFromManifest({R"({"separator":":","uid":"0",)"
182 R"("collections":[{"name":"$default", "uid":"0"},)"
183 R"( {"name":"dairy","uid":"2"}]})"});
184
185 // Deleted, but still exists in the manifest
186 EXPECT_TRUE(vb->lockCollections().exists("dairy"));
187 EXPECT_TRUE(vb->lockCollections().exists("fruit"));
188
189 flush_vbucket_to_disk(vbid, 2 /* 1x system */);
190
191 runEraser();
192
193 EXPECT_EQ(0, vb->getNumItems());
194
195 EXPECT_TRUE(vb->lockCollections().exists("dairy"));
196 EXPECT_FALSE(vb->lockCollections().exists("fruit"));
197 }
198
TEST_P(CollectionsEraserTest, default_Destroy)199 TEST_P(CollectionsEraserTest, default_Destroy) {
200 // add some items
201 store_item(vbid, {"dairy:milk", DocNamespace::DefaultCollection}, "nice");
202 store_item(
203 vbid, {"dairy:butter", DocNamespace::DefaultCollection}, "lovely");
204 store_item(vbid, {"fruit:apple", DocNamespace::DefaultCollection}, "nice");
205 store_item(
206 vbid, {"fruit:apricot", DocNamespace::DefaultCollection}, "lovely");
207
208 flush_vbucket_to_disk(vbid, 4);
209
210 EXPECT_EQ(4, vb->getNumItems());
211
212 // delete the default collection
213 vb->updateFromManifest({R"({"separator":":","uid":"0",)"
214 R"("collections":[]})"});
215
216 flush_vbucket_to_disk(vbid, 1 /* 1 x system */);
217
218 runEraser();
219
220 EXPECT_EQ(0, vb->getNumItems());
221
222 // Add default back - so we don't get collection unknown errors
223 vb->updateFromManifest(
224 {R"({"separator":":","uid":"0",)"
225 R"("collections":[{"name":"$default", "uid":"0"}]})"});
226
227 get_options_t options = static_cast<get_options_t>(
228 QUEUE_BG_FETCH | HONOR_STATES | TRACK_REFERENCE | DELETE_TEMP |
229 HIDE_LOCKED_CAS | TRACK_STATISTICS);
230
231 GetValue gv = store->get({"dairy:milk", DocNamespace::DefaultCollection},
232 vbid,
233 cookie,
234 options);
235 EXPECT_EQ(ENGINE_KEY_ENOENT, gv.getStatus());
236 }
237
238 // Demonstrate MB_26455. Here we trigger collection erasing after a separator
239 // change, the code processing the old items can no longer determine which
240 // collection they belong too and fails to remove the items
TEST_P(CollectionsEraserTest, MB_26455)241 TEST_P(CollectionsEraserTest, MB_26455) {
242 const int items = 3;
243 std::vector<std::string> separators = {
244 "::", "*", "**", "***", "-=-=-=-=-="};
245
246 for (size_t n = 0; n < separators.size(); n++) {
247 // change sep
248 std::string manifest =
249 R"({"separator":")" + separators.at(n) +
250 R"(","uid":"0","collections":[{"name":"$default", "uid":"0"}]})";
251 vb->updateFromManifest({manifest});
252
253 // add fruit
254 manifest =
255 R"({"separator":")" + separators.at(n) +
256 R"(","uid":"0","collections":[{"name":"$default", "uid":"0"},)" +
257 R"({"name":"fruit", "uid":")" + std::to_string(n) +
258 R"("}]})";
259
260 vb->updateFromManifest({manifest});
261
262 // Mutate fruit
263 const int items = 3;
264 for (int ii = 0; ii < items; ii++) {
265 std::string key = "fruit" + separators.at(n) + std::to_string(ii);
266 store_item(vbid, {key, DocNamespace::Collections}, "value");
267 }
268
269 // expect change_separator + create_collection + items
270 flush_vbucket_to_disk(vbid, 2 + items);
271
272 // Drop fruit
273 manifest =
274 R"({"separator":")" + separators.at(n) +
275 R"(","uid":"0","collections":[{"name":"$default", "uid":"0"}]})";
276 vb->updateFromManifest({manifest});
277
278 flush_vbucket_to_disk(vbid, 1);
279 }
280
281 EXPECT_EQ(items * separators.size(), vb->getNumItems());
282
283 // Eraser will fail to delete keys of the original fruit as it will be using
284 // the new separator and will never split the key correctly.
285 runEraser();
286
287 // All items erased
288 EXPECT_EQ(0, vb->getNumItems());
289
290 // Expected items on disk (the separator change keys)
291 EXPECT_EQ(separators.size(),
292 store->getROUnderlying(vbid)->getItemCount(vbid));
293
294 // Eraser should of generated some deletes of the now defunct separator
295 // change keys
296 flush_vbucket_to_disk(vbid, separators.size() - 1);
297
298 // Expect 1 item on disk (the last separator change key)
299 EXPECT_EQ(1, store->getROUnderlying(vbid)->getItemCount(vbid));
300 }
301
302 struct PrintTestName {
303 std::string operator()(
304 const ::testing::TestParamInfo<std::string>& info) const {
305 if ("bucket_type=persistent;item_eviction_policy=value_only" ==
306 info.param) {
307 return "PersistentVE";
308 } else if (
309 "bucket_type=persistent;item_eviction_policy=full_eviction" ==
310 info.param) {
311 return "PersistentFE";
312 } else if ("bucket_type=ephemeral" == info.param) {
313 return "Ephemeral";
314 } else {
315 throw std::invalid_argument("PrintTestName::Unknown info.param:" +
316 info.param);
317 }
318 }
319 };
320
321 // @todo add ephemeral config
322 INSTANTIATE_TEST_CASE_P(
323 CollectionsEraserTests,
324 CollectionsEraserTest,
325 ::testing::Values(
326 "bucket_type=persistent;item_eviction_policy=value_only",
327 "bucket_type=persistent;item_eviction_policy=full_eviction"),
328 PrintTestName());