1 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3  *     Copyright 2017 Couchbase, Inc
4  *
5  *   Licensed under the Apache License, Version 2.0 (the "License");
6  *   you may not use this file except in compliance with the License.
7  *   You may obtain a copy of the License at
8  *
9  *       http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *   Unless required by applicable law or agreed to in writing, software
12  *   distributed under the License is distributed on an "AS IS" BASIS,
13  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *   See the License for the specific language governing permissions and
15  *   limitations under the License.
16  */
17 
18 /**
19  * Tests for Collection functionality in EPStore.
20  */
21 #include "bgfetcher.h"
22 #include "ep_time.h"
23 #include "kvstore.h"
24 #include "programs/engine_testapp/mock_server.h"
25 #include "tests/mock/mock_global_task.h"
26 #include "tests/module_tests/evp_store_single_threaded_test.h"
27 #include "tests/module_tests/evp_store_test.h"
28 #include "tests/module_tests/test_helpers.h"
29 
30 #include <functional>
31 #include <thread>
32 
33 class CollectionsTest : public SingleThreadedKVBucketTest {
34 public:
35     void SetUp() override {
36         // Enable collections (which will enable namespace persistence).
37         config_string += "collections_prototype_enabled=true";
38         SingleThreadedKVBucketTest::SetUp();
39         // Start vbucket as active to allow us to store items directly to it.
40         store->setVBucketState(vbid, vbucket_state_active, false);
41     }
42 
getManifest(uint16_t vb) const43     std::string getManifest(uint16_t vb) const {
44         return store->getVBucket(vb)
45                 ->getShard()
46                 ->getRWUnderlying()
47                 ->getCollectionsManifest(vbid);
48     }
49 };
50 
51 // This test stores a key which matches what collections internally uses, but
52 // in a different namespace.
TEST_F(CollectionsTest, namespace_separation)53 TEST_F(CollectionsTest, namespace_separation) {
54     // Use the event factory to get an event which we'll borrow the key from
55     auto se = SystemEventFactory::make(SystemEvent::Collection, "meat", 0, {});
56     DocKey key(se->getKey().data(),
57                se->getKey().size(),
58                DocNamespace::DefaultCollection);
59 
60     store_item(vbid, key, "value");
61     VBucketPtr vb = store->getVBucket(vbid);
62     // Add the meat collection
63     vb->updateFromManifest({R"({"separator":":","uid":"0",
64                  "collections":[{"name":"$default", "uid":"0"},
65                                 {"name":"meat", "uid":"1"}]})"});
66     // Trigger a flush to disk. Flushes the meat create event and 1 item
67     flush_vbucket_to_disk(vbid, 2);
68 
69     // evict and load - should not see the system key for create collections
70     evict_key(vbid, key);
71     get_options_t options = static_cast<get_options_t>(
72             QUEUE_BG_FETCH | HONOR_STATES | TRACK_REFERENCE | DELETE_TEMP |
73             HIDE_LOCKED_CAS | TRACK_STATISTICS);
74     GetValue gv = store->get(key, vbid, cookie, options);
75     EXPECT_EQ(ENGINE_EWOULDBLOCK, gv.getStatus());
76 
77     // Manually run the BGFetcher task; to fetch the two outstanding
78     // requests (for the same key).
79     runBGFetcherTask();
80 
81     gv = store->get(key, vbid, cookie, options);
82     EXPECT_EQ(ENGINE_SUCCESS, gv.getStatus());
83     EXPECT_EQ(0, strncmp("value", gv.item->getData(), gv.item->getNBytes()));
84 }
85 
TEST_F(CollectionsTest, collections_basic)86 TEST_F(CollectionsTest, collections_basic) {
87     // Default collection is open for business
88     store_item(vbid, {"key", DocNamespace::DefaultCollection}, "value");
89     store_item(vbid,
90                {"meat:beef", DocNamespace::Collections},
91                "value",
92                0,
93                {cb::engine_errc::unknown_collection});
94 
95     VBucketPtr vb = store->getVBucket(vbid);
96 
97     // Add the meat collection
98     vb->updateFromManifest({R"({"separator":":","uid":"0",
99                  "collections":[{"name":"$default", "uid":"0"},
100                                 {"name":"meat", "uid":"1"}]})"});
101 
102     // Trigger a flush to disk. Flushes the meat create event and 1 item
103     flush_vbucket_to_disk(vbid, 2);
104 
105     // Now we can write to beef
106     store_item(vbid, {"meat:beef", DocNamespace::Collections}, "value");
107 
108     flush_vbucket_to_disk(vbid, 1);
109 
110     // And read a document from beef
111     get_options_t options = static_cast<get_options_t>(
112             QUEUE_BG_FETCH | HONOR_STATES | TRACK_REFERENCE | DELETE_TEMP |
113             HIDE_LOCKED_CAS | TRACK_STATISTICS);
114 
115     GetValue gv = store->get(
116             {"meat:beef", DocNamespace::Collections}, vbid, cookie, options);
117     ASSERT_EQ(ENGINE_SUCCESS, gv.getStatus());
118 
119     // A key in meat that doesn't exist
120     gv = store->get(
121             {"meat:sausage", DocNamespace::Collections}, vbid, cookie, options);
122     EXPECT_EQ(ENGINE_KEY_ENOENT, gv.getStatus());
123 
124     // Begin the deletion
125     vb->updateFromManifest({R"({"separator":":","uid":"0",
126                  "collections":[{"name":"$default", "uid":"0"}]})"});
127 
128     // We should have deleted the create marker
129     flush_vbucket_to_disk(vbid, 1);
130 
131     // Access denied (although the item still exists)
132     gv = store->get(
133             {"meat:beef", DocNamespace::Collections}, vbid, cookie, options);
134     EXPECT_EQ(ENGINE_UNKNOWN_COLLECTION, gv.getStatus());
135 }
136 
137 // Test demonstrates issue logged as MB_25344, when we delete a collection
138 // and then happen to perform a mutation against a new rev of the collection
139 // we may encounter the key which is pending deletion and then fail when we
140 // shouldn't. In this test the final add should logically work, but fails as the
141 // old key is found.
TEST_F(CollectionsTest, MB_25344)142 TEST_F(CollectionsTest, MB_25344) {
143     VBucketPtr vb = store->getVBucket(vbid);
144     // Add the dairy collection
145     vb->updateFromManifest({R"({"separator":":","uid":"0",
146                  "collections":[{"name":"$default", "uid":"0"},
147                                 {"name":"dairy", "uid":"1"}]})"});
148     // Trigger a flush to disk. Flushes the dairy create event.
149     flush_vbucket_to_disk(vbid, 1);
150 
151     auto item1 = make_item(
152             vbid, {"dairy:milk", DocNamespace::Collections}, "creamy", 0, 0);
153     EXPECT_EQ(ENGINE_SUCCESS, store->add(item1, cookie));
154     flush_vbucket_to_disk(vbid, 1);
155 
156     auto item2 = make_item(
157             vbid, {"dairy:cream", DocNamespace::Collections}, "creamy", 0, 0);
158     EXPECT_EQ(ENGINE_SUCCESS, store->add(item2, cookie));
159     flush_vbucket_to_disk(vbid, 1);
160 
161     // Delete the dairy collection (so all dairy keys become logically deleted)
162     vb->updateFromManifest({R"({"separator":":","uid":"0",
163                  "collections":[{"name":"$default", "uid":"0"}]})"});
164 
165     // Re-add the dairy collection
166     vb->updateFromManifest({R"({"separator":":","uid":"0",
167                  "collections":[{"name":"$default", "uid":"0"},
168                                 {"name":"dairy", "uid":"2"}]})"});
169 
170     // Trigger a flush to disk. Flushes the dairy create event.
171     flush_vbucket_to_disk(vbid, 1);
172 
173     // Expect that we can add item1 again, even though it is still present
174     item1.setCas(0);
175     EXPECT_EQ(ENGINE_SUCCESS, store->add(item1, cookie));
176 
177     // Replace should fail
178     EXPECT_EQ(ENGINE_KEY_ENOENT, store->replace(item2, cookie));
179 
180     // Delete should fail
181     uint64_t cas = 0;
182     mutation_descr_t mutation_descr;
183     EXPECT_EQ(ENGINE_KEY_ENOENT,
184               store->deleteItem(item2.getKey(),
185                                 cas,
186                                 vbid,
187                                 cookie,
188                                 nullptr,
189                                 mutation_descr));
190 
191     // Unlock should fail enoent rather than an unlock error
192     EXPECT_EQ(ENGINE_KEY_ENOENT,
193               store->unlockKey(item2.getKey(), vbid, 0, ep_current_time()));
194 
195     EXPECT_EQ(
196             "collection_unknown",
197             store->validateKey(
198                     {"meat:sausage", DocNamespace::Collections}, vbid, item2));
199     EXPECT_EQ("item_deleted", store->validateKey(item2.getKey(), vbid, item2));
200 
201     EXPECT_EQ(
202             ENGINE_UNKNOWN_COLLECTION,
203             store->statsVKey(
204                     {"meat:sausage", DocNamespace::Collections}, vbid, cookie));
205     EXPECT_EQ(ENGINE_KEY_ENOENT,
206               store->statsVKey(item2.getKey(), vbid, cookie));
207 
208     // GetKeyStats
209     struct key_stats ks;
210     EXPECT_EQ(ENGINE_KEY_ENOENT,
211               store->getKeyStats(
212                       item2.getKey(), vbid, cookie, ks, WantsDeleted::No));
213     EXPECT_EQ(ENGINE_SUCCESS,
214               store->getKeyStats(
215                       item2.getKey(), vbid, cookie, ks, WantsDeleted::Yes));
216     EXPECT_TRUE(ks.logically_deleted);
217 
218     uint32_t deleted = 0;
219     uint8_t dtype = 0;
220     ItemMetaData meta;
221     EXPECT_EQ(ENGINE_KEY_ENOENT,
222               store->getMetaData(
223                       item2.getKey(), vbid, nullptr, meta, deleted, dtype));
224 
225     // Prior to making deleteWithMeta isLogicallyDeleted aware, this would be
226     // success.
227     cas = 0;
228     meta.cas = 1;
229     EXPECT_EQ(ENGINE_KEY_ENOENT,
230               store->deleteWithMeta(item2.getKey(),
231                                     cas,
232                                     nullptr,
233                                     vbid,
234                                     nullptr,
235                                     {vbucket_state_active},
236                                     CheckConflicts::No,
237                                     meta,
238                                     false,
239                                     GenerateBySeqno::Yes,
240                                     GenerateCas::No,
241                                     0,
242                                     nullptr,
243                                     false));
244 
245     // Prior to making setWithMeta isLogicallyDeleted aware, this would conflict
246     // with the dead key - it should succeed
247     EXPECT_EQ(ENGINE_SUCCESS,
248               store->setWithMeta(item2,
249                                  0,
250                                  nullptr,
251                                  nullptr,
252                                  {vbucket_state_active},
253                                  CheckConflicts::Yes,
254                                  false,
255                                  GenerateBySeqno::Yes,
256                                  GenerateCas::No));
257 }
258 
259 // Test demonstrates issue logged as MB_25344, when we delete a collection
260 // and then happen to perform a mutation against a new rev of the collection
261 // we may encounter the key which is pending deletion and then fail when we
262 // shouldn't. In this test the final get should fail even though it does find
263 // a matching key
TEST_F(CollectionsTest, MB_25344_get)264 TEST_F(CollectionsTest, MB_25344_get) {
265     VBucketPtr vb = store->getVBucket(vbid);
266     // Add the dairy collection
267     vb->updateFromManifest({R"({"separator":":","uid":"0",
268                  "collections":[{"name":"$default", "uid":"0"},
269                                 {"name":"dairy", "uid":"1"}]})"});
270     // Trigger a flush to disk. Flushes the dairy create event.
271     flush_vbucket_to_disk(vbid, 1);
272 
273     auto item1 = make_item(
274             vbid, {"dairy:milk", DocNamespace::Collections}, "creamy", 0, 0);
275     EXPECT_EQ(ENGINE_SUCCESS, store->add(item1, cookie));
276     flush_vbucket_to_disk(vbid, 1);
277 
278     // Delete the dairy collection (so all dairy keys become logically deleted)
279     vb->updateFromManifest({R"({"separator":":","uid":"0",
280                  "collections":[{"name":"$default", "uid":"0"}]})"});
281 
282     // Re-add the dairy collection
283     vb->updateFromManifest({R"({"separator":":","uid":"0",
284                  "collections":[{"name":"$default", "uid":"0"},
285                                 {"name":"dairy", "uid":"2"}]})"});
286 
287     // Trigger a flush to disk. Flushes the dairy create event.
288     flush_vbucket_to_disk(vbid, 1);
289 
290     // The dairy:2 collection is empty
291     get_options_t options = static_cast<get_options_t>(
292             QUEUE_BG_FETCH | HONOR_STATES | TRACK_REFERENCE | DELETE_TEMP |
293             HIDE_LOCKED_CAS | TRACK_STATISTICS | GET_DELETED_VALUE);
294 
295     // Get deleted can't get it
296     auto gv = store->get(
297             {"dairy:milk", DocNamespace::Collections}, vbid, cookie, options);
298     EXPECT_EQ(ENGINE_KEY_ENOENT, gv.getStatus());
299 
300     options = static_cast<get_options_t>(QUEUE_BG_FETCH | HONOR_STATES |
301                                          TRACK_REFERENCE | DELETE_TEMP |
302                                          HIDE_LOCKED_CAS | TRACK_STATISTICS);
303 
304     // Normal Get can't get it
305     gv = store->get(
306             {"dairy:milk", DocNamespace::Collections}, vbid, cookie, options);
307     EXPECT_EQ(ENGINE_KEY_ENOENT, gv.getStatus());
308 
309     // Same for getLocked
310     gv = store->getLocked({"dairy:milk", DocNamespace::Collections},
311                           vbid,
312                           ep_current_time(),
313                           10,
314                           cookie);
315     EXPECT_EQ(ENGINE_KEY_ENOENT, gv.getStatus());
316 
317     // Same for getAndUpdateTtl
318     gv = store->getAndUpdateTtl({"dairy:milk", DocNamespace::Collections},
319                                 vbid,
320                                 cookie,
321                                 ep_current_time() + 20);
322     EXPECT_EQ(ENGINE_KEY_ENOENT, gv.getStatus());
323 }
324 
325 class CollectionsFlushTest : public CollectionsTest {
326 public:
327     void SetUp() override {
328         CollectionsTest::SetUp();
329     }
330 
331     void collectionsFlusher(int items);
332 
333 private:
334     std::string createCollectionAndFlush(const std::string& json,
335                                          const std::string& collection,
336                                          int items);
337     std::string deleteCollectionAndFlush(const std::string& json,
338                                          const std::string& collection,
339                                          int items);
340     std::string completeDeletionAndFlush(const std::string& collection,
341                                          int items);
342 
343     void storeItems(const std::string& collection, DocNamespace ns, int items);
344 
345     /**
346      * Create manifest object from jsonManifest and validate if we can write to
347      * the collection.
348      * @param jsonManifest - A JSON VB manifest
349      * @param collection - a collection name to test for writing
350      *
351      * @return true if the collection can be written
352      */
353     static bool canWrite(const std::string& jsonManifest,
354                          const std::string& collection);
355 
356     /**
357      * Create manifest object from jsonManifest and validate if we cannot write
358      * to the collection.
359      * @param jsonManifest - A JSON VB manifest
360      * @param collection - a collection name to test for writing
361      *
362      * @return true if the collection cannot be written
363      */
364     static bool cannotWrite(const std::string& jsonManifest,
365                             const std::string& collection);
366 };
367 
storeItems(const std::string& collection, DocNamespace ns, int items)368 void CollectionsFlushTest::storeItems(const std::string& collection,
369                                       DocNamespace ns,
370                                       int items) {
371     for (int ii = 0; ii < items; ii++) {
372         std::string key = collection + ":" + std::to_string(ii);
373         store_item(vbid, {key, ns}, "value");
374     }
375 }
376 
createCollectionAndFlush( const std::string& json, const std::string& collection, int items)377 std::string CollectionsFlushTest::createCollectionAndFlush(
378         const std::string& json, const std::string& collection, int items) {
379     VBucketPtr vb = store->getVBucket(vbid);
380     vb->updateFromManifest(json);
381     storeItems(collection, DocNamespace::Collections, items);
382     flush_vbucket_to_disk(vbid, 1 + items); // create event + items
383     return getManifest(vbid);
384 }
385 
deleteCollectionAndFlush( const std::string& json, const std::string& collection, int items)386 std::string CollectionsFlushTest::deleteCollectionAndFlush(
387         const std::string& json, const std::string& collection, int items) {
388     VBucketPtr vb = store->getVBucket(vbid);
389     storeItems(collection, DocNamespace::Collections, items);
390     vb->updateFromManifest(json);
391     flush_vbucket_to_disk(vbid, 1 + items); // del(create event) + items
392     return getManifest(vbid);
393 }
394 
completeDeletionAndFlush( const std::string& collection, int items)395 std::string CollectionsFlushTest::completeDeletionAndFlush(
396         const std::string& collection, int items) {
397     VBucketPtr vb = store->getVBucket(vbid);
398     vb->completeDeletion(collection);
399     storeItems("defaultcollection", DocNamespace::DefaultCollection, items);
400     flush_vbucket_to_disk(vbid, items); // just the items
401     return getManifest(vbid);
402 }
403 
canWrite(const std::string& jsonManifest, const std::string& collection)404 bool CollectionsFlushTest::canWrite(const std::string& jsonManifest,
405                                     const std::string& collection) {
406     Collections::VB::Manifest manifest(jsonManifest);
407     std::string key = collection + ":";
408     return manifest.lock().doesKeyContainValidCollection(
409             {key, DocNamespace::Collections});
410 }
411 
cannotWrite(const std::string& jsonManifest, const std::string& collection)412 bool CollectionsFlushTest::cannotWrite(const std::string& jsonManifest,
413                                        const std::string& collection) {
414     return !canWrite(jsonManifest, collection);
415 }
416 
417 /**
418  * Drive manifest state changes through the test's vbucket
419  *  1. Validate the flusher flushes the expected items
420  *  2. Validate the updated collections manifest changes
421  *  3. Use a validator function to check if a collection is (or is not)
422  *     writeable
423  */
collectionsFlusher(int items)424 void CollectionsFlushTest::collectionsFlusher(int items) {
425     struct testFuctions {
426         std::function<std::string()> function;
427         std::function<bool(const std::string&)> validator;
428     };
429 
430     using std::placeholders::_1;
431     // Setup the test using a vector of functions to run
432     std::vector<testFuctions> test{
433             // First 3 steps - add,delete,complete for the meat collection
434             {// 0
435              std::bind(&CollectionsFlushTest::createCollectionAndFlush,
436                        this,
437                        R"({"separator":":","uid":"0",
438                          "collections":[{"name":"$default", "uid":"0"},
439                                         {"name":"meat", "uid":"1"}]})",
440                        "meat",
441                        items),
442              std::bind(&CollectionsFlushTest::canWrite, _1, "meat")},
443 
444             {// 1
445              std::bind(&CollectionsFlushTest::deleteCollectionAndFlush,
446                        this,
447                        R"({"separator":":","uid":"0",
448                          "collections":[{"name":"$default", "uid":"0"}]})",
449                        "meat",
450                        items),
451              std::bind(&CollectionsFlushTest::cannotWrite, _1, "meat")},
452             {// 2
453              std::bind(&CollectionsFlushTest::completeDeletionAndFlush,
454                        this,
455                        "meat",
456                        items),
457              std::bind(&CollectionsFlushTest::cannotWrite, _1, "meat")},
458 
459             // Final 4 steps - add,delete,add,complete for the fruit collection
460             {// 3
461              std::bind(&CollectionsFlushTest::createCollectionAndFlush,
462                        this,
463                        R"({"separator":":","uid":"0",
464                          "collections":[{"name":"$default", "uid":"0"},
465                                         {"name":"fruit", "uid":"3"}]})",
466                        "fruit",
467                        items),
468              std::bind(&CollectionsFlushTest::canWrite, _1, "fruit")},
469             {// 4
470              std::bind(&CollectionsFlushTest::deleteCollectionAndFlush,
471                        this,
472                        R"({"separator":":","uid":"0",
473                          "collections":[{"name":"$default", "uid":"0"}]})",
474                        "fruit",
475                        items),
476              std::bind(&CollectionsFlushTest::cannotWrite, _1, "fruit")},
477             {// 5
478              std::bind(&CollectionsFlushTest::createCollectionAndFlush,
479                        this,
480                        R"({"separator":":","uid":"0",
481                          "collections":[{"name":"$default", "uid":"0"},
482                                         {"name":"fruit", "uid":"5"}]})",
483                        "fruit",
484                        items),
485              std::bind(&CollectionsFlushTest::canWrite, _1, "fruit")},
486             {// 6
487              std::bind(&CollectionsFlushTest::completeDeletionAndFlush,
488                        this,
489                        "fruit",
490                        items),
491              std::bind(&CollectionsFlushTest::canWrite, _1, "fruit")}};
492 
493     std::string m1;
494     int step = 0;
495     for (auto& f : test) {
496         auto m2 = f.function();
497         // The manifest should change for each step
498         EXPECT_NE(m1, m2);
499         EXPECT_TRUE(f.validator(m2))
500                 << "Failed step " + std::to_string(step) + " validating " + m2;
501         m1 = m2;
502         step++;
503     }
504 }
505 
TEST_F(CollectionsFlushTest, collections_flusher_no_items)506 TEST_F(CollectionsFlushTest, collections_flusher_no_items) {
507     collectionsFlusher(0);
508 }
509 
TEST_F(CollectionsFlushTest, collections_flusher_with_items)510 TEST_F(CollectionsFlushTest, collections_flusher_with_items) {
511     collectionsFlusher(3);
512 }
513 
514 class CollectionsWarmupTest : public SingleThreadedKVBucketTest {
515 public:
516     void SetUp() override {
517         // Enable collections (which will enable namespace persistence).
518         config_string += "collections_prototype_enabled=true";
519         SingleThreadedKVBucketTest::SetUp();
520         setVBucketStateAndRunPersistTask(vbid, vbucket_state_active);
521     }
522 };
523 
524 //
525 // Create a collection then create a second engine which will warmup from the
526 // persisted collection state and should have the collection accessible.
527 //
TEST_F(CollectionsWarmupTest, warmup)528 TEST_F(CollectionsWarmupTest, warmup) {
529     {
530         auto vb = store->getVBucket(vbid);
531 
532         // Add the meat collection *and* change the separator
533         vb->updateFromManifest({R"({"separator":"-+-","uid":"face1",
534               "collections":[{"name":"$default", "uid":"0"},
535                              {"name":"meat","uid":"1"}]})"});
536 
537         // Trigger a flush to disk. Flushes the meat create event and a separator
538         // changed event.
539         flush_vbucket_to_disk(vbid, 2);
540 
541         // Now we can write to beef
542         store_item(vbid, {"meat-+-beef", DocNamespace::Collections}, "value");
543         // But not dairy
544         store_item(vbid,
545                    {"dairy-+-milk", DocNamespace::Collections},
546                    "value",
547                    0,
548                    {cb::engine_errc::unknown_collection});
549 
550         flush_vbucket_to_disk(vbid, 1);
551     } // VBucketPtr scope ends
552 
553     resetEngineAndWarmup();
554 
555     // validate the manifest uid comes back
556     EXPECT_EQ(0xface1,
557               store->getVBucket(vbid)->lockCollections().getManifestUid());
558 
559     {
560         Item item({"meat-+-beef", DocNamespace::Collections},
561                   /*flags*/ 0,
562                   /*exp*/ 0,
563                   "rare",
564                   sizeof("rare"));
565         item.setVBucketId(vbid);
566         uint64_t cas;
567         EXPECT_EQ(ENGINE_SUCCESS,
568                   engine->store(cookie, &item, cas, OPERATION_SET));
569     }
570     {
571         Item item({"dairy-+-milk", DocNamespace::Collections},
572                   /*flags*/ 0,
573                   /*exp*/ 0,
574                   "skimmed",
575                   sizeof("skimmed"));
576         item.setVBucketId(vbid);
577         uint64_t cas;
578         EXPECT_EQ(ENGINE_UNKNOWN_COLLECTION,
579                   engine->store(cookie, &item, cas, OPERATION_SET));
580     }
581 }
582 
583 // When a collection is deleted - an event enters the checkpoint which does not
584 // enter the persisted seqno index - hence at the end of this test when we warm
585 // up, expect the highSeqno to be less than before the warmup.
TEST_F(CollectionsWarmupTest, MB_25381)586 TEST_F(CollectionsWarmupTest, MB_25381) {
587     int64_t highSeqno = 0;
588     {
589         auto vb = store->getVBucket(vbid);
590 
591         // Add the dairy collection *and* change the separator
592         vb->updateFromManifest({R"({"separator":"@","uid":"0",
593               "collections":[{"name":"$default", "uid":"0"},
594                              {"name":"dairy","uid":"1"}]})"});
595 
596         // Trigger a flush to disk. Flushes the dairy create event and a separator
597         // changed event.
598         flush_vbucket_to_disk(vbid, 2);
599 
600         // Now we can write to dairy
601         store_item(vbid, {"dairy@milk", DocNamespace::Collections}, "creamy");
602 
603         // Now delete the dairy collection
604         vb->updateFromManifest({R"({"separator":"@","uid":"0",
605               "collections":[{"name":"$default", "uid":"0"}]})"});
606 
607         flush_vbucket_to_disk(vbid, 2);
608 
609         // This pushes an Item which doesn't flush but has consumed a seqno
610         vb->completeDeletion("dairy");
611 
612         flush_vbucket_to_disk(vbid, 0); // 0 items but has written _local
613 
614         highSeqno = vb->getHighSeqno();
615     } // VBucketPtr scope ends
616     resetEngineAndWarmup();
617 
618     auto vb = store->getVBucket(vbid);
619     EXPECT_GT(highSeqno, vb->getHighSeqno());
620 }
621 
622 //
623 // Create a collection then create a second engine which will warmup from the
624 // persisted collection state and should have the collection accessible.
625 //
TEST_F(CollectionsWarmupTest, warmupIgnoreLogicallyDeleted)626 TEST_F(CollectionsWarmupTest, warmupIgnoreLogicallyDeleted) {
627     {
628         auto vb = store->getVBucket(vbid);
629 
630         // Add the meat collection
631         vb->updateFromManifest({R"({"separator":":","uid":"0",
632               "collections":[{"name":"$default", "uid":"0"},
633                              {"name":"meat","uid":"1"}]})"});
634 
635         // Trigger a flush to disk. Flushes the meat create event
636         flush_vbucket_to_disk(vbid, 1);
637         const int nitems = 10;
638         for (int ii = 0; ii < nitems; ii++) {
639             // Now we can write to beef
640             std::string key = "meat:" + std::to_string(ii);
641             store_item(vbid, {key, DocNamespace::Collections}, "value");
642         }
643 
644         flush_vbucket_to_disk(vbid, nitems);
645 
646         // Remove the meat collection
647         vb->updateFromManifest({R"({"separator":":","uid":"0",
648               "collections":[{"name":"$default", "uid":"0"}]})"});
649 
650         flush_vbucket_to_disk(vbid, 1);
651 
652         EXPECT_EQ(nitems, vb->ht.getNumInMemoryItems());
653     } // VBucketPtr scope ends
654 
655     resetEngineAndWarmup();
656 
657     EXPECT_EQ(0, store->getVBucket(vbid)->ht.getNumInMemoryItems());
658 }
659 
660 //
661 // Create a collection then create a second engine which will warmup from the
662 // persisted collection state and should have the collection accessible.
663 //
TEST_F(CollectionsWarmupTest, warmupIgnoreLogicallyDeletedDefault)664 TEST_F(CollectionsWarmupTest, warmupIgnoreLogicallyDeletedDefault) {
665     {
666         auto vb = store->getVBucket(vbid);
667 
668         // Add the meat collection
669         vb->updateFromManifest({R"({"separator":":","uid":"0",
670               "collections":[{"name":"$default", "uid":"0"},
671                              {"name":"meat","uid":"1"}]})"});
672 
673         // Trigger a flush to disk. Flushes the meat create event
674         flush_vbucket_to_disk(vbid, 1);
675         const int nitems = 10;
676         for (int ii = 0; ii < nitems; ii++) {
677             std::string key = "key" + std::to_string(ii);
678             store_item(vbid, {key, DocNamespace::DefaultCollection}, "value");
679         }
680 
681         flush_vbucket_to_disk(vbid, nitems);
682 
683         // Remove the default collection
684         vb->updateFromManifest({R"({"separator":":","uid":"0",
685               "collections":[{"name":"meat", "uid":"1"}]})"});
686 
687         flush_vbucket_to_disk(vbid, 1);
688 
689         EXPECT_EQ(nitems, vb->ht.getNumInMemoryItems());
690     } // VBucketPtr scope ends
691 
692     resetEngineAndWarmup();
693 
694     EXPECT_EQ(0, store->getVBucket(vbid)->ht.getNumInMemoryItems());
695 }
696 
697 class CollectionsManagerTest : public CollectionsTest {};
698 
699 /**
700  * Test checks that setCollections propagates the collection data to active
701  * vbuckets.
702  */
TEST_F(CollectionsManagerTest, basic)703 TEST_F(CollectionsManagerTest, basic) {
704     // Add some more VBuckets just so there's some iteration happening
705     const int extraVbuckets = 2;
706     for (int vb = vbid + 1; vb <= (vbid + extraVbuckets); vb++) {
707         store->setVBucketState(vb, vbucket_state_active, false);
708     }
709 
710     store->setCollections({R"({"separator": "@@","uid":"0",
711               "collections":[{"name":"$default", "uid":"0"},
712                              {"name":"meat", "uid":"1"}]})"});
713 
714     // Check all vbuckets got the collections
715     for (int vb = vbid; vb <= (vbid + extraVbuckets); vb++) {
716         auto vbp = store->getVBucket(vb);
717         EXPECT_EQ("@@", vbp->lockCollections().getSeparator());
718         EXPECT_TRUE(vbp->lockCollections().doesKeyContainValidCollection(
719                 {"meat@@bacon", DocNamespace::Collections}));
720         EXPECT_TRUE(vbp->lockCollections().doesKeyContainValidCollection(
721                 {"anykey", DocNamespace::DefaultCollection}));
722     }
723 }
724 
725 /**
726  * Test checks that setCollections propagates the collection data to active
727  * vbuckets and not the replicas
728  */
TEST_F(CollectionsManagerTest, basic2)729 TEST_F(CollectionsManagerTest, basic2) {
730     // Add some more VBuckets just so there's some iteration happening
731     const int extraVbuckets = 2;
732     // Add active and replica
733     for (int vb = vbid + 1; vb <= (vbid + extraVbuckets); vb++) {
734         if (vb & 1) {
735             store->setVBucketState(vb, vbucket_state_active, false);
736         } else {
737             store->setVBucketState(vb, vbucket_state_replica, false);
738         }
739     }
740 
741     store->setCollections({R"({"separator": "@@","uid":"0",
742               "collections":[{"name":"$default", "uid":"0"},
743                              {"name":"meat", "uid":"1"}]})"});
744 
745     // Check all vbuckets got the collections
746     for (int vb = vbid; vb <= (vbid + extraVbuckets); vb++) {
747         auto vbp = store->getVBucket(vb);
748         if (vbp->getState() == vbucket_state_active) {
749             EXPECT_EQ("@@", vbp->lockCollections().getSeparator());
750             EXPECT_TRUE(vbp->lockCollections().doesKeyContainValidCollection(
751                     {"meat@@bacon", DocNamespace::Collections}));
752             EXPECT_TRUE(vbp->lockCollections().doesKeyContainValidCollection(
753                     {"anykey", DocNamespace::DefaultCollection}));
754         } else {
755             // Replica will be in default constructed settings
756             EXPECT_EQ(Collections::DefaultSeparator,
757                       vbp->lockCollections().getSeparator());
758             EXPECT_FALSE(vbp->lockCollections().doesKeyContainValidCollection(
759                     {"meat@@bacon", DocNamespace::Collections}));
760             EXPECT_TRUE(vbp->lockCollections().doesKeyContainValidCollection(
761                     {"anykey", DocNamespace::DefaultCollection}));
762         }
763     }
764 }
765