1/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/*
3 *     Copyright 2017 Couchbase, Inc
4 *
5 *   Licensed under the Apache License, Version 2.0 (the "License");
6 *   you may not use this file except in compliance with the License.
7 *   You may obtain a copy of the License at
8 *
9 *       http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *   Unless required by applicable law or agreed to in writing, software
12 *   distributed under the License is distributed on an "AS IS" BASIS,
13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *   See the License for the specific language governing permissions and
15 *   limitations under the License.
16 */
17
18#include "checkpoint.h"
19#include "collections/manifest.h"
20#include "collections/vbucket_manifest.h"
21#include "collections/vbucket_serialised_manifest_entry.h"
22#include "ep_vb.h"
23#include "failover-table.h"
24#include "tests/module_tests/test_helpers.h"
25
26#include <cJSON_utils.h>
27
28#include <gtest/gtest.h>
29
30class MockVBManifest : public Collections::VB::Manifest {
31public:
32    MockVBManifest() : Collections::VB::Manifest({/* no collection data*/}) {
33    }
34
35    MockVBManifest(const std::string& json) : Collections::VB::Manifest(json) {
36    }
37
38    bool exists(Collections::Identifier identifier) const {
39        std::lock_guard<cb::ReaderLock> readLock(rwlock.reader());
40        return exists_UNLOCKED(identifier);
41    }
42
43    bool isOpen(Collections::Identifier identifier) const {
44        std::lock_guard<cb::ReaderLock> readLock(rwlock.reader());
45        expect_true(exists_UNLOCKED(identifier));
46        auto itr = map.find(identifier.getName());
47        return itr->second->isOpen();
48    }
49
50    bool isExclusiveOpen(Collections::Identifier identifier) const {
51        std::lock_guard<cb::ReaderLock> readLock(rwlock.reader());
52        expect_true(exists_UNLOCKED(identifier));
53        auto itr = map.find(identifier.getName());
54        return itr->second->isExclusiveOpen();
55    }
56
57    bool isDeleting(Collections::Identifier identifier) const {
58        std::lock_guard<cb::ReaderLock> readLock(rwlock.reader());
59        expect_true(exists_UNLOCKED(identifier));
60        auto itr = map.find(identifier.getName());
61        return itr->second->isDeleting();
62    }
63
64    bool isExclusiveDeleting(Collections::Identifier identifier) const {
65        std::lock_guard<cb::ReaderLock> readLock(rwlock.reader());
66        expect_true(exists_UNLOCKED(identifier));
67        auto itr = map.find(identifier.getName());
68        return itr->second->isExclusiveDeleting();
69    }
70
71    bool isOpenAndDeleting(Collections::Identifier identifier) const {
72        std::lock_guard<cb::ReaderLock> readLock(rwlock.reader());
73        expect_true(exists_UNLOCKED(identifier));
74        auto itr = map.find(identifier.getName());
75        return itr->second->isOpenAndDeleting();
76    }
77
78    size_t size() const {
79        std::lock_guard<cb::ReaderLock> readLock(rwlock.reader());
80        return map.size();
81    }
82
83    bool compareEntry(const Collections::VB::ManifestEntry& entry) const {
84        std::lock_guard<cb::ReaderLock> readLock(rwlock.reader());
85        if (exists_UNLOCKED(entry.getIdentifier())) {
86            auto itr = map.find(entry.getCollectionName());
87            const auto& myEntry = *itr->second;
88            return myEntry.getStartSeqno() == entry.getStartSeqno() &&
89                   myEntry.getEndSeqno() == entry.getEndSeqno() &&
90                   myEntry.getUid() == entry.getUid();
91        }
92        return false;
93    }
94
95    bool operator==(const MockVBManifest& rhs) const {
96        std::lock_guard<cb::ReaderLock> readLock(rwlock.reader());
97        if (rhs.size() != size()) {
98            return false;
99        }
100        // Check all collections match
101        for (const auto& e : map) {
102            if (!rhs.compareEntry(*e.second)) {
103                return false;
104            }
105        }
106
107        // finally check the separator's match
108        return rhs.separator == separator;
109    }
110
111    bool operator!=(const MockVBManifest& rhs) const {
112        return !(*this == rhs);
113    }
114
115    int64_t getGreatestEndSeqno() const {
116        std::lock_guard<cb::ReaderLock> readLock(rwlock.reader());
117        return greatestEndSeqno;
118    }
119
120    size_t getNumDeletingCollections() const {
121        std::lock_guard<cb::ReaderLock> readLock(rwlock.reader());
122        return nDeletingCollections;
123    }
124
125    bool isGreatestEndSeqnoCorrect() const {
126        std::lock_guard<cb::ReaderLock> readLock(rwlock.reader());
127        // If this is zero greatestEnd should not be a seqno
128        if (nDeletingCollections == 0) {
129            return greatestEndSeqno == StoredValue::state_collection_open;
130        }
131        return greatestEndSeqno >= 0;
132    }
133
134    bool isNumDeletingCollectionsoCorrect() const {
135        std::lock_guard<cb::ReaderLock> readLock(rwlock.reader());
136        // If this is zero greatestEnd should not be a seqno
137        if (greatestEndSeqno != StoredValue::state_collection_open) {
138            return nDeletingCollections > 0;
139        }
140        return nDeletingCollections == 0;
141    }
142
143protected:
144    bool exists_UNLOCKED(Collections::Identifier identifier) const {
145        auto itr = map.find(identifier.getName());
146        return itr != map.end() && itr->second->getUid() == identifier.getUid();
147    }
148
149    void expect_true(bool in) const {
150        if (!in) {
151            std::stringstream ss;
152            ss << *this;
153            throw std::logic_error("expect_true found false manifest:" +
154                                   ss.str());
155        }
156    }
157};
158
159/**
160 * Test class that owns an active and replica manifest.
161 * Updates applied to the active are applied to the replica by processing
162 * the active's checkpoint.
163 */
164class ActiveReplicaManifest {
165public:
166    /// Dummy callback to replace the flusher callback so we can create VBuckets
167    class DummyCB : public Callback<uint16_t> {
168    public:
169        DummyCB() {
170        }
171
172        void callback(uint16_t& dummy) {
173        }
174    };
175
176    ActiveReplicaManifest()
177        : active(),
178          replica(),
179          vbA(0,
180              vbucket_state_active,
181              global_stats,
182              checkpoint_config,
183              /*kvshard*/ nullptr,
184              /*lastSeqno*/ 0,
185              /*lastSnapStart*/ 0,
186              /*lastSnapEnd*/ 0,
187              /*table*/ nullptr,
188              std::make_shared<DummyCB>(),
189              /*newSeqnoCb*/ nullptr,
190              config,
191              VALUE_ONLY),
192          vbR(1,
193              vbucket_state_replica,
194              global_stats,
195              checkpoint_config,
196              /*kvshard*/ nullptr,
197              /*lastSeqno*/ 0,
198              /*lastSnapStart*/ 0,
199              /*lastSnapEnd*/ snapEnd,
200              /*table*/ nullptr,
201              std::make_shared<DummyCB>(),
202              /*newSeqnoCb*/ nullptr,
203              config,
204              VALUE_ONLY),
205          lastCompleteDeletionArgs({}, 0) {
206    }
207
208    ::testing::AssertionResult update(const char* json) {
209        try {
210            active.wlock().update(vbA, {json});
211        } catch (std::exception& e) {
212            return ::testing::AssertionFailure()
213                   << "Exception thrown for update with " << json
214                   << ", e.what:" << e.what();
215        }
216        queued_item manifest;
217        try {
218            manifest = applyCheckpointEventsToReplica();
219        } catch (std::exception& e) {
220            return ::testing::AssertionFailure()
221                   << "Exception thrown for replica update, e.what:"
222                   << e.what();
223        }
224        if (active != replica) {
225            return ::testing::AssertionFailure()
226                   << "active doesn't match replica active:\n"
227                   << active << " replica:\n"
228                   << replica;
229        }
230
231        auto rv = checkNumDeletingCollections();
232        if (rv != ::testing::AssertionSuccess()) {
233            return rv;
234        }
235        rv = checkGreatestEndSeqno();
236        if (rv != ::testing::AssertionSuccess()) {
237            return rv;
238        }
239
240        return checkJson(*manifest);
241    }
242
243    ::testing::AssertionResult completeDeletion(
244            Collections::Identifier identifier) {
245        try {
246            active.wlock().completeDeletion(vbA, identifier.getName());
247            lastCompleteDeletionArgs = identifier;
248        } catch (std::exception& e) {
249            return ::testing::AssertionFailure()
250                   << "Exception thrown for completeDeletion with e.what:"
251                   << e.what();
252        }
253
254        queued_item manifest;
255        try {
256            manifest = applyCheckpointEventsToReplica();
257        } catch (std::exception& e) {
258            return ::testing::AssertionFailure()
259                   << "completeDeletion: Exception thrown for replica update, "
260                      "e.what:"
261                   << e.what();
262        }
263
264        // completeDeletion adds a new item without a seqno, which closes
265        // the snapshot, re-open the snapshot so tests can continue.
266        vbR.checkpointManager->updateCurrentSnapshotEnd(snapEnd);
267        if (active != replica) {
268            return ::testing::AssertionFailure()
269                   << "completeDeletion: active doesn't match replica active:\n"
270                   << active << " replica:\n"
271                   << replica;
272        }
273        return checkJson(*manifest);
274    }
275
276    ::testing::AssertionResult doesKeyContainValidCollection(DocKey key) {
277        if (!active.lock().doesKeyContainValidCollection(key)) {
278            return ::testing::AssertionFailure() << "active failed the key";
279        } else if (!replica.lock().doesKeyContainValidCollection(key)) {
280            return ::testing::AssertionFailure() << "replica failed the key";
281        }
282        return ::testing::AssertionSuccess();
283    }
284
285    ::testing::AssertionResult isLogicallyDeleted(DocKey key, int64_t seqno) {
286        if (!active.lock().isLogicallyDeleted(key, seqno)) {
287            return ::testing::AssertionFailure()
288                   << "active failed the key seqno:" << seqno << "\n"
289                   << active;
290        } else if (!replica.lock().isLogicallyDeleted(key, seqno)) {
291            return ::testing::AssertionFailure()
292                   << "replica failed the key seqno:" << seqno << "\n"
293                   << replica;
294        }
295        return ::testing::AssertionSuccess();
296    }
297
298    bool isOpen(Collections::Identifier identifier) {
299        return active.isOpen(identifier) && replica.isOpen(identifier);
300    }
301
302    bool isDeleting(Collections::Identifier identifier) {
303        return active.isDeleting(identifier) && replica.isDeleting(identifier);
304    }
305
306    bool isExclusiveOpen(Collections::Identifier identifier) {
307        return active.isExclusiveOpen(identifier) &&
308               replica.isExclusiveOpen(identifier);
309    }
310
311    bool isExclusiveDeleting(Collections::Identifier identifier) {
312        return active.isExclusiveDeleting(identifier) &&
313               replica.isExclusiveDeleting(identifier);
314    }
315
316    bool isOpenAndDeleting(Collections::Identifier identifier) {
317        return active.isOpenAndDeleting(identifier) &&
318               replica.isOpenAndDeleting(identifier);
319    }
320
321    bool checkSize(size_t s) {
322        return active.size() == s && replica.size() == s;
323    }
324
325    VBucket& getActiveVB() {
326        return vbA;
327    }
328
329    MockVBManifest& getActiveManifest() {
330        return active;
331    }
332
333    int64_t getLastSeqno() const {
334        return lastSeqno;
335    }
336
337    ::testing::AssertionResult checkGreatestEndSeqno(int64_t expectedSeqno) {
338        if (active.getGreatestEndSeqno() != expectedSeqno) {
339            return ::testing::AssertionFailure()
340                   << "active failed expectedSeqno:" << expectedSeqno << "\n"
341                   << active;
342        } else if (replica.getGreatestEndSeqno() != expectedSeqno) {
343            return ::testing::AssertionFailure()
344                   << "replica failed expectedSeqno:" << expectedSeqno << "\n"
345                   << replica;
346        }
347        return ::testing::AssertionSuccess();
348    }
349
350    ::testing::AssertionResult checkNumDeletingCollections(size_t expected) {
351        if (active.getNumDeletingCollections() != expected) {
352            return ::testing::AssertionFailure()
353                   << "active failed expected:" << expected << "\n"
354                   << active;
355        } else if (replica.getNumDeletingCollections() != expected) {
356            return ::testing::AssertionFailure()
357                   << "replica failed expected:" << expected << "\n"
358                   << replica;
359        }
360        return ::testing::AssertionSuccess();
361    }
362
363    ::testing::AssertionResult checkNumDeletingCollections() {
364        if (!active.isNumDeletingCollectionsoCorrect()) {
365            return ::testing::AssertionFailure()
366                   << "checkNumDeletingCollections active failed " << active;
367        } else if (!replica.isNumDeletingCollectionsoCorrect()) {
368            return ::testing::AssertionFailure()
369                   << "checkNumDeletingCollections replica failed " << replica;
370        }
371        return ::testing::AssertionSuccess();
372    }
373
374    ::testing::AssertionResult checkGreatestEndSeqno() {
375        if (!active.isGreatestEndSeqnoCorrect()) {
376            return ::testing::AssertionFailure()
377                   << "checkGreatestEndSeqno active failed " << active;
378        } else if (!replica.isGreatestEndSeqnoCorrect()) {
379            return ::testing::AssertionFailure()
380                   << "checkGreatestEndSeqno replica failed " << replica;
381        }
382        return ::testing::AssertionSuccess();
383    }
384
385    static void getEventsFromCheckpoint(VBucket& vb,
386                                        std::vector<queued_item>& events) {
387        std::vector<queued_item> items;
388        vb.checkpointManager->getAllItemsForCursor(
389                CheckpointManager::pCursorName, items);
390        for (const auto& qi : items) {
391            if (qi->getOperation() == queue_op::system_event) {
392                events.push_back(qi);
393            }
394        }
395
396        if (events.empty()) {
397            throw std::logic_error("getEventsFromCheckpoint: no events in vb:" +
398                                   std::to_string(vb.getId()));
399        }
400    }
401
402    /**
403     * 1. scan the VBucketManifestTestVBucket's checkpoint for all system
404     * events.
405     * 2. for all system-events, pretend to be the DcpConsumer and call
406     *    the VBucket's manifest's replica functions on.
407     * @param replicaVB A vbucket acting as the replica, we will create/delete
408     *        collections against this VB.
409     * @param replicaManfiest The replica VB's manifest, we will create/delete
410     *         collections against this manifest.
411     *
412     * @returns the last queued_item (which would be used to create a json
413     *          manifest)
414     */
415    queued_item applyCheckpointEventsToReplica() {
416        std::vector<queued_item> events;
417        getEventsFromCheckpoint(vbA, events);
418        queued_item rv = events.back();
419        for (const auto& qi : events) {
420            lastSeqno = qi->getBySeqno();
421            if (qi->getOperation() == queue_op::system_event) {
422                auto dcpData = Collections::VB::Manifest::getSystemEventData(
423                        {qi->getData(), qi->getNBytes()});
424
425                switch (SystemEvent(qi->getFlags())) {
426                case SystemEvent::Collection: {
427                    if (qi->isDeleted()) {
428                        // A deleted create means beginDelete collection
429                        replica.wlock().replicaBeginDelete(
430                                vbR,
431                                dcpData.manifestUid,
432                                {dcpData.id.getName(), dcpData.id.getUid()},
433                                qi->getBySeqno());
434                    } else {
435                        replica.wlock().replicaAdd(
436                                vbR,
437                                dcpData.manifestUid,
438                                {dcpData.id.getName(), dcpData.id.getUid()},
439                                qi->getBySeqno());
440                    }
441                    break;
442                }
443                case SystemEvent::CollectionsSeparatorChanged: {
444                    auto dcpData = Collections::VB::Manifest::
445                            getSystemEventSeparatorData(
446                                    {qi->getData(), qi->getNBytes()});
447                    replica.wlock().replicaChangeSeparator(vbR,
448                                                           dcpData.manifestUid,
449                                                           dcpData.separator,
450                                                           qi->getBySeqno());
451                    break;
452                }
453                case SystemEvent::DeleteCollectionSoft:
454                case SystemEvent::DeleteCollectionHard:
455                    // DCP doesn't transmit these events, but to improve test
456                    // coverage call completeDeletion on the replica only in
457                    // response to these system events appearing in the
458                    // checkpoint. The data held in the system event isn't
459                    // suitable though for forming the arguments to the function
460                    // e.g. Delete hard, the serialised manifest doesn't have
461                    // the collection:rev we pass through, hence why we cache
462                    // the collection:rev data in lastCompleteDeletionArgs
463                    replica.wlock().completeDeletion(
464                            vbR, lastCompleteDeletionArgs.getName());
465                    break;
466                }
467            }
468        }
469        return rv;
470    }
471
472    /**
473     * Take SystemEvent item and obtain the JSON manifest.
474     * Next create a new/temp MockVBManifest from the JSON.
475     * Finally check that this new object is equal to the test class's active
476     *
477     * @returns gtest assertion fail (with details) or success
478     */
479    ::testing::AssertionResult checkJson(const Item& manifest) {
480        MockVBManifest newManifest(
481                Collections::VB::Manifest::serialToJson(manifest));
482        if (active != newManifest) {
483            return ::testing::AssertionFailure() << "manifest mismatch\n"
484                                                 << "generated\n"
485                                                 << newManifest << "\nvs\n"
486                                                 << active;
487        }
488        return ::testing::AssertionSuccess();
489    }
490
491    MockVBManifest active;
492    MockVBManifest replica;
493    EPStats global_stats;
494    CheckpointConfig checkpoint_config;
495    Configuration config;
496    EPVBucket vbA;
497    EPVBucket vbR;
498    int64_t lastSeqno;
499    Collections::Identifier lastCompleteDeletionArgs;
500
501    static const int64_t snapEnd{200};
502};
503
504class VBucketManifestTest : public ::testing::Test {
505public:
506    ActiveReplicaManifest manifest;
507};
508
509TEST_F(VBucketManifestTest, collectionExists) {
510    EXPECT_TRUE(manifest.update(
511            R"({"separator":":","uid":"0","collections":[{"name":"vegetable","uid":"1"}]})"));
512    EXPECT_TRUE(manifest.doesKeyContainValidCollection(
513            {"vegetable:carrot", DocNamespace::Collections}));
514    EXPECT_TRUE(manifest.isExclusiveOpen({"vegetable", 1}));
515}
516
517TEST_F(VBucketManifestTest, defaultCollectionExists) {
518    EXPECT_TRUE(manifest.doesKeyContainValidCollection(
519            {"anykey", DocNamespace::DefaultCollection}));
520    EXPECT_TRUE(manifest.update(
521            R"({"separator":":","uid":"0","collections":[]})"));
522    EXPECT_FALSE(manifest.doesKeyContainValidCollection(
523            {"anykey", DocNamespace::DefaultCollection}));
524}
525
526TEST_F(VBucketManifestTest, add_delete_in_one_update) {
527    EXPECT_TRUE(manifest.update(
528            R"({"separator":":","uid":"0",)"
529            R"("collections":[{"name":"$default","uid":"0"},)"
530            R"(               {"name":"vegetable","uid":"1"}]})"));
531    EXPECT_TRUE(manifest.isOpen({"vegetable", 1}));
532    EXPECT_TRUE(manifest.doesKeyContainValidCollection(
533            {"vegetable:cucumber", DocNamespace::Collections}));
534    EXPECT_TRUE(manifest.update(
535            R"({"separator":":","uid":"0",)"
536            R"("collections":[{"name":"$default","uid":"0"},)"
537            R"(               {"name":"vegetable","uid":"2"}]})"));
538    EXPECT_TRUE(manifest.doesKeyContainValidCollection(
539            {"vegetable:cucumber", DocNamespace::Collections}));
540    EXPECT_TRUE(manifest.isOpen({"vegetable", 2}));
541    EXPECT_TRUE(manifest.isDeleting({"vegetable", 2}));
542}
543
544TEST_F(VBucketManifestTest, updates) {
545    EXPECT_TRUE(manifest.checkSize(1));
546    EXPECT_TRUE(manifest.isExclusiveOpen({"$default", 0}));
547
548    EXPECT_TRUE(manifest.update(
549            R"({"separator":":","uid":"0",)"
550            R"("collections":[{"name":"$default","uid":"0"},)"
551            R"(               {"name":"vegetable","uid":"1"}]})"));
552    EXPECT_TRUE(manifest.checkSize(2));
553    EXPECT_TRUE(manifest.isExclusiveOpen({"vegetable", 1}));
554
555    EXPECT_TRUE(manifest.update(
556            R"({"separator":":","uid":"0",)"
557            R"("collections":[{"name":"$default","uid":"0"},)"
558            R"(               {"name":"vegetable","uid":"1"},)"
559            R"(               {"name":"fruit","uid":"2"}]})"));
560    EXPECT_TRUE(manifest.checkSize(3));
561    EXPECT_TRUE(manifest.isExclusiveOpen({"fruit", 2}));
562
563    EXPECT_TRUE(manifest.update(
564            R"({"separator":":","uid":"0",)"
565            R"("collections":[{"name":"$default","uid":"0"},)"
566            R"(               {"name":"vegetable","uid":"1"},)"
567            R"(               {"name":"fruit","uid":"2"},)"
568            R"(               {"name":"meat","uid":"3"},)"
569            R"(               {"name":"dairy","uid":"4"}]})"));
570    EXPECT_TRUE(manifest.checkSize(5));
571    EXPECT_TRUE(manifest.isExclusiveOpen({"meat", 3}));
572    EXPECT_TRUE(manifest.isExclusiveOpen({"dairy", 4}));
573}
574
575TEST_F(VBucketManifestTest, updates2) {
576    EXPECT_TRUE(manifest.update(
577            R"({"separator":":","uid":"0",)"
578            R"("collections":[{"name":"$default","uid":"0"},)"
579            R"(               {"name":"vegetable","uid":"1"},)"
580            R"(               {"name":"fruit","uid":"2"},)"
581            R"(               {"name":"meat","uid":"3"},)"
582            R"(               {"name":"dairy","uid":"4"}]})"));
583    EXPECT_TRUE(manifest.checkSize(5));
584
585    // Remove meat and dairy, size is not affected because the delete is only
586    // starting
587    EXPECT_TRUE(manifest.update(
588            R"({"separator":":","uid":"0",)"
589            R"("collections":[{"name":"$default","uid":"0"},)"
590            R"(               {"name":"vegetable","uid":"1"},)"
591            R"(               {"name":"fruit","uid":"2"}]})"));
592    EXPECT_TRUE(manifest.checkSize(5));
593    EXPECT_TRUE(manifest.isExclusiveDeleting({"meat", 3}));
594    EXPECT_TRUE(manifest.isExclusiveDeleting({"dairy", 4}));
595
596    // But vegetable is accessible, the others are locked out
597    EXPECT_TRUE(manifest.doesKeyContainValidCollection(
598            {"anykey", DocNamespace::DefaultCollection}));
599    EXPECT_TRUE(manifest.doesKeyContainValidCollection(
600            {"vegetable:carrot", DocNamespace::Collections}));
601    EXPECT_FALSE(manifest.doesKeyContainValidCollection(
602            {"dairy:milk", DocNamespace::Collections}));
603    EXPECT_FALSE(manifest.doesKeyContainValidCollection(
604            {"meat:chicken", DocNamespace::Collections}));
605}
606
607TEST_F(VBucketManifestTest, updates3) {
608    EXPECT_TRUE(manifest.update(
609            R"({"separator":":","uid":"0",)"
610            R"("collections":[{"name":"$default","uid":"0"},)"
611            R"(               {"name":"vegetable","uid":"1"},)"
612            R"(               {"name":"fruit","uid":"2"},)"
613            R"(               {"name":"meat","uid":"3"},)"
614            R"(               {"name":"dairy","uid":"4"}]})"));
615    EXPECT_TRUE(manifest.checkSize(5));
616
617    // Remove everything
618    EXPECT_TRUE(manifest.update(
619            R"({ "separator":":","uid":"0","collections":[]})"));
620    EXPECT_TRUE(manifest.checkSize(5));
621    EXPECT_TRUE(manifest.isExclusiveDeleting({"$default", 0}));
622    EXPECT_TRUE(manifest.isExclusiveDeleting({"vegetable", 1}));
623    EXPECT_TRUE(manifest.isExclusiveDeleting({"fruit", 2}));
624    EXPECT_TRUE(manifest.isExclusiveDeleting({"meat", 3}));
625    EXPECT_TRUE(manifest.isExclusiveDeleting({"dairy", 4}));
626
627    // But vegetable is accessible, the others are 'locked' out
628    EXPECT_FALSE(manifest.doesKeyContainValidCollection(
629            {"vegetable:carrot", DocNamespace::Collections}));
630    EXPECT_FALSE(manifest.doesKeyContainValidCollection(
631            {"dairy:milk", DocNamespace::Collections}));
632    EXPECT_FALSE(manifest.doesKeyContainValidCollection(
633            {"meat:chicken", DocNamespace::Collections}));
634    EXPECT_FALSE(manifest.doesKeyContainValidCollection(
635            {"fruit:apple", DocNamespace::Collections}));
636    EXPECT_FALSE(manifest.doesKeyContainValidCollection(
637            {"anykey", DocNamespace::DefaultCollection}));
638}
639
640TEST_F(VBucketManifestTest, add_beginDelete_add) {
641    // add vegetable
642    EXPECT_TRUE(manifest.update(
643            R"({"separator":":","uid":"0","collections":[{"name":"vegetable","uid":"1"}]})"));
644    auto seqno = manifest.getLastSeqno(); // seqno of the vegetable addition
645    EXPECT_TRUE(manifest.checkSize(2));
646    EXPECT_TRUE(manifest.isExclusiveOpen({"vegetable", 1}));
647    EXPECT_TRUE(manifest.doesKeyContainValidCollection(
648            {"vegetable:carrot", DocNamespace::Collections}));
649
650    // The first manifest.update has dropped default collection and added
651    // vegetable - test $default key with a seqno it could of existed with
652    EXPECT_TRUE(manifest.isLogicallyDeleted(
653            {"anykey", DocNamespace::DefaultCollection}, seqno - 1));
654    // But vegetable is still good
655    EXPECT_FALSE(manifest.isLogicallyDeleted(
656            {"vegetable:carrot", DocNamespace::Collections}, seqno));
657
658    // remove vegetable
659    EXPECT_TRUE(manifest.update(
660            R"({"separator":":","uid":"0","collections":[]})"));
661    seqno = manifest.getLastSeqno();
662    EXPECT_TRUE(manifest.checkSize(2));
663    EXPECT_TRUE(manifest.isExclusiveDeleting({"vegetable", 1}));
664    EXPECT_FALSE(manifest.doesKeyContainValidCollection(
665            {"vegetable:carrot", DocNamespace::Collections}));
666
667    // vegetable is now a deleting collection
668    EXPECT_TRUE(manifest.isLogicallyDeleted(
669            {"vegetable:carrot", DocNamespace::Collections}, seqno));
670
671    // add vegetable a second time
672    EXPECT_TRUE(manifest.update(
673            R"({"separator":":","uid":"0","collections":[{"name":"vegetable","uid":"1"}]})"));
674    auto oldSeqno = seqno;
675    auto newSeqno = manifest.getLastSeqno();
676    EXPECT_TRUE(manifest.checkSize(2));
677    EXPECT_TRUE(manifest.isOpenAndDeleting({"vegetable", 1}));
678
679    EXPECT_TRUE(manifest.doesKeyContainValidCollection(
680            {"vegetable:carrot", DocNamespace::Collections}));
681
682    // Now we expect older vegetables to be deleting and newer not to be.
683    EXPECT_FALSE(manifest.isLogicallyDeleted(
684            {"vegetable:carrot", DocNamespace::Collections}, newSeqno));
685    EXPECT_TRUE(manifest.isLogicallyDeleted(
686            {"vegetable:carrot", DocNamespace::Collections}, oldSeqno));
687}
688
689TEST_F(VBucketManifestTest, add_beginDelete_delete) {
690    // add vegetable
691    EXPECT_TRUE(manifest.update(
692            R"({"separator":":","uid":"0","collections":[{"name":"vegetable","uid":"1"}]})"));
693    EXPECT_TRUE(manifest.checkSize(2));
694    EXPECT_TRUE(manifest.checkSize(2));
695    EXPECT_TRUE(manifest.isExclusiveOpen({"vegetable", 1}));
696    EXPECT_TRUE(manifest.doesKeyContainValidCollection(
697            {"vegetable:carrot", DocNamespace::Collections}));
698
699    // remove vegetable
700    EXPECT_TRUE(manifest.update(
701            R"({"separator":":","uid":"0","collections":[]})"));
702    auto seqno = manifest.getLastSeqno();
703    EXPECT_TRUE(manifest.checkSize(2));
704    EXPECT_TRUE(manifest.isExclusiveDeleting({"vegetable", 1}));
705    EXPECT_FALSE(manifest.doesKeyContainValidCollection(
706            {"vegetable:carrot", DocNamespace::Collections}));
707    EXPECT_TRUE(manifest.isLogicallyDeleted(
708            {"vegetable:carrot", DocNamespace::Collections}, seqno));
709
710    // finally remove vegetable
711    EXPECT_TRUE(manifest.completeDeletion({"vegetable", 1}));
712    EXPECT_TRUE(manifest.checkSize(1));
713    EXPECT_FALSE(manifest.doesKeyContainValidCollection(
714            {"vegetable:carrot", DocNamespace::Collections}));
715}
716
717TEST_F(VBucketManifestTest, add_beginDelete_add_delete) {
718    // add vegetable:1
719    EXPECT_TRUE(manifest.update(
720            R"({"separator":":","uid":"0","collections":[{"name":"vegetable","uid":"1"}]})"));
721    EXPECT_TRUE(manifest.checkSize(2));
722    EXPECT_TRUE(manifest.isExclusiveOpen({"vegetable", 1}));
723    EXPECT_TRUE(manifest.doesKeyContainValidCollection(
724            {"vegetable:carrot", DocNamespace::Collections}));
725
726    // remove vegetable
727    EXPECT_TRUE(manifest.update(
728            R"({"separator":":","uid":"0","collections":[]})"));
729    EXPECT_TRUE(manifest.checkSize(2));
730    EXPECT_TRUE(manifest.isExclusiveDeleting({"vegetable", 1}));
731    EXPECT_FALSE(manifest.doesKeyContainValidCollection(
732            {"vegetable:carrot", DocNamespace::Collections}));
733
734    // add vegetable:2
735    EXPECT_TRUE(manifest.update(
736            R"({"separator":":","uid":"0","collections":[{"name":"vegetable","uid":"2"}]})"));
737    EXPECT_TRUE(manifest.checkSize(2));
738    EXPECT_TRUE(manifest.isOpenAndDeleting({"vegetable", 2}));
739
740    EXPECT_TRUE(manifest.doesKeyContainValidCollection(
741            {"vegetable:carrot", DocNamespace::Collections}));
742
743    // finally remove vegetable:1
744    EXPECT_TRUE(manifest.completeDeletion({"vegetable", 1}));
745    EXPECT_TRUE(manifest.checkSize(2));
746
747    // No longer OpenAndDeleting, now ExclusiveOpen
748    EXPECT_TRUE(manifest.isExclusiveOpen({"vegetable", 2}));
749
750    EXPECT_TRUE(manifest.doesKeyContainValidCollection(
751            {"vegetable:carrot", DocNamespace::Collections}));
752}
753
754TEST_F(VBucketManifestTest, invalidDeletes) {
755    // add vegetable
756    EXPECT_TRUE(manifest.update(
757            R"({"separator":":","uid":"0",)"
758            R"("collections":[{"name":"$default","uid":"0"},)"
759            R"(               {"name":"vegetable","uid":"1"}]})"));
760    // Delete vegetable
761    EXPECT_TRUE(manifest.update(
762            R"({"separator":":","uid":"0",)"
763            R"("collections":[{"name":"$default","uid":"0"}]})"));
764
765    // Invalid.
766    EXPECT_FALSE(manifest.completeDeletion({"unknown", 1}));
767    EXPECT_FALSE(manifest.completeDeletion({"$default", 1}));
768
769    EXPECT_TRUE(manifest.completeDeletion({"vegetable", 1}));
770
771    // Delete $default
772    EXPECT_TRUE(manifest.update(R"({"separator":":","uid":"0",)"
773                                R"("collections":[]})"));
774    // Add $default
775    EXPECT_TRUE(manifest.update(
776            R"({"separator":":","uid":"0",)"
777            R"("collections":[{"name":"$default","uid":"0"}]})"));
778    EXPECT_TRUE(manifest.completeDeletion({"$default", 1}));
779}
780
781// Check that a deleting collection doesn't keep adding system events
782TEST_F(VBucketManifestTest, doubleDelete) {
783    auto seqno = manifest.getActiveVB().getHighSeqno();
784    // add vegetable
785    EXPECT_TRUE(manifest.update(
786            R"({"separator":":","uid":"0",)"
787            R"("collections":[{"name":"$default","uid":"0"},)"
788            R"(               {"name":"vegetable","uid":"1"}]})"));
789    EXPECT_LT(seqno, manifest.getActiveVB().getHighSeqno());
790    seqno = manifest.getActiveVB().getHighSeqno();
791
792    // Apply same manifest (different revision). Nothing will be created or
793    // deleted. Apply direct to vbm, not via manifest.update as that would
794    // complain about the lack of events
795    manifest.getActiveManifest().wlock().update(
796            manifest.getActiveVB(),
797            {R"({"separator":":","uid":"0",)"
798             R"("collections":[{"name":"$default","uid":"0"},)"
799             R"(               {"name":"vegetable","uid":"1"}]})"});
800
801    EXPECT_EQ(seqno, manifest.getActiveVB().getHighSeqno());
802    seqno = manifest.getActiveVB().getHighSeqno();
803
804    // Now delete vegetable
805    EXPECT_TRUE(manifest.update(
806            R"({"separator":":","uid":"0","collections":[{"name":"$default","uid":"0"}]})"));
807
808    EXPECT_LT(seqno, manifest.getActiveVB().getHighSeqno());
809    seqno = manifest.getActiveVB().getHighSeqno();
810
811    // same again, should have be nothing created or deleted
812    manifest.getActiveManifest().wlock().update(
813            manifest.getActiveVB(),
814            {R"({"separator":":","uid":"0",)"
815             R"("collections":[{"name":"$default","uid":"0"}]})"});
816
817    EXPECT_EQ(seqno, manifest.getActiveVB().getHighSeqno());
818}
819
820// This test changes the separator and propagates to the replica (all done
821// via the noThrow helper functions).
822TEST_F(VBucketManifestTest, active_replica_separatorChanges) {
823    // Can change separator to @ as only default exists
824    EXPECT_TRUE(manifest.update(
825            R"({ "separator":"@","uid":"0","collections":[{"name":"$default","uid":"0"}]})"));
826
827    // Can change separator to / and add first collection
828    EXPECT_TRUE(manifest.update(
829            R"({ "separator":"/","uid":"0","collections":[{"name":"$default","uid":"0"},)"
830            R"(                                  {"name":"vegetable","uid":"1"}]})"));
831
832    // Cannot change separator to ## because non-default collections exist
833    EXPECT_FALSE(manifest.update(
834            R"({ "separator":"##","uid":"0","collections":[{"name":"$default","uid":"0"},)"
835            R"(                                   {"name":"vegetable","uid":"1"}]})"));
836
837    // Now just remove vegetable
838    EXPECT_TRUE(manifest.update(
839            R"({ "separator":"/","uid":"0","collections":[{"name":"$default","uid":"0"}]})"));
840
841    // vegetable still exists (isDeleting), but change to ##
842    EXPECT_TRUE(manifest.update(
843            R"({ "separator":"##","uid":"0","collections":[{"name":"$default","uid":"0"}]})"));
844
845    // Finish removal of vegetable
846    EXPECT_TRUE(manifest.completeDeletion({"vegetable", 1}));
847
848    // Can change separator as only default exists
849    EXPECT_TRUE(manifest.update(
850            R"({ "separator":"@","uid":"0","collections":[{"name":"$default","uid":"0"}]})"));
851
852    // Remove default
853    EXPECT_TRUE(manifest.update(
854            R"({ "separator":"/","uid":"0","collections":[]})"));
855
856    // $default still exists (isDeleting), so cannot change to ##
857    EXPECT_TRUE(manifest.update(
858            R"({ "separator":"##","uid":"0","collections":[{"name":"$default","uid":"0"}]})"));
859
860    EXPECT_TRUE(manifest.completeDeletion({"$default", 0}));
861
862    // Can change separator as no collection exists
863    EXPECT_TRUE(manifest.update(
864            R"({ "separator":"-=-=-=-","uid":"0","collections":[]})"));
865
866    // Add a collection and check the new separator
867    EXPECT_TRUE(manifest.update(
868            R"({ "separator":"-=-=-=-","uid":"0","collections":[{"name":"meat","uid":"3"}]})"));
869    EXPECT_TRUE(manifest.doesKeyContainValidCollection(
870            {"meat-=-=-=-bacon", DocNamespace::Collections}));
871}
872
873TEST_F(VBucketManifestTest, replica_add_remove) {
874    // add vegetable
875    EXPECT_TRUE(manifest.update(
876            R"({"separator":":","uid":"0","collections":)"
877            R"([{"name":"$default","uid":"0"},{"name":"vegetable","uid":"1"}]})"));
878
879    // add meat & dairy
880    EXPECT_TRUE(manifest.update(
881            R"({"separator":":","uid":"0","collections":)"
882            R"([{"name":"$default","uid":"0"},)"
883            R"( {"name":"vegetable","uid":"1"},)"
884            R"( {"name":"meat","uid":"3"},)"
885            R"( {"name":"dairy","uid":"4"}]})"));
886
887    // remove vegetable
888    EXPECT_TRUE(manifest.update(
889            R"({"separator":":","uid":"0","collections":)"
890            R"([{"name":"$default","uid":"0"},)"
891            R"( {"name":"meat","uid":"3"},)"
892            R"( {"name":"dairy","uid":"4"}]})"));
893
894    // remove $default
895    EXPECT_TRUE(manifest.update(
896            R"({"separator":":","uid":"0","collections":)"
897            R"([{"name":"meat","uid":"3"},)"
898            R"( {"name":"dairy","uid":"4"}]})"));
899
900    // Check we can access the remaining collections
901    EXPECT_FALSE(manifest.doesKeyContainValidCollection(
902            {"vegetable:carrot", DocNamespace::Collections}));
903    EXPECT_FALSE(manifest.doesKeyContainValidCollection(
904            {"anykey", DocNamespace::DefaultCollection}));
905    EXPECT_TRUE(manifest.doesKeyContainValidCollection(
906            {"meat:sausage", DocNamespace::Collections}));
907    EXPECT_TRUE(manifest.doesKeyContainValidCollection(
908            {"dairy:butter", DocNamespace::Collections}));
909}
910
911TEST_F(VBucketManifestTest, replica_add_remove_completeDelete) {
912    // add vegetable
913    EXPECT_TRUE(manifest.update(
914            R"({"separator":":","uid":"0","collections":[{"name":"$default","uid":"0"},)"
915            R"(                                 {"name":"vegetable","uid":"1"}]})"));
916
917    // remove vegetable
918    EXPECT_TRUE(manifest.update(
919            R"({"separator":":","uid":"0","collections":[{"name":"$default","uid":"0"}]})"));
920
921    // Finish removal of vegetable
922    EXPECT_TRUE(manifest.completeDeletion({"vegetable", 1}));
923}
924
925class VBucketManifestTestEndSeqno : public VBucketManifestTest {};
926
927TEST_F(VBucketManifestTestEndSeqno, singleAdd) {
928    EXPECT_TRUE(
929            manifest.checkGreatestEndSeqno(StoredValue::state_collection_open));
930    EXPECT_TRUE(manifest.checkNumDeletingCollections(0));
931    EXPECT_TRUE(manifest.update(
932            R"({"separator":":","uid":"0","collections":[{"name":"$default","uid":"0"},)"
933            R"(                                 {"name":"vegetable","uid":"1"}]})"));
934    EXPECT_TRUE(
935            manifest.checkGreatestEndSeqno(StoredValue::state_collection_open));
936    EXPECT_TRUE(manifest.checkNumDeletingCollections(0));
937    EXPECT_FALSE(manifest.isLogicallyDeleted(
938            {"vegetable:sprout", DocNamespace::Collections}, 1));
939}
940
941TEST_F(VBucketManifestTestEndSeqno, singleDelete) {
942    EXPECT_TRUE(
943            manifest.checkGreatestEndSeqno(StoredValue::state_collection_open));
944    EXPECT_TRUE(manifest.checkNumDeletingCollections(0));
945    EXPECT_TRUE(manifest.update( // no collections left
946            R"({"separator":":","uid":"0","collections":[]})"));
947    EXPECT_TRUE(manifest.checkGreatestEndSeqno(1));
948    EXPECT_TRUE(manifest.checkNumDeletingCollections(1));
949    EXPECT_TRUE(manifest.isLogicallyDeleted(
950            {"vegetable:sprout", DocNamespace::DefaultCollection}, 1));
951    EXPECT_FALSE(manifest.isLogicallyDeleted(
952            {"vegetable:sprout", DocNamespace::DefaultCollection}, 2));
953    EXPECT_TRUE(manifest.completeDeletion({"$default", 0}));
954    EXPECT_TRUE(
955            manifest.checkGreatestEndSeqno(StoredValue::state_collection_open));
956    EXPECT_TRUE(manifest.checkNumDeletingCollections(0));
957}
958
959TEST_F(VBucketManifestTestEndSeqno, addDeleteAdd) {
960    EXPECT_TRUE(
961            manifest.checkGreatestEndSeqno(StoredValue::state_collection_open));
962    EXPECT_TRUE(manifest.checkNumDeletingCollections(0));
963
964    // Add
965    EXPECT_TRUE(manifest.update(
966            R"({"separator":":","uid":"0","collections":[{"name":"$default","uid":"0"},)"
967            R"(                                 {"name":"vegetable","uid":"1"}]})"));
968
969    // Delete
970    EXPECT_TRUE(manifest.update(
971            R"({"separator":":","uid":"0","collections":[{"name":"$default","uid":"0"}]})"));
972
973    EXPECT_TRUE(manifest.checkGreatestEndSeqno(2));
974    EXPECT_TRUE(manifest.checkNumDeletingCollections(1));
975    EXPECT_TRUE(manifest.isLogicallyDeleted(
976            {"vegetable:sprout", DocNamespace::Collections}, 1));
977
978    EXPECT_FALSE(manifest.isLogicallyDeleted(
979            {"vegetable:sprout", DocNamespace::Collections}, 3));
980
981    EXPECT_TRUE(manifest.update(
982            R"({"separator":":","uid":"0","collections":[{"name":"$default","uid":"0"},)"
983            R"(                                 {"name":"vegetable","uid":"2"}]})"));
984
985    EXPECT_TRUE(manifest.checkGreatestEndSeqno(2));
986    EXPECT_TRUE(manifest.checkNumDeletingCollections(1));
987    EXPECT_TRUE(manifest.isLogicallyDeleted(
988            {"vegetable:sprout", DocNamespace::Collections}, 1));
989
990    EXPECT_FALSE(manifest.isLogicallyDeleted(
991            {"vegetable:sprout", DocNamespace::Collections}, 3));
992
993    EXPECT_TRUE(manifest.completeDeletion({"vegetable", 1}));
994    EXPECT_TRUE(
995            manifest.checkGreatestEndSeqno(StoredValue::state_collection_open));
996    EXPECT_TRUE(manifest.checkNumDeletingCollections(0));
997}
998
999class VBucketManifestCachingReadHandle : public VBucketManifestTest {};
1000
1001TEST_F(VBucketManifestCachingReadHandle, basic) {
1002    // Add
1003    EXPECT_TRUE(manifest.update(
1004            R"({"separator":":","uid":"0","collections":[{"name":"$default","uid":"0"},)"
1005            R"(                                 {"name":"vegetable","uid":"1"}]})"));
1006
1007    {
1008        auto rh = manifest.active.lock(
1009                {"vegetable:v1", DocNamespace::Collections});
1010        EXPECT_TRUE(rh.valid());
1011        EXPECT_EQ(DocNamespace::Collections, rh.getKey().getDocNamespace());
1012        EXPECT_STREQ("vegetable:v1",
1013                     reinterpret_cast<const char*>(rh.getKey().data()));
1014    }
1015
1016    {
1017        auto rh = manifest.active.lock({"fruit:v1", DocNamespace::Collections});
1018        EXPECT_FALSE(rh.valid());
1019
1020        // cached the key
1021        EXPECT_EQ(DocNamespace::Collections, rh.getKey().getDocNamespace());
1022        EXPECT_STREQ("fruit:v1",
1023                     reinterpret_cast<const char*>(rh.getKey().data()));
1024    }
1025    EXPECT_TRUE(manifest.update(
1026            R"({"separator":":","uid":"0","collections":[{"name":"$default","uid":"0"}]})"));
1027
1028    {
1029        auto rh = manifest.active.lock(
1030                {"vegetable:v1", DocNamespace::Collections});
1031        EXPECT_FALSE(rh.valid());
1032
1033        EXPECT_EQ(DocNamespace::Collections, rh.getKey().getDocNamespace());
1034        EXPECT_STREQ("vegetable:v1",
1035                     reinterpret_cast<const char*>(rh.getKey().data()));
1036    }
1037    {
1038        auto rh = manifest.active.lock({"fruit:v1", DocNamespace::Collections});
1039        EXPECT_FALSE(rh.valid());
1040        EXPECT_EQ(DocNamespace::Collections, rh.getKey().getDocNamespace());
1041        EXPECT_STREQ("fruit:v1",
1042                     reinterpret_cast<const char*>(rh.getKey().data()));
1043    }
1044}
1045