1 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3  *     Copyright 2017 Couchbase, Inc
4  *
5  *   Licensed under the Apache License, Version 2.0 (the "License");
6  *   you may not use this file except in compliance with the License.
7  *   You may obtain a copy of the License at
8  *
9  *       http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *   Unless required by applicable law or agreed to in writing, software
12  *   distributed under the License is distributed on an "AS IS" BASIS,
13  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *   See the License for the specific language governing permissions and
15  *   limitations under the License.
16  */
17 
18 #pragma once
19 
20 #include "collections/collections_dockey.h"
21 #include "collections/collections_types.h"
22 #include "collections/manifest.h"
23 #include "collections/vbucket_manifest_entry.h"
24 #include "systemevent.h"
25 
26 #include <platform/non_negative_counter.h>
27 #include <platform/rwlock.h>
28 #include <platform/sized_buffer.h>
29 
30 #include <mutex>
31 #include <unordered_map>
32 
33 class VBucket;
34 
35 namespace Collections {
36 namespace VB {
37 
38 /**
39  * Collections::VB::Manifest is a container for all of the collections a VBucket
40  * knows about.
41  *
42  * Each collection is represented by a Collections::VB::ManifestEntry and all of
43  * the collections are stored in an unordered_map. The map is implemented to
44  * allow look-up by collection-name without having to allocate a std::string,
45  * callers only need a cb::const_char_buffer for look-ups.
46  *
47  * The Manifest allows for an external manager to drive the lifetime of each
48  * collection - adding, begin/complete of the deletion phase.
49  *
50  * This class is intended to be thread-safe when accessed through the read
51  * or write handles (providing RAII locking).
52  *
53  * Access to the class is peformed by the ReadHandle and WriteHandle classes
54  * which perform RAII locking on the manifest's internal lock. A user of the
55  * manifest is required to hold the correct handle for the required scope to
56  * to ensure any actions they take based upon a collection's existence are
57  * consistent. The important consistency issue is the checkpoint. For example
58  * when setting a document code must first check the document's collection
59  * exists, the document must then only enter the checkpoint after the creation
60  * event for the collection and also before a delete event foe the collection.
61  * Thus the set path must obtain read access to collections and keep read access
62  * for the entire scope of the set path to ensure no other thread can interleave
63  * collection create/delete and cause an inconsistency in the checkpoint
64  * ordering.
65  */
66 class Manifest {
67 public:
68     /**
69      * RAII read locking for access to the Manifest.
70      */
71     class ReadHandle {
72     public:
ReadHandle(const Manifest& m, cb::RWLock& lock)73         ReadHandle(const Manifest& m, cb::RWLock& lock)
74             : readLock(lock), manifest(m) {
75         }
76 
ReadHandle(ReadHandle&& rhs)77         ReadHandle(ReadHandle&& rhs)
78             : readLock(std::move(rhs.readLock)), manifest(rhs.manifest) {
79         }
80 
81         /**
82          * Does the key contain a valid collection?
83          *
84          * - If the key applies to the default collection, the default
85          *   collection must exist.
86          *
87          * - If the key applies to a collection, the collection must exist and
88          *   must not be in the process of deletion.
89          */
doesKeyContainValidCollection(::DocKey key) const90         bool doesKeyContainValidCollection(::DocKey key) const {
91             return manifest.doesKeyContainValidCollection(key);
92         }
93 
94         /**
95          * Given a key and it's seqno, the manifest can determine if that key
96          * is logically deleted - that is part of a collection which is in the
97          * process of being erased.
98          *
99          * @return true if the key belongs to a deleted collection.
100          */
isLogicallyDeleted(::DocKey key, int64_t seqno) const101         bool isLogicallyDeleted(::DocKey key, int64_t seqno) const {
102             return manifest.isLogicallyDeleted(key, seqno);
103         }
104 
isLogicallyDeleted(::DocKey key, int64_t seqno, const std::string& separator) const105         bool isLogicallyDeleted(::DocKey key,
106                                 int64_t seqno,
107                                 const std::string& separator) const {
108             return manifest.isLogicallyDeleted(key, seqno, separator);
109         }
110 
111         /**
112          * Function intended for use by the KVBucketr collection's eraser code.
113          *
114          * @return if the key indicates that we've now hit the end of
115          *         a deleted collection and should call completeDeletion, then
116          *         the return value is initialised with the collection which is
117          *         to be deleted. If the key does not indicate the end, the
118          *         return value is an empty char buffer.
119          */
shouldCompleteDeletion( ::DocKey key) const120         boost::optional<cb::const_char_buffer> shouldCompleteDeletion(
121                 ::DocKey key) const {
122             return manifest.shouldCompleteDeletion(key);
123         }
124 
125         /**
126          * @returns a copy of the current separator
127          */
getSeparator() const128         std::string getSeparator() const {
129             return manifest.getSeparator();
130         }
131 
makeCollectionsDocKey(::DocKey key) const132         DocKey makeCollectionsDocKey(::DocKey key) const {
133             return Collections::DocKey::make(key, manifest.getSeparator());
134         }
135 
136         /**
137          * @returns true/false if $default exists
138          */
doesDefaultCollectionExist() const139         bool doesDefaultCollectionExist() const {
140             return manifest.doesDefaultCollectionExist();
141         }
142 
143         /**
144          * @returns true if collection is open, false if not or unknown
145          */
isCollectionOpen(cb::const_char_buffer collection) const146         bool isCollectionOpen(cb::const_char_buffer collection) const {
147             return manifest.isCollectionOpen(collection);
148         }
149 
150         /**
151          * @returns true if collection is open, false if not or unknown
152          */
isCollectionOpen(Identifier identifier) const153         bool isCollectionOpen(Identifier identifier) const {
154             return manifest.isCollectionOpen(identifier);
155         }
156 
157         /**
158          * @return true if the collection exists in the internal container
159          */
exists(cb::const_char_buffer collection) const160         bool exists(cb::const_char_buffer collection) const {
161             return manifest.exists(collection);
162         }
163 
164         /// @return the manifest UID that last updated this vb::manifest
getManifestUid() const165         uid_t getManifestUid() const {
166             return manifest.getManifestUid();
167         }
168 
169         /**
170          * Dump the manifest to std::cerr
171          */
dump() const172         void dump() const {
173             std::cerr << manifest << std::endl;
174         }
175 
176     protected:
177         friend std::ostream& operator<<(std::ostream& os,
178                                         const Manifest::ReadHandle& readHandle);
179         std::unique_lock<cb::ReaderLock> readLock;
180         const Manifest& manifest;
181     };
182 
183     /**
184      * Map from a 'string_view' to an entry.
185      * The key points to data stored in the value (which is actually a pointer
186      * to a value to remove issues where objects move).
187      * Using the string_view as the key allows faster lookups, the caller
188      * need not heap allocate.
189      */
190     using container = ::std::unordered_map<cb::const_char_buffer,
191                                            std::unique_ptr<ManifestEntry>>;
192 
193     /**
194      * CachingReadHandle provides a limited set of functions to allow various
195      * functional paths in KV-engine to perform collection 'legality' checks
196      * with minimal key scans and map lookups.
197      *
198      * The pattern is that the caller creates a CachingReadHandle and during
199      * creation of the object, the key is scanned for a collection and then
200      * an attempt to find a manifest entry is made, the result of which is
201      * stored.
202      *
203      * The caller next can check if the read handle represents a valid
204      * collection, allowing code to return 'unknown_collection'.
205      *
206      * Finally a caller can pass a seqno into the isLogicallyDeleted function
207      * to test if that seqno is a logically deleted key. The seqno should have
208      * been found by searching for the key used during in construction.
209      *
210      * Privately inherited from ReadHandle so we have a readlock/manifest
211      * without exposing the ReadHandle public methods that don't quite fit in
212      * this class.
213      */
214     class CachingReadHandle : private ReadHandle {
215     public:
CachingReadHandle(const Manifest& m, cb::RWLock& lock, ::DocKey key)216         CachingReadHandle(const Manifest& m, cb::RWLock& lock, ::DocKey key)
217             : ReadHandle(m, lock), itr(m.getManifestEntry(key)), key(key) {
218         }
219 
CachingReadHandle(const Manifest& m, cb::RWLock& lock, ::DocKey key, const std::string& separator)220         CachingReadHandle(const Manifest& m,
221                           cb::RWLock& lock,
222                           ::DocKey key,
223                           const std::string& separator)
224             : ReadHandle(m, lock), itr(m.getManifestEntry(key, separator)), key(key) {
225         }
226 
227         /**
228          * @return true if the key used in construction is associated with a
229          *         valid and open collection.
230          */
valid() const231         bool valid() const {
232             if (iteratorValid()) {
233                 return itr->second->isOpen();
234             }
235             return false;
236         }
237 
238         /**
239          * @return the key used in construction
240          */
getKey() const241         ::DocKey getKey() const {
242             return key;
243         }
244 
245         /**
246          * @return true if the seqno is logically deleted. Return of true would
247          *         mean that this seqno is below the start seqno of the
248          *         collection used in construction of this read handle instance.
249          */
isLogicallyDeleted(int64_t seqno) const250         bool isLogicallyDeleted(int64_t seqno) const {
251             return manifest.isLogicallyDeleted(itr, seqno);
252         }
253 
254         /// @return the manifest UID that last updated this vb::manifest
getManifestUid() const255         uid_t getManifestUid() const {
256             return manifest.getManifestUid();
257         }
258 
259         /**
260          * Dump the manifest to std::cerr
261          */
dump()262         void dump() {
263             std::cerr << manifest << std::endl;
264         }
265 
266     protected:
iteratorValid() const267         bool iteratorValid() const {
268             return itr != manifest.end();
269         }
270 
271         friend std::ostream& operator<<(
272                 std::ostream& os,
273                 const Manifest::CachingReadHandle& readHandle);
274 
275         /**
276          * An iterator for the key's collection, or end if the key has no valid
277          * collection.
278          */
279         container::const_iterator itr;
280 
281         /**
282          * The key used in construction of this handle.
283          */
284         ::DocKey key;
285     };
286 
287     /**
288      * RAII write locking for access and updates to the Manifest.
289      */
290     class WriteHandle {
291     public:
WriteHandle(Manifest& m, cb::RWLock& lock)292         WriteHandle(Manifest& m, cb::RWLock& lock)
293             : writeLock(lock), manifest(m) {
294         }
295 
WriteHandle(WriteHandle&& rhs)296         WriteHandle(WriteHandle&& rhs)
297             : writeLock(std::move(rhs.writeLock)), manifest(rhs.manifest) {
298         }
299 
300         /**
301          * Update from a Collections::Manifest
302          *
303          * Update compares the current collection set against the manifest and
304          * triggers collection creation and collection deletion.
305          *
306          * Creation and deletion of a collection are pushed into the VBucket and
307          * the seqno of updates is recorded in the manifest.
308          *
309          * @param vb The VBucket to update (queue data into).
310          * @param manifest The incoming manifest to compare this object with.
311          */
update(::VBucket& vb, const Collections::Manifest& newManifest)312         void update(::VBucket& vb, const Collections::Manifest& newManifest) {
313             manifest.update(vb, newManifest);
314         }
315 
316         /**
317          * Complete the deletion of a collection.
318          *
319          * Lookup the collection name and determine the deletion actions.
320          * A collection could of been added again during a background delete so
321          * completeDeletion may just update the state or fully drop all
322          * knowledge of the collection.
323          *
324          * @param vb The VBucket in which the deletion is occurring.
325          * @param collection The collection that has finished being deleted.
326          */
completeDeletion(::VBucket& vb, cb::const_char_buffer collection)327         void completeDeletion(::VBucket& vb, cb::const_char_buffer collection) {
328             manifest.completeDeletion(vb, collection);
329         }
330 
331         /**
332          * Add a collection for a replica VB, this is for receiving
333          * collection updates via DCP and the collection already has a start
334          * seqno assigned.
335          *
336          * @param vb The vbucket to add the collection to.
337          * @param manifestUid the uid of the manifest which made the change
338          * @param identifier Identifier of the new collection.
339          * @param startSeqno The start-seqno assigned to the collection.
340          */
replicaAdd(::VBucket& vb, uid_t manifestUid, Identifier identifier, int64_t startSeqno)341         void replicaAdd(::VBucket& vb,
342                         uid_t manifestUid,
343                         Identifier identifier,
344                         int64_t startSeqno) {
345             manifest.addCollection(
346                     vb, manifestUid, identifier, OptionalSeqno{startSeqno});
347         }
348 
349         /**
350          * Begin a delete collection for a replica VB, this is for receiving
351          * collection updates via DCP and the collection already has an end
352          * seqno assigned.
353          *
354          * @param vb The vbucket to begin collection deletion on.
355          * @param manifestUid the uid of the manifest which made the change
356          * @param identifier Identifier of the deleted collection.
357          * @param endSeqno The end-seqno assigned to the end collection.
358          */
replicaBeginDelete(::VBucket& vb, uid_t manifestUid, Identifier identifier, int64_t endSeqno)359         void replicaBeginDelete(::VBucket& vb,
360                                 uid_t manifestUid,
361                                 Identifier identifier,
362                                 int64_t endSeqno) {
363             manifest.beginCollectionDelete(
364                     vb, manifestUid, identifier, OptionalSeqno{endSeqno});
365         }
366 
367         /**
368          * Change the separator for a replica VB, this is for receiving
369          * collection updates via DCP and the event already has an end seqno
370          * assigned.
371          *
372          * @param vb The vbucket to begin collection deletion on.
373          * @param manifestUid the uid of the manifest which made the change
374          * @param separator The new separator.
375          * @param seqno The seqno originally assigned to the active's system
376          * event.
377          */
replicaChangeSeparator(::VBucket& vb, uid_t manifestUid, cb::const_char_buffer separator, int64_t seqno)378         void replicaChangeSeparator(::VBucket& vb,
379                                     uid_t manifestUid,
380                                     cb::const_char_buffer separator,
381                                     int64_t seqno) {
382             manifest.changeSeparator(
383                     vb, manifestUid, separator, OptionalSeqno{seqno});
384         }
385 
386         /**
387          * Dump the manifest to std::cerr
388          */
dump()389         void dump() {
390             std::cerr << manifest << std::endl;
391         }
392 
393     private:
394         std::unique_lock<cb::WriterLock> writeLock;
395         Manifest& manifest;
396     };
397 
398     friend ReadHandle;
399     friend CachingReadHandle;
400     friend WriteHandle;
401 
402     /**
403      * Construct a VBucket::Manifest from a JSON string or an empty string.
404      *
405      * Empty string allows for construction where no JSON data was found i.e.
406      * an upgrade occurred and this is the first construction of a manifest
407      * for a VBucket which has persisted data, but no manifest data. When an
408      * empty string is used, the manifest will initialise with default settings.
409      * - Default Collection enabled.
410      * - Separator defined as Collections::DefaultSeparator
411      *
412      * A non-empty string must be a valid JSON manifest that determines which
413      * collections to instantiate.
414      *
415      * @param manifest A std::string containing a JSON manifest. An empty string
416      *        indicates no manifest and is valid.
417      */
418     Manifest(const std::string& manifest);
419 
lock() const420     ReadHandle lock() const {
421         return {*this, rwlock};
422     }
423 
lock(::DocKey key) const424     CachingReadHandle lock(::DocKey key) const {
425         return {*this, rwlock, key};
426     }
427 
lock(::DocKey key, const std::string& separator) const428     CachingReadHandle lock(::DocKey key, const std::string& separator) const {
429         return {*this, rwlock, key, separator};
430     }
431 
wlock()432     WriteHandle wlock() {
433         return {*this, rwlock};
434     }
435 
436     /**
437      * Return a std::string containing a JSON representation of a
438      * VBucket::Manifest. The input is an Item previously created for an event
439      * with the value being a serialised binary blob which is turned into JSON.
440      *
441      * When the Item was created it did not have a seqno for the collection
442      * entry being modified, this function will return JSON data with the
443      * missing seqno 'patched' with the Item's seqno.
444      *
445      * @param collectionsEventItem an Item created to represent a collection
446      *        event. The value of which is converted to JSON.
447      * @return JSON representation of the Item's value.
448      */
449     static std::string serialToJson(const Item& collectionsEventItem);
450 
451     /**
452      * Get the system event collection create/deleye data from a SystemEvent
453      * Item's value. This returns the information that DCP needs to create a
454      * DCPSystemEvent packet for the create/delete.
455      *
456      * @param serialisedManifest Serialised manifest data created by
457      *        ::populateWithSerialisedData
458      * @returns SystemEventData which carries all of the data which needs to be
459      *          marshalled into a DCP system event message. Inside the returned
460      *          object maybe sized_buffer objects which point into
461      *          serialisedManifest.
462      */
463     static SystemEventData getSystemEventData(
464             cb::const_char_buffer serialisedManifest);
465 
466     /**
467      * Get the system event separator change data from a SystemEvent Item's
468      * value. This returns the information that DCP needs to create a
469      * DCPSystemEvent packet for the separator change.
470      *
471      * @param serialisedManifest Serialised manifest data created by
472      *        ::populateWithSerialisedData
473      * @returns A buffer that contains a pointer/size to the separator.
474      * @returns SystemEventSeparatorData which carries all of the data which
475      *          needs to be marshalled into a DCP system event message. Inside
476      *          the returned object maybe sized_buffer objects which point into
477      *          serialisedManifest.
478      */
479     static SystemEventSeparatorData getSystemEventSeparatorData(
480             cb::const_char_buffer serialisedManifest);
481 
482 private:
483 
484     /**
485      * Return a std::string containing a JSON representation of a
486      * VBucket::Manifest. The input data should be a previously serialised
487      * object, i.e. the input to this function is the output of
488      * populateWithSerialisedData(cb::char_buffer out)
489      *
490      * @param buffer The raw data to process.
491      * @returns std::string containing a JSON representation of the manifest
492      */
493     static std::string serialToJson(cb::const_char_buffer buffer);
494 
495     /**
496      * Update from a Collections::Manifest
497      *
498      * Update compares the current collection set against the manifest and
499      * triggers collection creation and collection deletion.
500      *
501      * Creation and deletion of a collection are pushed into the VBucket and
502      * the seqno of updates is recorded in the manifest.
503      *
504      * @param vb The VBucket to update (queue data into).
505      * @param manifest The incoming manifest to compare this object with.
506      */
507     void update(::VBucket& vb, const Collections::Manifest& manifest);
508 
509     /**
510      * Add a collection to the manifest.
511      *
512      * @param vb The vbucket to add the collection to.
513      * @param manifestUid the uid of the manifest which made the change
514      * @param identifier Identifier of the new collection.
515      * @param optionalSeqno Either a seqno to assign to the new collection or
516      *        none (none means the checkpoint will assign a seqno).
517      */
518     void addCollection(::VBucket& vb,
519                        uid_t manifestUid,
520                        Identifier identifier,
521                        OptionalSeqno optionalSeqno);
522 
523     /**
524      * Begin a delete of the collection.
525      *
526      * @param vb The vbucket to begin collection deletion on.
527      * @param manifestUid the uid of the manifest which made the change
528      * @param identifier Identifier of the deleted collection.
529      * @param revision manifest revision which started the deletion.
530      * @param optionalSeqno Either a seqno to assign to the delete of the
531      *        collection or none (none means the checkpoint assigns the seqno).
532      */
533     void beginCollectionDelete(::VBucket& vb,
534                                uid_t manifestUid,
535                                Identifier identifier,
536                                OptionalSeqno optionalSeqno);
537 
538     /**
539      * Change the separator.
540      *
541      * @param vb The vbucket to begin collection deletion on.
542      * @param manifestUid the uid of the manifest which made the change
543      * @param separator The new separator.
544      * @param optionalSeqno Either a seqno to assign to the change event or none
545      *        (none means the checkpoint assigns the seqno).
546      */
547     void changeSeparator(::VBucket& vb,
548                          uid_t manifestUid,
549                          cb::const_char_buffer separator,
550                          OptionalSeqno optionalSeqno);
551 
552     /**
553      * Complete the deletion of a collection.
554      *
555      * Lookup the collection name and determine the deletion actions.
556      * A collection could of been added again during a background delete so
557      * completeDeletion may just update the state or fully drop all knowledge of
558      * the collection.
559      *
560      * @param vb The VBucket in which the deletion is occuring.
561      * @param identifier Identifier of the collection that has finished being
562      *        deleted.
563      */
564     void completeDeletion(::VBucket& vb, cb::const_char_buffer collection);
565 
566     /**
567      * Does the key contain a valid collection?
568      *
569      * - If the key applies to the default collection, the default collection
570      *   must exist.
571      *
572      * - If the key applies to a collection, the collection must exist and must
573      *   not be in the process of deletion.
574      */
575     bool doesKeyContainValidCollection(const ::DocKey& key) const;
576 
577 
578     /**
579      * Given a key and it's seqno, the manifest can determine if that key
580      * is logically deleted - that is part of a collection which is in the
581      * process of being erased.
582      *
583      * @return true if the key belongs to a deleted collection.
584      */
585     bool isLogicallyDeleted(const ::DocKey& key, int64_t seqno) const;
586 
587     /**
588      * Perform the job of isLogicallyDeleted, but with an iterator for the
589      * manifest container instead of a key. This means no new key scan and
590      * map lookup is performed.
591      *
592      *  @return true if the seqno/entry represents a logically deleted
593      *          collection.
594      */
595     bool isLogicallyDeleted(const container::const_iterator entry,
596                             int64_t seqno) const;
597 
598     /**
599      * Variant of isLogicallyDeleted where the caller specifies the separator.
600      *
601      * @return true if the key belongs to a deleted collection.
602      */
603     bool isLogicallyDeleted(const ::DocKey& key,
604                             int64_t seqno,
605                             const std::string& separator) const;
606 
607     /**
608      * Function intended for use by the collection eraser code, checking
609      * keys/seqno in seqno order.
610      *
611      * @return if the key@seqno indicates that we've now hit the end of
612      *         a deleted collection and should call completeDeletion, then
613      *         the return value is initialised with the collection of the
614      *         key (the value to which we pass to completeDeletion). If the
615      *         key@seqno does not indicate the end, the return value is an
616      *         empty char buffer.
617      */
618     boost::optional<cb::const_char_buffer> shouldCompleteDeletion(
619             const ::DocKey& key) const;
620 
621     /**
622      * @returns the current separator
623      */
getSeparator() const624     std::string getSeparator() const {
625         return separator;
626     }
627 
628 
629     /**
630      * @returns true/false if $default exists
631      */
doesDefaultCollectionExist() const632     bool doesDefaultCollectionExist() const {
633         return defaultCollectionExists;
634     }
635 
636     /**
637      * @returns true if the collection isOpen - false if not (or doesn't exist)
638      */
isCollectionOpen(cb::const_char_buffer collection) const639     bool isCollectionOpen(cb::const_char_buffer collection) const {
640         auto itr = map.find(collection);
641         if (itr != map.end()) {
642             return itr->second->isOpen();
643         }
644         return false;
645     }
646 
647     /**
648      * @returns true if the collection isOpen - false if not (or doesn't exist)
649      */
isCollectionOpen(Identifier identifier) const650     bool isCollectionOpen(Identifier identifier) const {
651         auto itr = map.find(identifier.getName());
652         if (itr != map.end()) {
653             return itr->second->getUid() == identifier.getUid() &&
654                    itr->second->isOpen();
655         }
656         return false;
657     }
658 
659     /**
660      * @return true if the collection exists in the internal container
661      */
exists(cb::const_char_buffer collection) const662     bool exists(cb::const_char_buffer collection) const {
663         return map.count(collection) > 0;
664     }
665 
end() const666     container::const_iterator end() const {
667         return map.end();
668     }
669 
670     /**
671      * Get a manifest entry for the collection associated with the key. Can
672      * return map.end() for unknown collections.
673      */
674     container::const_iterator getManifestEntry(const ::DocKey& key) const;
675 
676     /**
677      * Get a manifest entry for the collection associated with the key. Can
678      * return map.end() for unknown collections.
679      */
680     container::const_iterator getManifestEntry(
681             const ::DocKey& key, const std::string& separator) const;
682 
683     /// @return the manifest UID that last updated this vb::manifest
getManifestUid() const684     uid_t getManifestUid() const {
685         return manifestUid;
686     }
687 
688 protected:
689     /**
690      * Add a collection entry to the manifest specifing the revision that it was
691      * seen in and the sequence number for the point in 'time' it was created.
692      *
693      * @param identifier Identifier of the collection to add.
694      * @return a non const reference to the new/updated ManifestEntry so the
695      *         caller can set the correct seqno.
696      */
697     ManifestEntry& addCollectionEntry(Identifier identifier);
698 
699     /**
700      * Add a collection entry to the manifest specifing the revision that it was
701      * seen in and the sequence number span covering it.
702      *
703      * @param identifier Identifier of the collection to add.
704      * @param startSeqno The seqno where the collection begins
705      * @param endSeqno The seqno where it ends (can be the special open marker)
706      * @return a non const reference to the new ManifestEntry so the caller can
707      *         set the correct seqno.
708      */
709     ManifestEntry& addNewCollectionEntry(Identifier identifier,
710                                          int64_t startSeqno,
711                                          int64_t endSeqno);
712 
713     /**
714      * Begin the deletion process by marking the collection entry with the seqno
715      * that represents its end.
716      *
717      * After "begin" delete a collection can be added again or fully deleted
718      * by the completeDeletion method.
719      *
720      * @param identifier Identifier of the collection to delete.
721      * @return a reference to the updated ManifestEntry
722      */
723     ManifestEntry& beginDeleteCollectionEntry(Identifier identifier);
724 
725     /**
726      * Process a Collections::Manifest to determine if collections need adding
727      * or removing.
728      *
729      * This function returns two sets of collections. Those which are being
730      * added and those which are being deleted.
731      *
732      * @param manifest The Manifest to compare with.
733      * @returns A pair of vectors containing the required changes, first
734      *          contains collections that need adding whilst second contains
735      *          those which should be deleted.
736      */
737     using processResult =
738             std::pair<std::vector<Collections::Manifest::Identifier>,
739                       std::vector<Collections::Manifest::Identifier>>;
740     processResult processManifest(const Collections::Manifest& manifest) const;
741 
742     /**
743      * Create a SystemEvent Item, the Item's value will contain data for later
744      * consumption by serialToJson
745      *
746      * @param se SystemEvent to create.
747      * @param identifier The Identifier of the collection which is changing.
748      * @param deleted If the Item should be marked deleted.
749      * @param seqno An optional sequence number. If a seqno is specified, the
750      *        returned item will have its bySeqno set to seqno.
751      *
752      * @returns unique_ptr to a new Item that represents the requested
753      *          SystemEvent.
754      */
755     std::unique_ptr<Item> createSystemEvent(SystemEvent se,
756                                             Identifier identifier,
757                                             bool deleted,
758                                             OptionalSeqno seqno) const;
759 
760     /**
761      * Create a SystemEvent Item for a Separator Changed. The Item's
762      * value will contain data for later consumption by serialToJson.
763      *
764      * @param highSeqno the current high-seqno which gets mixed into the items
765      *        key
766      * @param seqno optional-seqno, defined when event is driven by DCP
767      * @returns unique_ptr to a new Item that represents the requested
768      *          SystemEvent.
769      */
770     std::unique_ptr<Item> createSeparatorChangedEvent(
771             int64_t highSeqno,
772             OptionalSeqno seqno) const;
773 
774     /**
775      * Create an Item that carries a system event and queue it to the vb
776      * checkpoint.
777      *
778      * @param vb The vbucket onto which the Item is queued.
779      * @param se The SystemEvent to create and queue.
780      * @param identifier The Identifier of the collection being added/deleted.
781      * @param deleted If the Item created should be marked as deleted.
782      * @param seqno An optional seqno which if set will be assigned to the
783      *        system event.
784      *
785      * @returns The sequence number of the queued Item.
786      */
787     int64_t queueSystemEvent(::VBucket& vb,
788                              SystemEvent se,
789                              Identifier identifier,
790                              bool deleted,
791                              OptionalSeqno seqno) const;
792 
793     /**
794      * Create an Item that carries a CollectionsSeparatorChanged system event
795      * and queue it to the vb checkpoint.
796      *
797      * @param vb The vbucket onto which the Item is queued.
798      * @param seqno An optional seqno which if set will be assigned to the
799      *        system event
800      */
801     int64_t queueSeparatorChanged(::VBucket& vb,
802                                   const std::string& oldSeparator,
803                                   OptionalSeqno seqno) const;
804 
805     /**
806      * Obtain how many bytes of storage are needed for a serialised copy
807      * of this object including the size of the modified collection.
808      *
809      * @param collection The name of the collection being changed. It's size is
810      *        included in the returned value.
811      * @returns how many bytes will be needed to serialise the manifest and
812      *          the collection being changed.
813      */
814     size_t getSerialisedDataSize(cb::const_char_buffer collection) const;
815 
816     /**
817      * Obtain how many bytes of storage are needed for a serialised copy
818      * of this object.
819      *
820      * @returns how many bytes will be needed to serialise the manifest.
821      */
822     size_t getSerialisedDataSize() const;
823 
824     /**
825      * Populate a buffer with the serialised state of the manifest and one
826      * additional entry that is the collection being changed, i.e. the addition
827      * or deletion.
828      *
829      * @param out A buffer for the data to be written into.
830      * @param revision The Manifest revision we are processing
831      * @param identifier The Identifier of the collection being added/deleted
832      */
833     void populateWithSerialisedData(cb::char_buffer out,
834                                     Identifier identifier) const;
835 
836     /**
837      * Populate a buffer with the serialised state of this object.
838      *
839      * @param out A buffer for the data to be written into.
840      */
841     void populateWithSerialisedData(cb::char_buffer out) const;
842 
843     /**
844      * @returns the string for the given key from the cJSON object.
845      */
846     const char* getJsonEntry(cJSON* cJson, const char* key);
847 
848     /**
849      * @returns true if the separator cannot be changed.
850      */
851     bool cannotChangeSeparator() const;
852 
853     /**
854      * Update greatestEndSeqno if the seqno is larger
855      * @param seqno an endSeqno for a deleted collection
856      */
857     void trackEndSeqno(int64_t seqno);
858 
859     /**
860      * Return a string for use in throwException, returns:
861      *   "VB::Manifest::<thrower>:<error>, this:<ostream *this>"
862      *
863      * @param thrower a string for who is throwing, typically __FUNCTION__
864      * @param error a string containing the error and useful data
865      * @returns string as per description above
866      */
867     std::string getExceptionString(const std::string& thrower,
868                                    const std::string& error) const;
869 
870     /**
871      * throw exception with the following error string:
872      *   "VB::Manifest::<thrower>:<error>, this:<ostream *this>"
873      *
874      * @param thrower a string for who is throwing, typically __FUNCTION__
875      * @param error a string containing the error and useful data
876      * @throws exception
877      */
878     template <class exception>
throwException(const std::string& thrower, const std::string& error) const879     [[noreturn]] void throwException(const std::string& thrower,
880                                      const std::string& error) const {
881         throw exception(getExceptionString(thrower, error));
882     }
883 
884     /**
885      * The current set of collections
886      */
887     container map;
888 
889     /**
890      * Does the current set contain the default collection?
891      */
892     bool defaultCollectionExists;
893 
894     /**
895      * The collection separator
896      */
897     std::string separator;
898 
899     /**
900      * A key below this seqno might be logically deleted and should be checked
901      * against the manifest. This member will be set to
902      * StoredValue::state_collection_open when no collections are being deleted
903      * and the greatestEndSeqno has no use (all collections exclusively open)
904      */
905     int64_t greatestEndSeqno;
906 
907     /**
908      * The manifest tracks how many collections are being deleted so we know
909      * when greatestEndSeqno can be set to StoredValue::state_collection_open
910      */
911     cb::NonNegativeCounter<size_t, cb::ThrowExceptionUnderflowPolicy>
912             nDeletingCollections;
913 
914     /// The manifest UID which updated this vb::manifest
915     uid_t manifestUid;
916 
917     /**
918      * shared lock to allow concurrent readers and safe updates
919      */
920     mutable cb::RWLock rwlock;
921 
922     friend std::ostream& operator<<(std::ostream& os, const Manifest& manifest);
923 };
924 
925 /// Note that the VB::Manifest << operator does not obtain the rwlock
926 /// it is used internally in the object for exception string generation so must
927 /// not double lock.
928 std::ostream& operator<<(std::ostream& os, const Manifest& manifest);
929 
930 /// This is the locked version for printing the manifest
931 std::ostream& operator<<(std::ostream& os,
932                          const Manifest::ReadHandle& readHandle);
933 
934 } // end namespace VB
935 } // end namespace Collections
936