18bf86d8fSJim Walker/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
28bf86d8fSJim Walker/*
38bf86d8fSJim Walker *     Copyright 2017 Couchbase, Inc
48bf86d8fSJim Walker *
58bf86d8fSJim Walker *   Licensed under the Apache License, Version 2.0 (the "License");
68bf86d8fSJim Walker *   you may not use this file except in compliance with the License.
78bf86d8fSJim Walker *   You may obtain a copy of the License at
88bf86d8fSJim Walker *
98bf86d8fSJim Walker *       http://www.apache.org/licenses/LICENSE-2.0
108bf86d8fSJim Walker *
118bf86d8fSJim Walker *   Unless required by applicable law or agreed to in writing, software
128bf86d8fSJim Walker *   distributed under the License is distributed on an "AS IS" BASIS,
138bf86d8fSJim Walker *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
148bf86d8fSJim Walker *   See the License for the specific language governing permissions and
158bf86d8fSJim Walker *   limitations under the License.
168bf86d8fSJim Walker */
178bf86d8fSJim Walker
188bf86d8fSJim Walker#pragma once
198bf86d8fSJim Walker
208bf86d8fSJim Walker#include "collections/collections_dockey.h"
218bf86d8fSJim Walker#include "collections/collections_types.h"
228bf86d8fSJim Walker#include "collections/manifest.h"
238bf86d8fSJim Walker#include "collections/vbucket_manifest_entry.h"
2496803ff8SJim Walker#include "systemevent.h"
2596803ff8SJim Walker
26ea1d84e5SJim Walker#include <platform/non_negative_counter.h>
27401bc1dbSTrond Norbye#include <platform/rwlock.h>
28ea1d84e5SJim Walker#include <platform/sized_buffer.h>
298bf86d8fSJim Walker
308bf86d8fSJim Walker#include <mutex>
318bf86d8fSJim Walker#include <unordered_map>
328bf86d8fSJim Walker
3396803ff8SJim Walkerclass VBucket;
3496803ff8SJim Walker
358bf86d8fSJim Walkernamespace Collections {
368bf86d8fSJim Walkernamespace VB {
378bf86d8fSJim Walker
388bf86d8fSJim Walker/**
398bf86d8fSJim Walker * Collections::VB::Manifest is a container for all of the collections a VBucket
408bf86d8fSJim Walker * knows about.
418bf86d8fSJim Walker *
428bf86d8fSJim Walker * Each collection is represented by a Collections::VB::ManifestEntry and all of
438bf86d8fSJim Walker * the collections are stored in an unordered_map. The map is implemented to
44d7615449SJim Walker * allow look-up by collection-name without having to allocate a std::string,
45d7615449SJim Walker * callers only need a cb::const_char_buffer for look-ups.
468bf86d8fSJim Walker *
478bf86d8fSJim Walker * The Manifest allows for an external manager to drive the lifetime of each
488bf86d8fSJim Walker * collection - adding, begin/complete of the deletion phase.
498bf86d8fSJim Walker *
50d7615449SJim Walker * This class is intended to be thread-safe when accessed through the read
51d7615449SJim Walker * or write handles (providing RAII locking).
528bf86d8fSJim Walker *
53d7615449SJim Walker * Access to the class is peformed by the ReadHandle and WriteHandle classes
54d7615449SJim Walker * which perform RAII locking on the manifest's internal lock. A user of the
55d7615449SJim Walker * manifest is required to hold the correct handle for the required scope to
56d7615449SJim Walker * to ensure any actions they take based upon a collection's existence are
57d7615449SJim Walker * consistent. The important consistency issue is the checkpoint. For example
58d7615449SJim Walker * when setting a document code must first check the document's collection
59d7615449SJim Walker * exists, the document must then only enter the checkpoint after the creation
60d7615449SJim Walker * event for the collection and also before a delete event foe the collection.
61d7615449SJim Walker * Thus the set path must obtain read access to collections and keep read access
62d7615449SJim Walker * for the entire scope of the set path to ensure no other thread can interleave
63d7615449SJim Walker * collection create/delete and cause an inconsistency in the checkpoint
64d7615449SJim Walker * ordering.
658bf86d8fSJim Walker */
668bf86d8fSJim Walkerclass Manifest {
678bf86d8fSJim Walkerpublic:
68d7615449SJim Walker    /**
69d7615449SJim Walker     * RAII read locking for access to the Manifest.
70d7615449SJim Walker     */
71d7615449SJim Walker    class ReadHandle {
72d7615449SJim Walker    public:
73d7615449SJim Walker        ReadHandle(const Manifest& m, cb::RWLock& lock)
74d7615449SJim Walker            : readLock(lock), manifest(m) {
75d7615449SJim Walker        }
76d7615449SJim Walker
77d7615449SJim Walker        ReadHandle(ReadHandle&& rhs)
78d7615449SJim Walker            : readLock(std::move(rhs.readLock)), manifest(rhs.manifest) {
79d7615449SJim Walker        }
80d7615449SJim Walker
81d7615449SJim Walker        /**
82d7615449SJim Walker         * Does the key contain a valid collection?
83d7615449SJim Walker         *
84d7615449SJim Walker         * - If the key applies to the default collection, the default
85d7615449SJim Walker         *   collection must exist.
86d7615449SJim Walker         *
87d7615449SJim Walker         * - If the key applies to a collection, the collection must exist and
88d7615449SJim Walker         *   must not be in the process of deletion.
89d7615449SJim Walker         */
906c282c2cSJim Walker        bool doesKeyContainValidCollection(::DocKey key) const {
91d7615449SJim Walker            return manifest.doesKeyContainValidCollection(key);
92d7615449SJim Walker        }
93d7615449SJim Walker
94fedefdb1SJim Walker        /**
9555e05834SJim Walker         * Given a key and it's seqno, the manifest can determine if that key
9655e05834SJim Walker         * is logically deleted - that is part of a collection which is in the
9755e05834SJim Walker         * process of being erased.
9855e05834SJim Walker         *
9955e05834SJim Walker         * @return true if the key belongs to a deleted collection.
100fedefdb1SJim Walker         */
10155e05834SJim Walker        bool isLogicallyDeleted(::DocKey key, int64_t seqno) const {
10255e05834SJim Walker            return manifest.isLogicallyDeleted(key, seqno);
103fedefdb1SJim Walker        }
104fedefdb1SJim Walker
105dea18910SJim Walker        bool isLogicallyDeleted(::DocKey key,
106dea18910SJim Walker                                int64_t seqno,
107dea18910SJim Walker                                const std::string& separator) const {
108dea18910SJim Walker            return manifest.isLogicallyDeleted(key, seqno, separator);
109dea18910SJim Walker        }
110dea18910SJim Walker
111a33e0978SJim Walker        /**
112a33e0978SJim Walker         * Function intended for use by the KVBucketr collection's eraser code.
113a33e0978SJim Walker         *
114a33e0978SJim Walker         * @return if the key indicates that we've now hit the end of
115a33e0978SJim Walker         *         a deleted collection and should call completeDeletion, then
116a33e0978SJim Walker         *         the return value is initialised with the collection which is
117a33e0978SJim Walker         *         to be deleted. If the key does not indicate the end, the
118a33e0978SJim Walker         *         return value is an empty char buffer.
119a33e0978SJim Walker         */
120a33e0978SJim Walker        boost::optional<cb::const_char_buffer> shouldCompleteDeletion(
121a33e0978SJim Walker                ::DocKey key) const {
122a33e0978SJim Walker            return manifest.shouldCompleteDeletion(key);
123a33e0978SJim Walker        }
124a33e0978SJim Walker
1254885256fSJim Walker        /**
126042bf504SJim Walker         * @returns a copy of the current separator
1274885256fSJim Walker         */
1284885256fSJim Walker        std::string getSeparator() const {
1294885256fSJim Walker            return manifest.getSeparator();
1304885256fSJim Walker        }
1314885256fSJim Walker
13214fcb066SJim Walker        DocKey makeCollectionsDocKey(::DocKey key) const {
13314fcb066SJim Walker            return Collections::DocKey::make(key, manifest.getSeparator());
13414fcb066SJim Walker        }
13514fcb066SJim Walker
136042bf504SJim Walker        /**
137042bf504SJim Walker         * @returns true/false if $default exists
138042bf504SJim Walker         */
139042bf504SJim Walker        bool doesDefaultCollectionExist() const {
140042bf504SJim Walker            return manifest.doesDefaultCollectionExist();
141042bf504SJim Walker        }
142042bf504SJim Walker
143042bf504SJim Walker        /**
1445cf47c05SJim Walker         * @returns true if collection is open, false if not or unknown
145042bf504SJim Walker         */
1465cf47c05SJim Walker        bool isCollectionOpen(cb::const_char_buffer collection) const {
1475cf47c05SJim Walker            return manifest.isCollectionOpen(collection);
148042bf504SJim Walker        }
149042bf504SJim Walker
1501f8437cfSJim Walker        /**
1511f8437cfSJim Walker         * @returns true if collection is open, false if not or unknown
1521f8437cfSJim Walker         */
1531f8437cfSJim Walker        bool isCollectionOpen(Identifier identifier) const {
1541f8437cfSJim Walker            return manifest.isCollectionOpen(identifier);
1551f8437cfSJim Walker        }
1561f8437cfSJim Walker
157a33e0978SJim Walker        /**
158a33e0978SJim Walker         * @return true if the collection exists in the internal container
159a33e0978SJim Walker         */
160a33e0978SJim Walker        bool exists(cb::const_char_buffer collection) const {
161a33e0978SJim Walker            return manifest.exists(collection);
162a33e0978SJim Walker        }
163a33e0978SJim Walker
164f9444d17SJim Walker        /// @return the manifest UID that last updated this vb::manifest
165f9444d17SJim Walker        uid_t getManifestUid() const {
166f9444d17SJim Walker            return manifest.getManifestUid();
167f9444d17SJim Walker        }
168f9444d17SJim Walker
1699e4dcf1fSJim Walker        /**
1709e4dcf1fSJim Walker         * Dump the manifest to std::cerr
1719e4dcf1fSJim Walker         */
1726c282c2cSJim Walker        void dump() const {
1739e4dcf1fSJim Walker            std::cerr << manifest << std::endl;
1749e4dcf1fSJim Walker        }
1759e4dcf1fSJim Walker
176a33e0978SJim Walker    protected:
1779e4dcf1fSJim Walker        friend std::ostream& operator<<(std::ostream& os,
1789e4dcf1fSJim Walker                                        const Manifest::ReadHandle& readHandle);
179d7615449SJim Walker        std::unique_lock<cb::ReaderLock> readLock;
180d7615449SJim Walker        const Manifest& manifest;
181d7615449SJim Walker    };
182d7615449SJim Walker
183c66ee788SJim Walker    /**
184c66ee788SJim Walker     * Map from a 'string_view' to an entry.
185c66ee788SJim Walker     * The key points to data stored in the value (which is actually a pointer
186c66ee788SJim Walker     * to a value to remove issues where objects move).
187c66ee788SJim Walker     * Using the string_view as the key allows faster lookups, the caller
188c66ee788SJim Walker     * need not heap allocate.
189c66ee788SJim Walker     */
190c66ee788SJim Walker    using container = ::std::unordered_map<cb::const_char_buffer,
191c66ee788SJim Walker                                           std::unique_ptr<ManifestEntry>>;
192c66ee788SJim Walker
193c66ee788SJim Walker    /**
194c66ee788SJim Walker     * CachingReadHandle provides a limited set of functions to allow various
195c66ee788SJim Walker     * functional paths in KV-engine to perform collection 'legality' checks
196c66ee788SJim Walker     * with minimal key scans and map lookups.
197c66ee788SJim Walker     *
198c66ee788SJim Walker     * The pattern is that the caller creates a CachingReadHandle and during
199c66ee788SJim Walker     * creation of the object, the key is scanned for a collection and then
200c66ee788SJim Walker     * an attempt to find a manifest entry is made, the result of which is
201c66ee788SJim Walker     * stored.
202c66ee788SJim Walker     *
203c66ee788SJim Walker     * The caller next can check if the read handle represents a valid
204c66ee788SJim Walker     * collection, allowing code to return 'unknown_collection'.
205c66ee788SJim Walker     *
206c66ee788SJim Walker     * Finally a caller can pass a seqno into the isLogicallyDeleted function
207c66ee788SJim Walker     * to test if that seqno is a logically deleted key. The seqno should have
208c66ee788SJim Walker     * been found by searching for the key used during in construction.
209c66ee788SJim Walker     *
210c66ee788SJim Walker     * Privately inherited from ReadHandle so we have a readlock/manifest
211c66ee788SJim Walker     * without exposing the ReadHandle public methods that don't quite fit in
212c66ee788SJim Walker     * this class.
213c66ee788SJim Walker     */
214c66ee788SJim Walker    class CachingReadHandle : private ReadHandle {
215c66ee788SJim Walker    public:
216c66ee788SJim Walker        CachingReadHandle(const Manifest& m, cb::RWLock& lock, ::DocKey key)
217c66ee788SJim Walker            : ReadHandle(m, lock), itr(m.getManifestEntry(key)), key(key) {
218c66ee788SJim Walker        }
219c66ee788SJim Walker
220dea18910SJim Walker        CachingReadHandle(const Manifest& m,
221dea18910SJim Walker                          cb::RWLock& lock,
222dea18910SJim Walker                          ::DocKey key,
223dea18910SJim Walker                          const std::string& separator)
224dea18910SJim Walker            : ReadHandle(m, lock), itr(m.getManifestEntry(key, separator)), key(key) {
225dea18910SJim Walker        }
226dea18910SJim Walker
227c66ee788SJim Walker        /**
228c66ee788SJim Walker         * @return true if the key used in construction is associated with a
229c66ee788SJim Walker         *         valid and open collection.
230c66ee788SJim Walker         */
231c66ee788SJim Walker        bool valid() const {
232c66ee788SJim Walker            if (iteratorValid()) {
233c66ee788SJim Walker                return itr->second->isOpen();
234c66ee788SJim Walker            }
235c66ee788SJim Walker            return false;
236c66ee788SJim Walker        }
237c66ee788SJim Walker
238c66ee788SJim Walker        /**
239c66ee788SJim Walker         * @return the key used in construction
240c66ee788SJim Walker         */
241c66ee788SJim Walker        ::DocKey getKey() const {
242c66ee788SJim Walker            return key;
243c66ee788SJim Walker        }
244c66ee788SJim Walker
245c66ee788SJim Walker        /**
246c66ee788SJim Walker         * @return true if the seqno is logically deleted. Return of true would
247c66ee788SJim Walker         *         mean that this seqno is below the start seqno of the
248c66ee788SJim Walker         *         collection used in construction of this read handle instance.
249c66ee788SJim Walker         */
250c66ee788SJim Walker        bool isLogicallyDeleted(int64_t seqno) const {
251c66ee788SJim Walker            return manifest.isLogicallyDeleted(itr, seqno);
252c66ee788SJim Walker        }
253c66ee788SJim Walker
254f9444d17SJim Walker        /// @return the manifest UID that last updated this vb::manifest
255f9444d17SJim Walker        uid_t getManifestUid() const {
256f9444d17SJim Walker            return manifest.getManifestUid();
257f9444d17SJim Walker        }
258f9444d17SJim Walker
259c66ee788SJim Walker        /**
260c66ee788SJim Walker         * Dump the manifest to std::cerr
261c66ee788SJim Walker         */
262c66ee788SJim Walker        void dump() {
263c66ee788SJim Walker            std::cerr << manifest << std::endl;
264c66ee788SJim Walker        }
265c66ee788SJim Walker
266c66ee788SJim Walker    protected:
267c66ee788SJim Walker        bool iteratorValid() const {
268c66ee788SJim Walker            return itr != manifest.end();
269c66ee788SJim Walker        }
270c66ee788SJim Walker
271c66ee788SJim Walker        friend std::ostream& operator<<(
272c66ee788SJim Walker                std::ostream& os,
273c66ee788SJim Walker                const Manifest::CachingReadHandle& readHandle);
274c66ee788SJim Walker
275c66ee788SJim Walker        /**
276c66ee788SJim Walker         * An iterator for the key's collection, or end if the key has no valid
277c66ee788SJim Walker         * collection.
278c66ee788SJim Walker         */
279c66ee788SJim Walker        container::const_iterator itr;
280c66ee788SJim Walker
281c66ee788SJim Walker        /**
282c66ee788SJim Walker         * The key used in construction of this handle.
283c66ee788SJim Walker         */
284c66ee788SJim Walker        ::DocKey key;
285c66ee788SJim Walker    };
286c66ee788SJim Walker
287d7615449SJim Walker    /**
288d7615449SJim Walker     * RAII write locking for access and updates to the Manifest.
289d7615449SJim Walker     */
290d7615449SJim Walker    class WriteHandle {
291d7615449SJim Walker    public:
292d7615449SJim Walker        WriteHandle(Manifest& m, cb::RWLock& lock)
293d7615449SJim Walker            : writeLock(lock), manifest(m) {
294d7615449SJim Walker        }
295d7615449SJim Walker
296d7615449SJim Walker        WriteHandle(WriteHandle&& rhs)
297d7615449SJim Walker            : writeLock(std::move(rhs.writeLock)), manifest(rhs.manifest) {
298d7615449SJim Walker        }
299d7615449SJim Walker
300d7615449SJim Walker        /**
301d7615449SJim Walker         * Update from a Collections::Manifest
302d7615449SJim Walker         *
303d7615449SJim Walker         * Update compares the current collection set against the manifest and
304d7615449SJim Walker         * triggers collection creation and collection deletion.
305d7615449SJim Walker         *
306d7615449SJim Walker         * Creation and deletion of a collection are pushed into the VBucket and
307d7615449SJim Walker         * the seqno of updates is recorded in the manifest.
308d7615449SJim Walker         *
309d7615449SJim Walker         * @param vb The VBucket to update (queue data into).
310d7615449SJim Walker         * @param manifest The incoming manifest to compare this object with.
311d7615449SJim Walker         */
312d7615449SJim Walker        void update(::VBucket& vb, const Collections::Manifest& newManifest) {
313d7615449SJim Walker            manifest.update(vb, newManifest);
314d7615449SJim Walker        }
315d7615449SJim Walker
316d7615449SJim Walker        /**
317d7615449SJim Walker         * Complete the deletion of a collection.
318d7615449SJim Walker         *
319d7615449SJim Walker         * Lookup the collection name and determine the deletion actions.
320d7615449SJim Walker         * A collection could of been added again during a background delete so
321d7615449SJim Walker         * completeDeletion may just update the state or fully drop all
322d7615449SJim Walker         * knowledge of the collection.
323d7615449SJim Walker         *
324a33e0978SJim Walker         * @param vb The VBucket in which the deletion is occurring.
325a33e0978SJim Walker         * @param collection The collection that has finished being deleted.
326d7615449SJim Walker         */
327a33e0978SJim Walker        void completeDeletion(::VBucket& vb, cb::const_char_buffer collection) {
328a33e0978SJim Walker            manifest.completeDeletion(vb, collection);
329d7615449SJim Walker        }
330d7615449SJim Walker
331cc92fc76SJim Walker        /**
332cc92fc76SJim Walker         * Add a collection for a replica VB, this is for receiving
333cc92fc76SJim Walker         * collection updates via DCP and the collection already has a start
334cc92fc76SJim Walker         * seqno assigned.
335cc92fc76SJim Walker         *
336cc92fc76SJim Walker         * @param vb The vbucket to add the collection to.
337cd24b1b3SJim Walker         * @param manifestUid the uid of the manifest which made the change
3389160b844SJim Walker         * @param identifier Identifier of the new collection.
339cc92fc76SJim Walker         * @param startSeqno The start-seqno assigned to the collection.
340cc92fc76SJim Walker         */
341cc92fc76SJim Walker        void replicaAdd(::VBucket& vb,
342cd24b1b3SJim Walker                        uid_t manifestUid,
3439160b844SJim Walker                        Identifier identifier,
344cc92fc76SJim Walker                        int64_t startSeqno) {
345cd24b1b3SJim Walker            manifest.addCollection(
346cd24b1b3SJim Walker                    vb, manifestUid, identifier, OptionalSeqno{startSeqno});
347cc92fc76SJim Walker        }
348cc92fc76SJim Walker
349cc92fc76SJim Walker        /**
350cc92fc76SJim Walker         * Begin a delete collection for a replica VB, this is for receiving
351cc92fc76SJim Walker         * collection updates via DCP and the collection already has an end
352cc92fc76SJim Walker         * seqno assigned.
353cc92fc76SJim Walker         *
354cc92fc76SJim Walker         * @param vb The vbucket to begin collection deletion on.
355cd24b1b3SJim Walker         * @param manifestUid the uid of the manifest which made the change
3569160b844SJim Walker         * @param identifier Identifier of the deleted collection.
357cc92fc76SJim Walker         * @param endSeqno The end-seqno assigned to the end collection.
358cc92fc76SJim Walker         */
359cc92fc76SJim Walker        void replicaBeginDelete(::VBucket& vb,
360cd24b1b3SJim Walker                                uid_t manifestUid,
3619160b844SJim Walker                                Identifier identifier,
362cc92fc76SJim Walker                                int64_t endSeqno) {
363cc92fc76SJim Walker            manifest.beginCollectionDelete(
364cd24b1b3SJim Walker                    vb, manifestUid, identifier, OptionalSeqno{endSeqno});
365cc92fc76SJim Walker        }
366cc92fc76SJim Walker
367cc92fc76SJim Walker        /**
368cc92fc76SJim Walker         * Change the separator for a replica VB, this is for receiving
369cc92fc76SJim Walker         * collection updates via DCP and the event already has an end seqno
370cc92fc76SJim Walker         * assigned.
371cc92fc76SJim Walker         *
372cc92fc76SJim Walker         * @param vb The vbucket to begin collection deletion on.
373cd24b1b3SJim Walker         * @param manifestUid the uid of the manifest which made the change
374cc92fc76SJim Walker         * @param separator The new separator.
3759e4dcf1fSJim Walker         * @param seqno The seqno originally assigned to the active's system
376cc92fc76SJim Walker         * event.
377cc92fc76SJim Walker         */
378cc92fc76SJim Walker        void replicaChangeSeparator(::VBucket& vb,
379cd24b1b3SJim Walker