1/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/*
3 *     Copyright 2016 Couchbase, Inc
4 *
5 *   Licensed under the Apache License, Version 2.0 (the "License");
6 *   you may not use this file except in compliance with the License.
7 *   You may obtain a copy of the License at
8 *
9 *       http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *   Unless required by applicable law or agreed to in writing, software
12 *   distributed under the License is distributed on an "AS IS" BASIS,
13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *   See the License for the specific language governing permissions and
15 *   limitations under the License.
16 */
17
18/*
19 * Unit tests for the EPBucket class.
20 */
21
22#pragma once
23
24#include "fakes/fake_executorpool.h"
25#include "kv_bucket_test.h"
26#include <libcouchstore/couch_db.h>
27#include <nlohmann/json.hpp>
28
29struct dcp_message_producers;
30class EPBucket;
31class MockActiveStreamWithOverloadedRegisterCursor;
32class MockDcpMessageProducers;
33class MockDcpProducer;
34
35/*
36 * A subclass of KVBucketTest which uses a fake ExecutorPool,
37 * which will not spawn ExecutorThreads and hence not run any tasks
38 * automatically in the background. All tasks must be manually run().
39 */
40class SingleThreadedKVBucketTest : public KVBucketTest {
41public:
42    /*
43     * Run the next task from the taskQ
44     * The task must match the expectedTaskName parameter
45     */
46    std::chrono::steady_clock::time_point runNextTask(
47            TaskQueue& taskQ, const std::string& expectedTaskName);
48
49    /*
50     * Run the next task from the taskQ
51     */
52    std::chrono::steady_clock::time_point runNextTask(TaskQueue& taskQ);
53
54    /*
55     * DCP helper. Create a MockDcpProducer configured with (or without)
56     * collections and/or delete_times enabled
57     * @param cookie cookie to associate with the new producer
58     * @param deleteTime yes/no - enable/disable delete times
59     */
60    std::shared_ptr<MockDcpProducer> createDcpProducer(
61            const void* cookie,
62            IncludeDeleteTime deleteTime);
63
64    /*
65     * DCP helper.
66     * Notify and step the given producer
67     * @param expectedOp once stepped we expect to see this DCP opcode produced
68     * @param fromMemory if false then step a backfill
69     */
70    void notifyAndStepToCheckpoint(
71            MockDcpProducer& producer,
72            MockDcpMessageProducers& producers,
73            cb::mcbp::ClientOpcode expectedOp =
74                    cb::mcbp::ClientOpcode::DcpSnapshotMarker,
75            bool fromMemory = true);
76
77    /*
78     * DCP helper.
79     * Run the active-checkpoint processor task for the given producer
80     * @param producer The producer whose task will be ran
81     * @param producers The dcp callbacks
82     */
83    void runCheckpointProcessor(MockDcpProducer& producer,
84                                dcp_message_producers& producers);
85
86    /*
87     * DCP helper - Run the backfill tasks
88     */
89    void runBackfill();
90
91    /**
92     * Create a DCP stream on the producer for this->vbid
93     */
94    void createDcpStream(MockDcpProducer& producer);
95
96    /**
97     * Create a DCP stream on the producer for vbid
98     */
99    void createDcpStream(MockDcpProducer& producer, Vbid vbid);
100
101    /**
102     * Run the compaction task
103     * @param purgeBeforeTime purge tombstones with timestamps less than this
104     * @param purgeBeforeSeq purge tombstones with seqnos less than this
105     */
106    void runCompaction(uint64_t purgeBeforeTime = 0,
107                       uint64_t purgeBeforeSeq = 0);
108
109    /**
110     * Run the task responsible for iterating the documents and erasing them
111     * For persistent buckets integrated into compaction.
112     * For ephemeral buckets integrated into stale item removal task
113     */
114    void runCollectionsEraser();
115
116protected:
117    void SetUp() override;
118
119    void TearDown() override;
120
121    /**
122     * Change the vbucket state, and run the VBStatePeristTask (if necessary
123     * for this bucket type).
124     * On return the state will be changed and the task completed.
125     *
126     * @param vbid
127     * @param newState
128     * @param meta Optional meta information to apply alongside the state
129     * @param transfer Should vBucket be transferred without adding failover
130     *                 table entry (i.e. takeover)?
131     */
132    void setVBucketStateAndRunPersistTask(Vbid vbid,
133                                          vbucket_state_t newState,
134                                          const nlohmann::json& meta = {},
135                                          TransferVB transfer = TransferVB::No);
136
137    /*
138     * Set the stats isShutdown and attempt to drive all tasks to cancel for
139     * the specified engine.
140     */
141    void shutdownAndPurgeTasks(EventuallyPersistentEngine* ep);
142
143    void cancelAndPurgeTasks();
144
145    /**
146     * This method will keep running reader tasks until the engine shows warmup
147     * is complete.
148     */
149    void runReadersUntilWarmedUp();
150
151    /**
152     * Destroy engine and replace it with a new engine that can be warmed up.
153     */
154    void resetEngineAndEnableWarmup(std::string new_config = "");
155
156    /**
157     * Destroy engine and replace it with a new engine that can be warmed up.
158     * Finally, run warmup.
159     */
160    void resetEngineAndWarmup(std::string new_config = "");
161
162    /*
163     * Fake callback emulating dcp_add_failover_log
164     */
165    static ENGINE_ERROR_CODE fakeDcpAddFailoverLog(
166            vbucket_failover_t* entry,
167            size_t nentries,
168            gsl::not_null<const void*> cookie) {
169        return ENGINE_SUCCESS;
170    }
171
172    SingleThreadedExecutorPool* task_executor;
173};
174
175/**
176 * Test fixture for single-threaded tests on EPBucket.
177 */
178class SingleThreadedEPBucketTest : public SingleThreadedKVBucketTest {
179public:
180    enum class BackfillBufferLimit { StreamByte, StreamItem, ConnectionByte };
181
182    void backfillExpiryOutput(bool xattr);
183    void producerReadyQLimitOnBackfill(BackfillBufferLimit limitType);
184
185protected:
186    EPBucket& getEPBucket();
187};
188
189/**
190 * Test fixture for KVBucket tests running in single-threaded mode, for some
191 * combination of bucket type, eviction mode and KVStore type.
192 *
193 * Allows tests to be defined once which are applicable to more than one
194 * configuration, and then instantiated with appropriate config parameters.
195 *
196 * Parameterised on a pair of:
197 * - bucket type (ephemeral or persistent, and additional persistent variants
198 *   (e.g. RocksDB) for additional storage backends.
199 * - eviction type.
200 *   - For ephemeral buckets: used for specifying ephemeral auto-delete /
201 *     fail_new_data
202 *   - For persistent buckets: used for specifying value_only or full_eviction
203 *
204 * See `allConfigValues(), persistentConfigValues(), etc methods to instantiate
205 * tests for some set / subset of the avbove parameters.
206 *
207 * Note that specific instantiations of tests may not instantiate for all
208 * possible variants - a test may only be applicable to persistent buckets and
209 * hence will only instantiate for persistentConfigValues.
210 *
211 * Suggested usage:
212 * 1. For a given group of tests (e.g. CollectionsDCP tests), create a subclass
213 *   of this class:
214 *
215 *     class MyTestSuite : public STParameterizedBucketTest {};
216 *
217 * 2. Write some (parameterized) tests:
218 *
219 *     TEST_P(MyTestSuite, DoesFoo) { ... }
220 *
221 * 3. Instantiate your test suite with the config values applicable to it -
222 * for example a test which is applicable to all variants of a Persistent
223 * bucket:
224 *
225 *     INSTANTIATE_TEST_CASE_P(
226 *         Persistent,
227 *         MyTestSuite,
228 *         STParameterizedBucketTest::persistentConfigValues(),
229 *         STParameterizedBucketTest::PrintToStringParamName);
230 *
231 * Advanced usage:
232 * - If you have some tests in a suite which only work for some config params
233 *   but not others (e.g. some don't work under Ephemeral), split your suite
234 *   into two sibling classes then instantiate each class with a different
235 *   config:
236 *
237 *   class DcpActiveStreamTest : public STParameterizedBucketTest {};
238 *   class DcpActiveStreamTestPersistent : public STParameterizedBucketTest {};
239 *
240 *   ... define some TEST_P() for each suite...
241 *
242 *     INSTANTIATE_TEST_CASE_P(
243 *         PersistentAndEphemeral,
244 *         DcpActiveStreamTest,
245 *         STParameterizedBucketTest::allConfigValues(),
246 *         STParameterizedBucketTest::PrintToStringParamName);
247 *
248 *     INSTANTIATE_TEST_CASE_P(
249 *         Persistent,
250 *         DcpActiveStreamTestPersistent,
251 *         STParameterizedBucketTest::persistentAllBackendsConfigValues(),
252 *         STParameterizedBucketTest::PrintToStringParamName());
253 */
254class STParameterizedBucketTest
255    : virtual public SingleThreadedKVBucketTest,
256      public ::testing::WithParamInterface<
257              std::tuple<std::string, std::string>> {
258public:
259    static auto ephConfigValues() {
260        using namespace std::string_literals;
261        return ::testing::Values(
262                std::make_tuple("ephemeral"s, "auto_delete"s),
263                std::make_tuple("ephemeral"s, "fail_new_data"s));
264    }
265
266    static auto allConfigValues() {
267        using namespace std::string_literals;
268        return ::testing::Values(
269                std::make_tuple("ephemeral"s, "auto_delete"s),
270                std::make_tuple("ephemeral"s, "fail_new_data"),
271                std::make_tuple("persistent"s, "value_only"s),
272                std::make_tuple("persistent"s, "full_eviction"s));
273    }
274
275    static auto persistentConfigValues() {
276        using namespace std::string_literals;
277        return ::testing::Values(
278                std::make_tuple("persistent"s, "value_only"s),
279                std::make_tuple("persistent"s, "full_eviction"s));
280    }
281
282    static auto persistentAllBackendsConfigValues() {
283        using namespace std::string_literals;
284        return ::testing::Values(
285                std::make_tuple("persistent"s, "value_only"s),
286                std::make_tuple("persistent"s, "full_eviction"s)
287#ifdef EP_USE_ROCKSDB
288                ,std::make_tuple("persistentRocksdb"s, "value_only"s),
289                std::make_tuple("persistentRocksdb"s, "full_eviction"s)
290#endif
291        );
292    }
293
294    bool persistent() const {
295        return std::get<0>(GetParam()).find("persistent") != std::string::npos;
296    }
297
298    bool ephemeral() const {
299        return std::get<0>(GetParam()).find("ephemeral") != std::string::npos;
300    }
301
302    bool fullEviction() const {
303        return persistent() && std::get<1>(GetParam()) == "full_eviction";
304    }
305
306    bool isRocksDB() const {
307        return std::get<0>(GetParam()).find("Rocksdb") != std::string::npos;
308    }
309
310    /// @returns a string representing this tests' parameters.
311    static std::string PrintToStringParamName(
312            const ::testing::TestParamInfo<ParamType>& info);
313
314protected:
315    void SetUp() override {
316        if (!config_string.empty()) {
317            config_string += ";";
318        }
319        auto bucketType = std::get<0>(GetParam());
320        if (bucketType == "persistentRocksdb") {
321            config_string += "bucket_type=persistent;backend=rocksdb";
322        } else {
323            config_string += "bucket_type=" + bucketType;
324        }
325        auto evictionPolicy = std::get<1>(GetParam());
326
327        if (!evictionPolicy.empty()) {
328            if (persistent()) {
329                config_string += ";item_eviction_policy=" + evictionPolicy;
330            } else {
331                config_string += ";ephemeral_full_policy=" + evictionPolicy;
332            }
333        }
334
335        SingleThreadedKVBucketTest::SetUp();
336    }
337
338    // Test replicating delete times.
339    void test_replicateDeleteTime(time_t deleteTime);
340};
341
342class STParamPersistentBucketTest : public STParameterizedBucketTest {
343protected:
344    void testAbortDoesNotIncrementOpsDelete(bool flusherDedup);
345
346    /**
347     * All the tests below check that we don't lose any item, any vbstate and
348     * that we update flush-stats properly when flush fails and we re-attemt the
349     * flush later.
350     *
351     * @param failureCode How the flush fails, this is the injected error-code
352     *  return by KVStore::commit in our tests
353     * @param vbDeletion Some tests get this additional arg to verify that all
354     *  goes as expected when the flusher processes VBuckets set for deferred
355     *  deletion
356     */
357    void testFlushFailureAtPersistNonMetaItems(couchstore_error_t failureCode);
358    void testFlushFailureAtPersistVBStateOnly(couchstore_error_t failureCode);
359    void testFlushFailureStatsAtDedupedNonMetaItems(
360            couchstore_error_t failureCode, bool vbDeletion = false);
361    void testFlushFailureAtPersistDelete(couchstore_error_t failureCode,
362                                         bool vbDeletion = false);
363};