1/* -*- MODE: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/*
3 *     Copyright 2010 Couchbase, Inc
4 *
5 *   Licensed under the Apache License, Version 2.0 (the "License");
6 *   you may not use this file except in compliance with the License.
7 *   You may obtain a copy of the License at
8 *
9 *       http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *   Unless required by applicable law or agreed to in writing, software
12 *   distributed under the License is distributed on an "AS IS" BASIS,
13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *   See the License for the specific language governing permissions and
15 *   limitations under the License.
16 */
17
18// Usage: (to run just a single test case)
19// make engine_tests EP_TEST_NUM=3
20
21#include "config.h"
22
23#include <stdio.h>
24#include <stdlib.h>
25#include <string.h>
26
27#include <chrono>
28#include <condition_variable>
29#include <cstdlib>
30#include <iomanip>
31#include <iostream>
32#include <map>
33#include <mutex>
34#include <regex>
35#include <set>
36#include <sstream>
37#include <string>
38#include <thread>
39#include <unordered_map>
40#include <unordered_set>
41#include <vector>
42
43#include "atomic.h"
44#include "couch-kvstore/couch-kvstore-metadata.h"
45#include "ep_test_apis.h"
46
47#include "ep_testsuite_common.h"
48#include "locks.h"
49#include <libcouchstore/couch_db.h>
50#include <memcached/engine.h>
51#include <memcached/engine_testapp.h>
52#include <platform/cb_malloc.h>
53#include <platform/dirutils.h>
54#include <JSON_checker.h>
55#include <memcached/types.h>
56#include <string_utilities.h>
57#include <xattr/blob.h>
58#include <xattr/utils.h>
59
60#ifdef linux
61/* /usr/include/netinet/in.h defines macros from ntohs() to _bswap_nn to
62 * optimize the conversion functions, but the prototypes generate warnings
63 * from gcc. The conversion methods isn't the bottleneck for my app, so
64 * just remove the warnings by undef'ing the optimization ..
65 */
66#undef ntohs
67#undef ntohl
68#undef htons
69#undef htonl
70#endif
71
72// ptr_fun don't like the extern "C" thing for unlock cookie.. cast it
73// away ;)
74typedef void (*UNLOCK_COOKIE_T)(const void *cookie);
75
76class ThreadData {
77public:
78    ThreadData(ENGINE_HANDLE *eh, ENGINE_HANDLE_V1 *ehv1,
79               int e=0) : h(eh), h1(ehv1), extra(e) {}
80    ENGINE_HANDLE    *h;
81    ENGINE_HANDLE_V1 *h1;
82    int               extra;
83};
84
85enum class BucketType { EP, Ephemeral };
86
87static void check_observe_seqno(bool failover,
88                                BucketType bucket_type,
89                                uint8_t format_type,
90                                uint16_t vb_id,
91                                uint64_t vb_uuid,
92                                uint64_t last_persisted_seqno,
93                                uint64_t current_seqno,
94                                uint64_t failover_vbuuid = 0,
95                                uint64_t failover_seqno = 0) {
96    uint8_t recv_format_type;
97    uint16_t recv_vb_id;
98    uint64_t recv_vb_uuid;
99    uint64_t recv_last_persisted_seqno;
100    uint64_t recv_current_seqno;
101    uint64_t recv_failover_vbuuid;
102    uint64_t recv_failover_seqno;
103
104    memcpy(&recv_format_type, last_body.data(), sizeof(uint8_t));
105    checkeq(format_type, recv_format_type, "Wrong format type in result");
106    memcpy(&recv_vb_id, last_body.data() + 1, sizeof(uint16_t));
107    checkeq(vb_id, ntohs(recv_vb_id), "Wrong vbucket id in result");
108    memcpy(&recv_vb_uuid, last_body.data() + 3, sizeof(uint64_t));
109    checkeq(vb_uuid, ntohll(recv_vb_uuid), "Wrong vbucket uuid in result");
110    memcpy(&recv_last_persisted_seqno, last_body.data() + 11, sizeof(uint64_t));
111
112    switch (bucket_type) {
113    case BucketType::EP:
114        // Should get the "real" persisted seqno:
115        checkeq(last_persisted_seqno,
116                ntohll(recv_last_persisted_seqno),
117                "Wrong persisted seqno in result (EP)");
118        break;
119    case BucketType::Ephemeral:
120        // For ephemeral, this should always be zero, as there is no
121        // persistence.
122        checkeq(uint64_t(0),
123                ntohll(recv_last_persisted_seqno),
124                "Wrong persisted seqno in result (Ephemeral)");
125        break;
126    }
127
128    memcpy(&recv_current_seqno, last_body.data() + 19, sizeof(uint64_t));
129    checkeq(current_seqno, ntohll(recv_current_seqno), "Wrong current seqno in result");
130
131    if (failover) {
132        memcpy(&recv_failover_vbuuid, last_body.data() + 27, sizeof(uint64_t));
133        checkeq(failover_vbuuid, ntohll(recv_failover_vbuuid),
134                "Wrong failover uuid in result");
135        memcpy(&recv_failover_seqno, last_body.data() + 35, sizeof(uint64_t));
136        checkeq(failover_seqno, ntohll(recv_failover_seqno),
137                "Wrong failover seqno in result");
138    }
139}
140
141static enum test_result test_replace_with_eviction(ENGINE_HANDLE *h,
142                                                   ENGINE_HANDLE_V1 *h1) {
143    checkeq(ENGINE_SUCCESS,
144            store(h, h1, NULL, OPERATION_SET, "key", "somevalue"),
145            "Failed to set value.");
146    wait_for_flusher_to_settle(h, h1);
147    evict_key(h, h1, "key");
148    int numBgFetched = get_int_stat(h, h1, "ep_bg_fetched");
149
150    checkeq(ENGINE_SUCCESS,
151            store(h, h1, NULL, OPERATION_REPLACE, "key", "somevalue1"),
152            "Failed to replace existing value.");
153
154    checkeq(ENGINE_SUCCESS,
155            get_stats(h, {}, add_stats),
156            "Failed to get stats.");
157    std::string eviction_policy = vals.find("ep_item_eviction_policy")->second;
158    if (eviction_policy == "full_eviction") {
159        numBgFetched++;
160    }
161
162    checkeq(numBgFetched,
163            get_int_stat(h, h1, "ep_bg_fetched"),
164            "Bg fetched value didn't match");
165
166    check_key_value(h, h1, "key", "somevalue1", 10);
167    return SUCCESS;
168}
169
170static enum test_result test_wrong_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
171                                               ENGINE_STORE_OPERATION op) {
172    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
173    uint64_t cas = 11;
174    if (op == OPERATION_ADD) {
175        // Add operation with cas != 0 doesn't make sense
176        cas = 0;
177    }
178    checkeq(ENGINE_NOT_MY_VBUCKET,
179            store(h, h1, NULL, op, "key", "somevalue", nullptr, cas, 1),
180            "Expected not_my_vbucket");
181    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
182    return SUCCESS;
183}
184
185static enum test_result test_pending_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
186                                                 ENGINE_STORE_OPERATION op) {
187    const void *cookie = testHarness.create_cookie();
188    testHarness.set_ewouldblock_handling(cookie, false);
189    check(set_vbucket_state(h, h1, 1, vbucket_state_pending),
190          "Failed to set vbucket state.");
191    check(verify_vbucket_state(h, h1, 1, vbucket_state_pending),
192          "Bucket state was not set to pending.");
193    uint64_t cas = 11;
194    if (op == OPERATION_ADD) {
195        // Add operation with cas != 0 doesn't make sense..
196        cas = 0;
197    }
198    checkeq(ENGINE_EWOULDBLOCK,
199            store(h, h1, cookie, op, "key", "somevalue", nullptr, cas, 1),
200            "Expected ewouldblock");
201    testHarness.destroy_cookie(cookie);
202    return SUCCESS;
203}
204
205static enum test_result test_replica_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
206                                                 ENGINE_STORE_OPERATION op) {
207    check(set_vbucket_state(h, h1, 1, vbucket_state_replica),
208          "Failed to set vbucket state.");
209    check(verify_vbucket_state(h, h1, 1, vbucket_state_replica),
210          "Bucket state was not set to replica.");
211    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
212
213    uint64_t cas = 11;
214    if (op == OPERATION_ADD) {
215        // performing add with a CAS != 0 doesn't make sense...
216        cas = 0;
217    }
218    checkeq(ENGINE_NOT_MY_VBUCKET,
219            store(h, h1, NULL, op, "key", "somevalue", nullptr, cas, 1),
220            "Expected not my vbucket");
221    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
222    return SUCCESS;
223}
224
225//
226// ----------------------------------------------------------------------
227// The actual tests are below.
228// ----------------------------------------------------------------------
229//
230
231static int checkCurrItemsAfterShutdown(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
232                                       int numItems2Load, bool shutdownForce) {
233    if (!isWarmupEnabled(h, h1)) {
234        return SKIPPED;
235    }
236
237    std::vector<std::string> keys;
238    for (int index = 0; index < numItems2Load; ++index) {
239        std::stringstream s;
240        s << "keys_2_load-" << index;
241        std::string key(s.str());
242        keys.push_back(key);
243    }
244
245    // Check preconditions.
246    checkeq(0, get_int_stat(h, h1, "ep_total_persisted"),
247            "Expected ep_total_persisted equals 0");
248    checkeq(0, get_int_stat(h, h1, "curr_items"),
249            "Expected curr_items equals 0");
250
251    // stop flusher before loading new items
252    protocol_binary_request_header *pkt = createPacket(PROTOCOL_BINARY_CMD_STOP_PERSISTENCE);
253    checkeq(ENGINE_SUCCESS,
254            h1->unknown_command(h, NULL, pkt, add_response, testHarness.doc_namespace),
255            "CMD_STOP_PERSISTENCE failed!");
256    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS,
257            last_status.load(),
258            "Failed to stop persistence!");
259    cb_free(pkt);
260
261    std::vector<std::string>::iterator itr;
262    for (itr = keys.begin(); itr != keys.end(); ++itr) {
263        checkeq(ENGINE_SUCCESS,
264                store(h, h1, NULL, OPERATION_SET, itr->c_str(), "oracle"),
265                "Failed to store a value");
266    }
267
268    checkeq(0, get_int_stat(h, h1, "ep_total_persisted"),
269            "Incorrect ep_total_persisted, expected 0");
270
271    // Can only check curr_items in value_only eviction; full-eviction
272    // relies on persistence to complete (via flusher) to update count.
273    const auto evictionPolicy = get_str_stat(h, h1, "ep_item_eviction_policy");
274    if (evictionPolicy == "value_only") {
275        checkeq(numItems2Load,
276                get_int_stat(h, h1, "curr_items"),
277                "Expected curr_items to reflect item count");
278    }
279
280    // resume flusher before shutdown + warmup
281    pkt = createPacket(PROTOCOL_BINARY_CMD_START_PERSISTENCE);
282    checkeq(ENGINE_SUCCESS, h1->unknown_command(h, NULL, pkt, add_response, testHarness.doc_namespace),
283            "CMD_START_PERSISTENCE failed!");
284    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
285          "Failed to start persistence!");
286    cb_free(pkt);
287
288    // shutdown engine force and restart
289    testHarness.reload_engine(&h, &h1,
290                              testHarness.engine_path,
291                              testHarness.get_current_testcase()->cfg,
292                              true, shutdownForce);
293    wait_for_warmup_complete(h, h1);
294    return get_int_stat(h, h1, "curr_items");
295}
296
297static enum test_result test_flush_shutdown_force(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
298    if (!isWarmupEnabled(h, h1)) {
299        return SKIPPED;
300    }
301
302    int numItems2load = 3000;
303    bool shutdownForce = true;
304    int currItems = checkCurrItemsAfterShutdown(h, h1, numItems2load, shutdownForce);
305    check (currItems <= numItems2load,
306           "Number of curr items should be <= 3000, unless previous "
307           "shutdown force had to wait for the flusher");
308    return SUCCESS;
309}
310
311static enum test_result test_flush_shutdown_noforce(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
312    if (!isWarmupEnabled(h, h1)) {
313        return SKIPPED;
314    }
315
316    int numItems2load = 3000;
317    bool shutdownForce = false;
318    int currItems = checkCurrItemsAfterShutdown(h, h1, numItems2load, shutdownForce);
319    check (currItems == numItems2load,
320           "Number of curr items should be equal to 3000, unless previous "
321           "shutdown did not wait for the flusher");
322    return SUCCESS;
323}
324
325static enum test_result test_shutdown_snapshot_range(ENGINE_HANDLE *h,
326                                                     ENGINE_HANDLE_V1 *h1) {
327    if (!isWarmupEnabled(h, h1)) {
328        return SKIPPED;
329    }
330
331    const int num_items = 100;
332    for (int j = 0; j < num_items; ++j) {
333        std::stringstream ss;
334        ss << "key" << j;
335        checkeq(ENGINE_SUCCESS,
336                store(h, h1, NULL, OPERATION_SET, ss.str().c_str(), "data"),
337                "Failed to store a value");
338    }
339
340    wait_for_flusher_to_settle(h, h1);
341    int end = get_int_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno");
342
343    /* change vb state to replica before restarting (as it happens in graceful
344       failover)*/
345    check(set_vbucket_state(h, h1, 0, vbucket_state_replica),
346          "Failed set vbucket 0 to replica state.");
347
348    /* trigger persist vb state task */
349    check(set_param(h, h1, protocol_binary_engine_param_flush,
350                    "vb_state_persist_run", "0"),
351          "Failed to trigger vb state persist");
352
353    /* restart the engine */
354    testHarness.reload_engine(&h, &h1,
355                              testHarness.engine_path,
356                              testHarness.get_current_testcase()->cfg,
357                              true, false);
358    wait_for_warmup_complete(h, h1);
359
360    /* Check if snapshot range is persisted correctly */
361    checkeq(end, get_int_stat(h, h1, "vb_0:last_persisted_snap_start",
362                              "vbucket-seqno"),
363            "Wrong snapshot start persisted");
364    checkeq(end, get_int_stat(h, h1, "vb_0:last_persisted_snap_end",
365                                    "vbucket-seqno"),
366            "Wrong snapshot end persisted");
367
368    return SUCCESS;
369}
370
371static enum test_result test_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
372    if (!isWarmupEnabled(h, h1)) {
373        return SKIPPED;
374    }
375
376    static const char val[] = "somevalue";
377    ENGINE_ERROR_CODE ret = store(h, h1, NULL, OPERATION_SET, "key", val);
378    checkeq(ENGINE_SUCCESS, ret, "Failed set.");
379
380    testHarness.reload_engine(&h, &h1,
381                              testHarness.engine_path,
382                              testHarness.get_current_testcase()->cfg,
383                              true, false);
384    wait_for_warmup_complete(h, h1);
385    check_key_value(h, h1, "key", val, strlen(val));
386    return SUCCESS;
387}
388
389static enum test_result test_specialKeys(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
390    ENGINE_ERROR_CODE ret;
391
392    // Simplified Chinese "Couchbase"
393    static const char key0[] = "沙发数据库";
394    static const char val0[] = "some Chinese value";
395    check((ret = store(h, h1, NULL, OPERATION_SET, key0, val0)) ==
396                  ENGINE_SUCCESS,
397          "Failed set Chinese key");
398    check_key_value(h, h1, key0, val0, strlen(val0));
399
400    // Traditional Chinese "Couchbase"
401    static const char key1[] = "沙發數據庫";
402    static const char val1[] = "some Traditional Chinese value";
403    check((ret = store(h, h1, NULL, OPERATION_SET, key1, val1)) ==
404                  ENGINE_SUCCESS,
405          "Failed set Traditional Chinese key");
406
407    // Korean "couch potato"
408    static const char key2[] = "쇼파감자";
409    static const char val2[] = "some Korean value";
410    check((ret = store(h, h1, NULL, OPERATION_SET, key2, val2)) ==
411                  ENGINE_SUCCESS,
412          "Failed set Korean key");
413
414    // Russian "couch potato"
415    static const char key3[] = "лодырь, лентяй";
416    static const char val3[] = "some Russian value";
417    check((ret = store(h, h1, NULL, OPERATION_SET, key3, val3)) ==
418                  ENGINE_SUCCESS,
419          "Failed set Russian key");
420
421    // Japanese "couch potato"
422    static const char key4[] = "カウチポテト";
423    static const char val4[] = "some Japanese value";
424    check((ret = store(h, h1, NULL, OPERATION_SET, key4, val4)) ==
425                  ENGINE_SUCCESS,
426          "Failed set Japanese key");
427
428    // Indian char key, and no idea what it is
429    static const char key5[] = "हरियानवी";
430    static const char val5[] = "some Indian value";
431    check((ret = store(h, h1, NULL, OPERATION_SET, key5, val5)) ==
432                  ENGINE_SUCCESS,
433          "Failed set Indian key");
434
435    // Portuguese translation "couch potato"
436    static const char key6[] = "sedentário";
437    static const char val6[] = "some Portuguese value";
438    check((ret = store(h, h1, NULL, OPERATION_SET, key6, val6)) ==
439                  ENGINE_SUCCESS,
440          "Failed set Portuguese key");
441
442    // Arabic translation "couch potato"
443    static const char key7[] = "الحافلةالبطاطة";
444    static const char val7[] = "some Arabic value";
445    check((ret = store(h, h1, NULL, OPERATION_SET, key7, val7)) ==
446                  ENGINE_SUCCESS,
447          "Failed set Arabic key");
448
449    if (isWarmupEnabled(h, h1)) {
450        // Check that after warmup the keys are still present.
451        testHarness.reload_engine(&h, &h1,
452                                  testHarness.engine_path,
453                                  testHarness.get_current_testcase()->cfg,
454                                  true, false);
455        wait_for_warmup_complete(h, h1);
456        check_key_value(h, h1, key0, val0, strlen(val0));
457        check_key_value(h, h1, key1, val1, strlen(val1));
458        check_key_value(h, h1, key2, val2, strlen(val2));
459        check_key_value(h, h1, key3, val3, strlen(val3));
460        check_key_value(h, h1, key4, val4, strlen(val4));
461        check_key_value(h, h1, key5, val5, strlen(val5));
462        check_key_value(h, h1, key6, val6, strlen(val6));
463        check_key_value(h, h1, key7, val7, strlen(val7));
464    }
465    return SUCCESS;
466}
467
468static enum test_result test_binKeys(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
469    ENGINE_ERROR_CODE ret;
470
471    // binary key with char values beyond 0x7F
472    static const char key0[] = "\xe0\xed\xf1\x6f\x7f\xf8\xfa";
473    static const char val0[] = "some value val8";
474    check((ret = store(h, h1, NULL, OPERATION_SET, key0, val0)) ==
475                  ENGINE_SUCCESS,
476          "Failed set binary key0");
477    check_key_value(h, h1, key0, val0, strlen(val0));
478
479    // binary keys with char values beyond 0x7F
480    static const char key1[] = "\xf1\xfd\xfe\xff\xf0\xf8\xef";
481    static const char val1[] = "some value val9";
482    check((ret = store(h, h1, NULL, OPERATION_SET, key1, val1)) ==
483                  ENGINE_SUCCESS,
484          "Failed set binary key1");
485    check_key_value(h, h1, key1, val1, strlen(val1));
486
487    // binary keys with special utf-8 BOM (Byte Order Mark) values 0xBB 0xBF
488    // 0xEF
489    static const char key2[] = "\xff\xfe\xbb\xbf\xef";
490    static const char val2[] = "some utf-8 bom value";
491    check((ret = store(h, h1, NULL, OPERATION_SET, key2, val2)) ==
492                  ENGINE_SUCCESS,
493          "Failed set binary utf-8 bom key");
494    check_key_value(h, h1, key2, val2, strlen(val2));
495
496    // binary keys with special utf-16BE BOM values "U+FEFF"
497    static const char key3[] = "U+\xfe\xff\xefU+\xff\xfe";
498    static const char val3[] = "some utf-16 bom value";
499    check((ret = store(h, h1, NULL, OPERATION_SET, key3, val3)) ==
500                  ENGINE_SUCCESS,
501          "Failed set binary utf-16 bom key");
502    check_key_value(h, h1, key3, val3, strlen(val3));
503
504    if (isWarmupEnabled(h, h1)) {
505        testHarness.reload_engine(&h, &h1,
506                                  testHarness.engine_path,
507                                  testHarness.get_current_testcase()->cfg,
508                                  true, false);
509        wait_for_warmup_complete(h, h1);
510        check_key_value(h, h1, key0, val0, strlen(val0));
511        check_key_value(h, h1, key1, val1, strlen(val1));
512        check_key_value(h, h1, key2, val2, strlen(val2));
513        check_key_value(h, h1, key3, val3, strlen(val3));
514    }
515    return SUCCESS;
516}
517
518static enum test_result test_restart_bin_val(ENGINE_HANDLE *h,
519                                             ENGINE_HANDLE_V1 *h1) {
520    if (!isWarmupEnabled(h, h1)) {
521        return SKIPPED;
522    }
523
524    char binaryData[] = "abcdefg\0gfedcba";
525    cb_assert(sizeof(binaryData) != strlen(binaryData));
526
527    checkeq(cb::engine_errc::success,
528            storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
529                         binaryData, sizeof(binaryData), 82758, 0, 0).first,
530            "Failed set.");
531
532    testHarness.reload_engine(&h, &h1,
533                              testHarness.engine_path,
534                              testHarness.get_current_testcase()->cfg,
535                              true, false);
536    wait_for_warmup_complete(h, h1);
537
538    check_key_value(h, h1, "key", binaryData, sizeof(binaryData));
539    return SUCCESS;
540}
541
542static enum test_result test_wrong_vb_get(ENGINE_HANDLE *h,
543                                          ENGINE_HANDLE_V1 *h1) {
544    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
545    checkeq(ENGINE_NOT_MY_VBUCKET, verify_key(h, h1, "key", 1),
546            "Expected wrong bucket.");
547    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
548    return SUCCESS;
549}
550
551static enum test_result test_vb_get_pending(ENGINE_HANDLE *h,
552                                            ENGINE_HANDLE_V1 *h1) {
553    check(set_vbucket_state(h, h1, 1, vbucket_state_pending),
554          "Failed to set vbucket state.");
555    const void *cookie = testHarness.create_cookie();
556    testHarness.set_ewouldblock_handling(cookie, false);
557
558    checkeq(cb::engine_errc::would_block,
559            get(h, h1, cookie, "key", 1).first,
560            "Expected woodblock.");
561
562    testHarness.destroy_cookie(cookie);
563    return SUCCESS;
564}
565
566static enum test_result test_vb_get_replica(ENGINE_HANDLE *h,
567                                            ENGINE_HANDLE_V1 *h1) {
568    check(set_vbucket_state(h, h1, 1, vbucket_state_replica),
569          "Failed to set vbucket state.");
570    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
571    checkeq(ENGINE_NOT_MY_VBUCKET,
572            verify_key(h, h1, "key", 1),
573            "Expected not my bucket.");
574    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
575    return SUCCESS;
576}
577
578static enum test_result test_wrong_vb_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
579    return test_wrong_vb_mutation(h, h1, OPERATION_SET);
580}
581
582static enum test_result test_wrong_vb_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
583    return test_wrong_vb_mutation(h, h1, OPERATION_CAS);
584}
585
586static enum test_result test_wrong_vb_add(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
587    return test_wrong_vb_mutation(h, h1, OPERATION_ADD);
588}
589
590static enum test_result test_wrong_vb_replace(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
591    return test_wrong_vb_mutation(h, h1, OPERATION_REPLACE);
592}
593
594static enum test_result test_wrong_vb_del(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
595    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
596    checkeq(ENGINE_NOT_MY_VBUCKET, del(h, h1, "key", 0, 1),
597            "Expected wrong bucket.");
598    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
599    return SUCCESS;
600}
601
602/* Returns a string in the format "%Y-%m-%d %H:%M:%S" of the specified
603 * time point.
604 */
605std::string make_time_string(std::chrono::system_clock::time_point time_point) {
606    time_t tt = std::chrono::system_clock::to_time_t(time_point);
607#ifdef _MSC_VER
608    // Windows' gmtime() is already thread-safe.
609    struct tm* split = gmtime(&tt);
610#else
611    struct tm local_storage;
612    struct tm* split = gmtime_r(&tt, &local_storage);
613#endif
614    char timeStr[20];
615    strftime(timeStr, 20, "%Y-%m-%d %H:%M:%S", split);
616    return timeStr;
617}
618
619static enum test_result test_expiry_pager_settings(ENGINE_HANDLE *h,
620                                                   ENGINE_HANDLE_V1 *h1) {
621
622    cb_assert(!get_bool_stat(h, h1, "ep_exp_pager_enabled"));
623    checkeq(3600, get_int_stat(h, h1, "ep_exp_pager_stime"),
624            "Expiry pager sleep time not expected");
625    set_param(h, h1, protocol_binary_engine_param_flush,
626              "exp_pager_stime", "1");
627    checkeq(1, get_int_stat(h, h1, "ep_exp_pager_stime"),
628            "Expiry pager sleep time not updated");
629    cb_assert(!get_bool_stat(h, h1, "ep_exp_pager_enabled"));
630    sleep(1);
631    checkeq(0, get_int_stat(h, h1, "ep_num_expiry_pager_runs"),
632            "Expiry pager run count is not zero");
633
634    set_param(h, h1, protocol_binary_engine_param_flush,
635              "exp_pager_enabled", "true");
636    checkeq(1, get_int_stat(h, h1, "ep_exp_pager_stime"),
637            "Expiry pager sleep time not updated");
638    wait_for_stat_to_be_gte(h, h1, "ep_num_expiry_pager_runs", 1);
639
640    // Reload engine
641    testHarness.reload_engine(&h, &h1,
642                              testHarness.engine_path,
643                              testHarness.get_current_testcase()->cfg,
644                              true, false);
645    wait_for_warmup_complete(h, h1);
646    cb_assert(!get_bool_stat(h, h1, "ep_exp_pager_enabled"));
647
648    // Enable expiry pager again
649    set_param(h, h1, protocol_binary_engine_param_flush,
650              "exp_pager_enabled", "true");
651
652    checkeq(get_int_stat(h, h1, "ep_exp_pager_initial_run_time"), -1,
653            "Task time should be disable upon warmup");
654
655    std::string err_msg;
656    // Update exp_pager_initial_run_time and ensure the update is successful
657    set_param(h, h1, protocol_binary_engine_param_flush,
658              "exp_pager_initial_run_time", "3");
659    std::string expected_time = "03:00";
660    std::string str;
661    // [MB-21806] - Need to repeat the fetch as the set_param for
662    // "exp_pager_initial_run_time" schedules a task that sets the stats later
663    repeat_till_true([&]() {
664        str = get_str_stat(h, h1, "ep_expiry_pager_task_time");
665        return 0 == str.substr(11, 5).compare(expected_time);
666    });
667    err_msg.assign("Updated time incorrect, expect: " +
668                   expected_time + ", actual: " + str.substr(11, 5));
669    checkeq(0, str.substr(11, 5).compare(expected_time), err_msg.c_str());
670
671    // Update exp_pager_stime by 30 minutes and ensure that the update is successful
672    const std::chrono::minutes update_by{30};
673    std::string targetTaskTime1{make_time_string(std::chrono::system_clock::now() +
674                                                 update_by)};
675
676    set_param(h, h1, protocol_binary_engine_param_flush, "exp_pager_stime",
677              std::to_string(update_by.count() * 60).c_str());
678    str = get_str_stat(h, h1, "ep_expiry_pager_task_time");
679
680    std::string targetTaskTime2{make_time_string(std::chrono::system_clock::now() +
681                                                 update_by)};
682
683    // ep_expiry_pager_task_time should fall within the range of
684    // targetTaskTime1 and targetTaskTime2
685    err_msg.assign("Unexpected task time range, expect: " +
686                   targetTaskTime1 + " <= " + str + " <= " + targetTaskTime2);
687    check(targetTaskTime1 <= str, err_msg.c_str());
688    check(str <= targetTaskTime2, err_msg.c_str());
689
690    return SUCCESS;
691}
692
693static enum test_result test_expiry_with_xattr(ENGINE_HANDLE* h,
694                                               ENGINE_HANDLE_V1* h1) {
695    const char* key = "test_expiry";
696    cb::xattr::Blob blob;
697
698    //Add a few XAttrs
699    blob.set("user", "{\"author\":\"bubba\"}");
700    blob.set("_sync", "{\"cas\":\"0xdeadbeefcafefeed\"}");
701    blob.set("meta", "{\"content-type\":\"text\"}");
702
703    auto xattr_value = blob.finalize();
704
705    //Now, append user data to the xattrs and store the data
706    std::string value_data("test_expiry_value");
707    std::vector<char> data;
708    std::copy(xattr_value.buf, xattr_value.buf + xattr_value.len,
709              std::back_inserter(data));
710    std::copy(value_data.c_str(), value_data.c_str() + value_data.length(),
711              std::back_inserter(data));
712
713    const void* cookie = testHarness.create_cookie();
714
715    checkeq(cb::engine_errc::success,
716            storeCasVb11(h, h1, cookie, OPERATION_SET, key,
717                         reinterpret_cast<char*>(data.data()),
718                         data.size(), 9258, 0, 0, 10,
719                         PROTOCOL_BINARY_DATATYPE_XATTR).first,
720            "Failed to store xattr document");
721
722    if (isPersistentBucket(h, h1)) {
723        wait_for_flusher_to_settle(h, h1);
724    }
725
726    testHarness.time_travel(11);
727
728    cb::EngineErrorMetadataPair errorMetaPair;
729
730    check(get_meta(h, h1, "test_expiry", errorMetaPair, cookie),
731          "Get meta command failed");
732    auto prev_revseqno = errorMetaPair.second.seqno;
733
734    checkeq(PROTOCOL_BINARY_DATATYPE_XATTR,
735            errorMetaPair.second.datatype,
736            "Datatype is not XATTR");
737
738    auto ret = get(h, h1, cookie, key, 0, DocStateFilter::AliveOrDeleted);
739    checkeq(cb::engine_errc::success,
740            ret.first,
741            "Unable to get a deleted item");
742
743    check(get_meta(h, h1, "test_expiry", errorMetaPair, cookie),
744          "Get meta command failed");
745
746    checkeq(errorMetaPair.second.seqno,
747            prev_revseqno + 1,
748            "rev seqno must have incremented by 1");
749
750    /* Retrieve the item info and create a new blob out of the data */
751    item_info info;
752    checkeq(true,
753            h1->get_item_info(h, ret.second.get(), &info),
754            "Unable to retrieve item info");
755
756    cb::char_buffer value_buf{static_cast<char*>(info.value[0].iov_base),
757                              info.value[0].iov_len};
758
759    cb::xattr::Blob new_blob(value_buf, false);
760
761    /* Only system extended attributes need to be present at this point.
762     * Thus, check the blob length with the system size.
763     */
764    const auto systemsize = new_blob.finalize().len;
765
766    checkeq(systemsize, new_blob.get_system_size(),
767            "The size of the blob doesn't match the size of system attributes");
768
769    const std::string& cas_str{"{\"cas\":\"0xdeadbeefcafefeed\"}"};
770    const std::string& sync_str = to_string(blob.get("_sync"));
771
772    checkeq(cas_str, sync_str , "system xattr is invalid");
773
774    testHarness.destroy_cookie(cookie);
775
776    return SUCCESS;
777}
778
779static enum test_result test_expiry(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
780    const char *key = "test_expiry";
781    const char *data = "some test data here.";
782
783    const void* cookie = testHarness.create_cookie();
784    auto ret = allocate(h,
785                        h1,
786                        cookie,
787                        key,
788                        strlen(data),
789                        0,
790                        2,
791                        PROTOCOL_BINARY_RAW_BYTES,
792                        0);
793    checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
794
795    item_info info;
796    if (!h1->get_item_info(h, ret.second.get(), &info)) {
797        abort();
798    }
799    memcpy(info.value[0].iov_base, data, strlen(data));
800
801    uint64_t cas = 0;
802    auto rv = h1->store(h,
803                        cookie,
804                        ret.second.get(),
805                        cas,
806                        OPERATION_SET,
807                        DocumentState::Alive);
808    checkeq(ENGINE_SUCCESS, rv, "Set failed.");
809    check_key_value(h, h1, key, data, strlen(data));
810
811    testHarness.time_travel(5);
812    checkeq(cb::engine_errc::no_such_key,
813            get(h, h1, cookie, key, 0).first,
814            "Item didn't expire");
815
816    int expired_access = get_int_stat(h, h1, "ep_expired_access");
817    int expired_pager = get_int_stat(h, h1, "ep_expired_pager");
818    int active_expired = get_int_stat(h, h1, "vb_active_expired");
819    checkeq(0, expired_pager, "Expected zero expired item by pager");
820    checkeq(1, expired_access, "Expected an expired item on access");
821    checkeq(1, active_expired, "Expected an expired active item");
822    checkeq(ENGINE_SUCCESS,
823            store(h, h1, cookie, OPERATION_SET, key, data),
824            "Failed set.");
825
826    // When run under full eviction, the total item stats are set from the
827    // flusher. So we need to wait for it to finish before checking the
828    // total number of items.
829    wait_for_flusher_to_settle(h, h1);
830
831    std::stringstream ss;
832    ss << "curr_items stat should be still 1 after ";
833    ss << "overwriting the key that was expired, but not purged yet";
834    checkeq(1, get_int_stat(h, h1, "curr_items"), ss.str().c_str());
835
836    testHarness.destroy_cookie(cookie);
837    return SUCCESS;
838}
839
840static enum test_result test_expiry_loader(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
841    if (!isWarmupEnabled(h, h1)) {
842        return SKIPPED;
843    }
844    const char *key = "test_expiry_loader";
845    const char *data = "some test data here.";
846
847    const void* cookie = testHarness.create_cookie();
848    auto ret = allocate(h,
849                        h1,
850                        cookie,
851                        key,
852                        strlen(data),
853                        0,
854                        2,
855                        PROTOCOL_BINARY_RAW_BYTES,
856                        0);
857    checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
858
859    item_info info;
860    if (!h1->get_item_info(h, ret.second.get(), &info)) {
861        abort();
862    }
863    memcpy(info.value[0].iov_base, data, strlen(data));
864
865    uint64_t cas = 0;
866    auto rv = h1->store(h,
867                        cookie,
868                        ret.second.get(),
869                        cas,
870                        OPERATION_SET,
871                        DocumentState::Alive);
872    checkeq(ENGINE_SUCCESS, rv, "Set failed.");
873    check_key_value(h, h1, key, data, strlen(data));
874
875    testHarness.time_travel(3);
876
877    ret = get(h, h1, cookie, key, 0);
878    checkeq(cb::engine_errc::no_such_key, ret.first, "Item didn't expire");
879
880    // Restart the engine to ensure the above expired item is not loaded
881    testHarness.reload_engine(&h, &h1,
882                              testHarness.engine_path,
883                              testHarness.get_current_testcase()->cfg,
884                              true, false);
885    wait_for_warmup_complete(h, h1);
886    cb_assert(0 == get_int_stat(h, h1, "ep_warmup_value_count", "warmup"));
887
888    testHarness.destroy_cookie(cookie);
889
890    return SUCCESS;
891}
892
893static enum test_result test_expiration_on_compaction(ENGINE_HANDLE *h,
894                                                      ENGINE_HANDLE_V1 *h1) {
895    if (get_bool_stat(h, h1, "ep_exp_pager_enabled")) {
896        set_param(h, h1, protocol_binary_engine_param_flush,
897                  "exp_pager_enabled", "false");
898    }
899
900    checkeq(1,
901            get_int_stat(h, h1, "vb_0:persistence:num_visits", "checkpoint"),
902            "Cursor moved before item load");
903
904    for (int i = 0; i < 25; i++) {
905        std::stringstream ss;
906        ss << "key" << i;
907        checkeq(ENGINE_SUCCESS,
908                store(h,
909                      h1,
910                      NULL,
911                      OPERATION_SET,
912                      ss.str().c_str(),
913                      "somevalue",
914                      nullptr,
915                      0,
916                      0,
917                      10,
918                      PROTOCOL_BINARY_RAW_BYTES),
919                "Set failed.");
920    }
921
922    // Throw an xattr document in (and later compressed)
923    cb::xattr::Blob builder;
924    builder.set("_ep", "{\"foo\":\"bar\"}");
925    builder.set("key", "{\"foo\":\"bar\"}");
926    builder.set("_sync", "{\"foo\":\"bar\"}");
927    builder.set("stuff", "{\"foo\":\"bar\"}");
928    builder.set("misc", "{\"foo\":\"bar\"}");
929    builder.set("things", "{\"foo\":\"bar\"}");
930    builder.set("that", "{\"foo\":\"bar\"}");
931
932    auto blob = builder.finalize();
933    std::string data;
934    std::copy(blob.buf, blob.buf + blob.size(), std::back_inserter(data));
935    for (int i = 0; i < 12; i++) {
936        std::stringstream ss;
937        ss << "xattr_key" << i;
938
939        checkeq(cb::engine_errc::success,
940                storeCasVb11(h,
941                             h1,
942                             nullptr,
943                             OPERATION_SET,
944                             ss.str().c_str(),
945                             data.data(),
946                             data.size(),
947                             0,
948                             0,
949                             0,
950                             10,
951                             PROTOCOL_BINARY_DATATYPE_XATTR,
952                             DocumentState::Alive)
953                        .first,
954                "Unable to store item");
955    }
956
957    cb::compression::Buffer compressedDoc;
958    cb::compression::deflate(
959            cb::compression::Algorithm::Snappy, data, compressedDoc);
960
961    for (int i = 0; i < 13; i++) {
962        std::stringstream ss;
963        ss << "compressed_xattr_key" << i;
964
965        checkeq(cb::engine_errc::success,
966                storeCasVb11(h,
967                             h1,
968                             nullptr,
969                             OPERATION_SET,
970                             ss.str().c_str(),
971                             compressedDoc.data(),
972                             compressedDoc.size(),
973                             0,
974                             0,
975                             0,
976                             10,
977                             PROTOCOL_BINARY_DATATYPE_XATTR |
978                                     PROTOCOL_BINARY_DATATYPE_SNAPPY,
979                             DocumentState::Alive)
980                        .first,
981                "Unable to store item");
982    }
983
984    wait_for_flusher_to_settle(h, h1);
985    checkeq(50, get_int_stat(h, h1, "curr_items"),
986            "Unexpected number of items on database");
987    check(1 < get_int_stat(h, h1, "vb_0:persistence:num_visits", "checkpoint"),
988          "Cursor not moved even after flusher runs");
989
990    testHarness.time_travel(15);
991
992    // Compaction on VBucket
993    compact_db(h, h1, 0, 0, 0, 0, 0);
994    wait_for_stat_to_be(h, h1, "ep_pending_compactions", 0);
995
996    checkeq(50, get_int_stat(h, h1, "ep_expired_compactor"),
997            "Unexpected expirations by compactor");
998
999    return SUCCESS;
1000}
1001
1002static enum test_result test_expiration_on_warmup(ENGINE_HANDLE *h,
1003                                                  ENGINE_HANDLE_V1 *h1) {
1004    if (!isWarmupEnabled(h, h1)) {
1005        return SKIPPED;
1006    }
1007
1008    const auto* cookie = testHarness.create_cookie();
1009    set_param(h, h1, protocol_binary_engine_param_flush,
1010              "exp_pager_enabled", "false");
1011    int pager_runs = get_int_stat(h, h1, "ep_num_expiry_pager_runs");
1012
1013    const char *key = "KEY";
1014    const char *data = "VALUE";
1015
1016    auto ret = allocate(h,
1017                        h1,
1018                        cookie,
1019                        key,
1020                        strlen(data),
1021                        0,
1022                        10,
1023                        PROTOCOL_BINARY_RAW_BYTES,
1024                        0);
1025    checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
1026
1027    item_info info;
1028    if (!h1->get_item_info(h, ret.second.get(), &info)) {
1029        abort();
1030    }
1031    memcpy(info.value[0].iov_base, data, strlen(data));
1032
1033    uint64_t cas = 0;
1034    auto rv = h1->store(h,
1035                        cookie,
1036                        ret.second.get(),
1037                        cas,
1038                        OPERATION_SET,
1039                        DocumentState::Alive);
1040    checkeq(ENGINE_SUCCESS, rv, "Set failed.");
1041    check_key_value(h, h1, key, data, strlen(data));
1042    ret.second.reset();
1043    wait_for_flusher_to_settle(h, h1);
1044
1045    checkeq(1, get_int_stat(h, h1, "curr_items"), "Failed store item");
1046    testHarness.time_travel(15);
1047
1048    checkeq(pager_runs, get_int_stat(h, h1, "ep_num_expiry_pager_runs"),
1049            "Expiry pager shouldn't have run during this time");
1050
1051    // Restart the engine to ensure the above item is expired
1052    testHarness.reload_engine(&h, &h1,
1053                              testHarness.engine_path,
1054                              testHarness.get_current_testcase()->cfg,
1055                              true, false);
1056    wait_for_warmup_complete(h, h1);
1057    check(get_bool_stat(h, h1, "ep_exp_pager_enabled"),
1058          "Expiry pager should be enabled on warmup");
1059
1060    // Wait for the expiry pager to run and expire our item.
1061    wait_for_stat_to_be_gte(h, h1, "ep_expired_pager", 1, nullptr, /*secs*/10);
1062
1063    // Note: previously we checked that curr_items was zero here (immediately
1064    // after waiting for ep_expired_pager == 1), however we cannot assume that
1065    // - items are actually expired asynchronously.
1066    // See EPStore::deleteExpiredItem - for non-temporary, expired items we
1067    // call processSoftDelete (soft-marking the item as deleted in the
1068    // hashtable), and then call queueDirty to queue a deletion, and then
1069    // increment the expired stat. Only when that delete is actually persisted
1070    // and the deleted callback is invoked -
1071    // PeristenceCallback::callback(int&) - is curr_items finally decremented.
1072    // Therefore we need to wait for the flusher to settle (i.e. delete
1073    // callback to be called) for the curr_items stat to be accurate.
1074    wait_for_flusher_to_settle(h, h1);
1075
1076    checkeq(0, get_int_stat(h, h1, "curr_items"),
1077            "The item should have been expired.");
1078
1079    testHarness.destroy_cookie(cookie);
1080    return SUCCESS;
1081
1082}
1083
1084static enum test_result test_bug3454(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1085    if (!isWarmupEnabled(h, h1)) {
1086        return SKIPPED;
1087    }
1088
1089    const char *key = "test_expiry_duplicate_warmup";
1090    const char *data = "some test data here.";
1091
1092    const void* cookie = testHarness.create_cookie();
1093    auto ret = allocate(h,
1094                        h1,
1095                        cookie,
1096                        key,
1097                        strlen(data),
1098                        0,
1099                        5,
1100                        PROTOCOL_BINARY_RAW_BYTES,
1101                        0);
1102    checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
1103
1104    item_info info;
1105    if (!h1->get_item_info(h, ret.second.get(), &info)) {
1106        abort();
1107    }
1108    memcpy(info.value[0].iov_base, data, strlen(data));
1109
1110    uint64_t cas = 0;
1111    auto rv = h1->store(h,
1112                        cookie,
1113                        ret.second.get(),
1114                        cas,
1115                        OPERATION_SET,
1116                        DocumentState::Alive);
1117    checkeq(ENGINE_SUCCESS, rv, "Set failed.");
1118    check_key_value(h, h1, key, data, strlen(data));
1119    wait_for_flusher_to_settle(h, h1);
1120
1121    // Advance the ep_engine time by 10 sec for the above item to be expired.
1122    testHarness.time_travel(10);
1123    ret = get(h, h1, cookie, key, 0);
1124    checkeq(cb::engine_errc::no_such_key, ret.first, "Item didn't expire");
1125
1126    ret = allocate(h,
1127                   h1,
1128                   cookie,
1129                   key,
1130                   strlen(data),
1131                   0,
1132                   0,
1133                   PROTOCOL_BINARY_RAW_BYTES,
1134                   0);
1135    checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
1136
1137    if (!h1->get_item_info(h, ret.second.get(), &info)) {
1138        abort();
1139    }
1140    memcpy(info.value[0].iov_base, data, strlen(data));
1141
1142    cas = 0;
1143    // Add a new item with the same key.
1144    rv = h1->store(h,
1145                   cookie,
1146                   ret.second.get(),
1147                   cas,
1148                   OPERATION_ADD,
1149                   DocumentState::Alive);
1150    checkeq(ENGINE_SUCCESS, rv, "Add failed.");
1151    check_key_value(h, h1, key, data, strlen(data));
1152    ret.second.reset();
1153    wait_for_flusher_to_settle(h, h1);
1154
1155    checkeq(cb::engine_errc::success,
1156            get(h, h1, cookie, key, 0).first,
1157            "Item shouldn't expire");
1158
1159    // Restart the engine to ensure the above unexpired new item is loaded
1160    testHarness.reload_engine(&h, &h1,
1161                              testHarness.engine_path,
1162                              testHarness.get_current_testcase()->cfg,
1163                              true, false);
1164    wait_for_warmup_complete(h, h1);
1165    cb_assert(1 == get_int_stat(h, h1, "ep_warmup_value_count", "warmup"));
1166    cb_assert(0 == get_int_stat(h, h1, "ep_warmup_dups", "warmup"));
1167
1168    testHarness.destroy_cookie(cookie);
1169    return SUCCESS;
1170}
1171
1172static enum test_result test_bug3522(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1173    if (!isWarmupEnabled(h, h1)) {
1174        return SKIPPED;
1175    }
1176
1177    const char *key = "test_expiry_no_items_warmup";
1178    const char *data = "some test data here.";
1179
1180    const void* cookie = testHarness.create_cookie();
1181    auto ret = allocate(h,
1182                        h1,
1183                        cookie,
1184                        key,
1185                        strlen(data),
1186                        0,
1187                        0,
1188                        PROTOCOL_BINARY_RAW_BYTES,
1189                        0);
1190    checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
1191
1192    item_info info;
1193    if (!h1->get_item_info(h, ret.second.get(), &info)) {
1194        abort();
1195    }
1196    memcpy(info.value[0].iov_base, data, strlen(data));
1197
1198    uint64_t cas = 0;
1199    auto rv = h1->store(h,
1200                        cookie,
1201                        ret.second.get(),
1202                        cas,
1203                        OPERATION_SET,
1204                        DocumentState::Alive);
1205    checkeq(ENGINE_SUCCESS, rv, "Set failed.");
1206    check_key_value(h, h1, key, data, strlen(data));
1207    wait_for_flusher_to_settle(h, h1);
1208
1209    // Add a new item with the same key and 2 sec of expiration.
1210    const char *new_data = "new data here.";
1211    ret = allocate(h,
1212                   h1,
1213                   cookie,
1214                   key,
1215                   strlen(new_data),
1216                   0,
1217                   2,
1218                   PROTOCOL_BINARY_RAW_BYTES,
1219                   0);
1220    checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
1221
1222    if (!h1->get_item_info(h, ret.second.get(), &info)) {
1223        abort();
1224    }
1225    memcpy(info.value[0].iov_base, new_data, strlen(new_data));
1226
1227    int pager_runs = get_int_stat(h, h1, "ep_num_expiry_pager_runs");
1228    cas = 0;
1229    rv = h1->store(h,
1230                   cookie,
1231                   ret.second.get(),
1232                   cas,
1233                   OPERATION_SET,
1234                   DocumentState::Alive);
1235    checkeq(ENGINE_SUCCESS, rv, "Set failed.");
1236    check_key_value(h, h1, key, new_data, strlen(new_data));
1237    ret.second.reset();
1238    testHarness.time_travel(3);
1239    wait_for_stat_change(h, h1, "ep_num_expiry_pager_runs", pager_runs);
1240    wait_for_flusher_to_settle(h, h1);
1241
1242    // Restart the engine.
1243    testHarness.reload_engine(&h, &h1,
1244                              testHarness.engine_path,
1245                              testHarness.get_current_testcase()->cfg,
1246                              true, false);
1247    wait_for_warmup_complete(h, h1);
1248    // TODO: modify this for a better test case
1249    cb_assert(0 == get_int_stat(h, h1, "ep_warmup_dups", "warmup"));
1250
1251    testHarness.destroy_cookie(cookie);
1252    return SUCCESS;
1253}
1254
1255static enum test_result test_get_replica_active_state(ENGINE_HANDLE *h,
1256                                                      ENGINE_HANDLE_V1 *h1) {
1257    protocol_binary_request_header *pkt;
1258    pkt = prepare_get_replica(h, h1, vbucket_state_active);
1259    checkeq(ENGINE_NOT_MY_VBUCKET,
1260            h1->unknown_command(
1261                    h, NULL, pkt, add_response, testHarness.doc_namespace),
1262            "Get Replica Failed");
1263
1264    cb_free(pkt);
1265    return SUCCESS;
1266}
1267
1268static enum test_result test_get_replica_pending_state(ENGINE_HANDLE *h,
1269                                                       ENGINE_HANDLE_V1 *h1) {
1270    protocol_binary_request_header *pkt;
1271
1272    const void *cookie = testHarness.create_cookie();
1273    testHarness.set_ewouldblock_handling(cookie, false);
1274    pkt = prepare_get_replica(h, h1, vbucket_state_pending);
1275    checkeq(ENGINE_NOT_MY_VBUCKET,
1276            h1->unknown_command(h, cookie, pkt, add_response, testHarness.doc_namespace),
1277            "Should have returned NOT_MY_VBUCKET for pending state");
1278    checkeq(1, get_int_stat(h, h1, "ep_num_not_my_vbuckets"), "Expected 1 get");
1279    testHarness.destroy_cookie(cookie);
1280    cb_free(pkt);
1281    return SUCCESS;
1282}
1283
1284static enum test_result test_get_replica_dead_state(ENGINE_HANDLE *h,
1285                                                    ENGINE_HANDLE_V1 *h1) {
1286    protocol_binary_request_header *pkt;
1287    pkt = prepare_get_replica(h, h1, vbucket_state_dead);
1288    checkeq(ENGINE_NOT_MY_VBUCKET,
1289            h1->unknown_command(
1290                    h, NULL, pkt, add_response, testHarness.doc_namespace),
1291            "Get Replica Failed");
1292    cb_free(pkt);
1293    return SUCCESS;
1294}
1295
1296static enum test_result test_get_replica(ENGINE_HANDLE *h,
1297                                         ENGINE_HANDLE_V1 *h1) {
1298    protocol_binary_request_header *pkt;
1299    pkt = prepare_get_replica(h, h1, vbucket_state_replica);
1300    checkeq(ENGINE_SUCCESS,
1301            h1->unknown_command(h, NULL, pkt, add_response, testHarness.doc_namespace),
1302            "Get Replica Failed");
1303    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1304            "Expected PROTOCOL_BINARY_RESPONSE_SUCCESS response.");
1305    checkeq(std::string("replicadata"), last_body,
1306            "Should have returned identical value");
1307
1308    cb_free(pkt);
1309    return SUCCESS;
1310}
1311
1312static enum test_result test_get_replica_non_resident(ENGINE_HANDLE *h,
1313                                                      ENGINE_HANDLE_V1 *h1) {
1314    checkeq(ENGINE_SUCCESS,
1315            store(h, h1, NULL, OPERATION_SET, "key", "value"),
1316            "Store Failed");
1317    wait_for_flusher_to_settle(h, h1);
1318    wait_for_stat_to_be(h, h1, "ep_total_persisted", 1);
1319
1320    evict_key(h, h1, "key", 0, "Ejected.");
1321    check(set_vbucket_state(h, h1, 0, vbucket_state_replica),
1322          "Failed to set vbucket to replica");
1323
1324    get_replica(h, h1, "key", 0);
1325    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1326            "Expected success");
1327
1328    return SUCCESS;
1329}
1330
1331static enum test_result test_get_replica_invalid_key(ENGINE_HANDLE *h,
1332                                                     ENGINE_HANDLE_V1 *h1) {
1333    protocol_binary_request_header *pkt;
1334    bool makeinvalidkey = true;
1335    pkt = prepare_get_replica(h, h1, vbucket_state_replica, makeinvalidkey);
1336    checkeq(ENGINE_NOT_MY_VBUCKET,
1337            h1->unknown_command(
1338                    h, NULL, pkt, add_response, testHarness.doc_namespace),
1339            "Get Replica Failed");
1340    cb_free(pkt);
1341    return SUCCESS;
1342}
1343
1344static enum test_result test_vb_del_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1345    const void *cookie = testHarness.create_cookie();
1346    testHarness.set_ewouldblock_handling(cookie, false);
1347    check(set_vbucket_state(h, h1, 1, vbucket_state_pending),
1348          "Failed to set vbucket state.");
1349    checkeq(ENGINE_EWOULDBLOCK, del(h, h1, "key", 0, 1, cookie),
1350            "Expected woodblock.");
1351    testHarness.destroy_cookie(cookie);
1352    return SUCCESS;
1353}
1354
1355static enum test_result test_vb_del_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1356    check(set_vbucket_state(h, h1, 1, vbucket_state_replica),
1357          "Failed to set vbucket state.");
1358    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1359    checkeq(ENGINE_NOT_MY_VBUCKET, del(h, h1, "key", 0, 1),
1360            "Expected not my vbucket.");
1361    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1362    return SUCCESS;
1363}
1364
1365static enum test_result test_vbucket_get_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1366    return verify_vbucket_missing(h, h1, 1) ? SUCCESS : FAIL;
1367}
1368
1369static enum test_result test_vbucket_get(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1370    return verify_vbucket_state(h, h1, 0, vbucket_state_active) ? SUCCESS : FAIL;
1371}
1372
1373static enum test_result test_vbucket_create(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1374    if (!verify_vbucket_missing(h, h1, 1)) {
1375        fprintf(stderr, "vbucket wasn't missing.\n");
1376        return FAIL;
1377    }
1378
1379    if (!set_vbucket_state(h, h1, 1, vbucket_state_active)) {
1380        fprintf(stderr, "set state failed.\n");
1381        return FAIL;
1382    }
1383
1384    return verify_vbucket_state(h, h1, 1, vbucket_state_active) ? SUCCESS : FAIL;
1385}
1386
1387static enum test_result test_takeover_stats_race_with_vb_create_DCP(
1388                                    ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1389    check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1390          "Failed to set vbucket state information");
1391
1392    checkeq(0,
1393            get_int_stat(h, h1, "on_disk_deletes", "dcp-vbtakeover 1"),
1394            "Invalid number of on-disk deletes");
1395
1396    return SUCCESS;
1397}
1398
1399static enum test_result test_takeover_stats_num_persisted_deletes(
1400                                    ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1401    /* set an item */
1402    std::string key("key");
1403    checkeq(ENGINE_SUCCESS,
1404            store(h, h1, NULL, OPERATION_SET, key.c_str(), "data"),
1405            "Failed to store an item");
1406
1407    /* delete the item */
1408    checkeq(ENGINE_SUCCESS, del(h, h1, key.c_str(), 0, 0),
1409            "Failed to delete the item");
1410
1411    /* wait for persistence */
1412    wait_for_flusher_to_settle(h, h1);
1413
1414    /* check if persisted deletes stats is got correctly */
1415    checkeq(1,
1416            get_int_stat(h, h1, "on_disk_deletes", "dcp-vbtakeover 0"),
1417            "Invalid number of on-disk deletes");
1418
1419    return SUCCESS;
1420}
1421
1422static enum test_result test_vbucket_compact(ENGINE_HANDLE *h,
1423                                             ENGINE_HANDLE_V1 *h1) {
1424    const char* exp_key = "Carss";
1425    const char* exp_value = "pollute";
1426    const char* non_exp_key = "trees";
1427    const char* non_exp_value = "cleanse";
1428
1429    // Set two keys - one to be expired and other to remain...
1430    // Set expiring key
1431    checkeq(ENGINE_SUCCESS,
1432            store(h, h1, NULL, OPERATION_SET, exp_key, exp_value),
1433            "Failed to set expiring key");
1434    check_key_value(h, h1, exp_key, exp_value, strlen(exp_value));
1435
1436    // Set a non-expiring key...
1437    checkeq(ENGINE_SUCCESS,
1438            store(h, h1, NULL, OPERATION_SET, non_exp_key, non_exp_value),
1439            "Failed to set non-expiring key");
1440    check_key_value(h, h1, non_exp_key, non_exp_value, strlen(non_exp_value));
1441
1442    // Touch expiring key with an expire time
1443    const int exp_time = 11;
1444    checkeq(ENGINE_SUCCESS,
1445            touch(h, h1, exp_key, 0, exp_time),
1446            "Touch expiring key failed");
1447
1448    // Move beyond expire time
1449    testHarness.time_travel(exp_time + 1);
1450
1451    // Wait for the item to be expired
1452    wait_for_stat_to_be(h, h1, "vb_active_expired", 1);
1453    const int exp_purge_seqno =
1454            get_int_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno");
1455
1456    // non_exp_key and its value should be intact...
1457    checkeq(ENGINE_SUCCESS,
1458            verify_key(h, h1, non_exp_key),
1459            "key trees should be found.");
1460    // exp_key should have disappeared...
1461    ENGINE_ERROR_CODE val = verify_key(h, h1, exp_key);
1462    checkeq(ENGINE_KEY_ENOENT, val, "Key Carss has not expired.");
1463
1464    // Store a dummy item since we do not purge the item with highest seqno
1465    checkeq(ENGINE_SUCCESS,
1466            store(h, h1, NULL, OPERATION_SET, "dummykey", "dummyvalue"),
1467            "Error setting dummy key");
1468    wait_for_flusher_to_settle(h, h1);
1469
1470    checkeq(0, get_int_stat(h, h1, "vb_0:purge_seqno", "vbucket-seqno"),
1471            "purge_seqno not found to be zero before compaction");
1472
1473    // Compaction on VBucket
1474    compact_db(
1475            h,
1476            h1,
1477            0 /* vbucket_id */,
1478            0 /* db_file_id */,
1479            2 /* purge_before_ts */,
1480            exp_purge_seqno - 1 /* purge_before_seq */,
1481            1 /* drop deletes (forces purge irrespective purge_before_seq) */);
1482    wait_for_stat_to_be(h, h1, "ep_pending_compactions", 0);
1483    checkeq(exp_purge_seqno,
1484            get_int_stat(h, h1, "vb_0:purge_seqno", "vbucket-seqno"),
1485            "purge_seqno didn't match expected value");
1486
1487    return SUCCESS;
1488}
1489
1490static enum test_result test_compaction_config(ENGINE_HANDLE *h,
1491                                               ENGINE_HANDLE_V1 *h1) {
1492
1493    checkeq(10000,
1494            get_int_stat(h, h1, "ep_compaction_write_queue_cap"),
1495            "Expected compaction queue cap to be 10000");
1496    set_param(h, h1, protocol_binary_engine_param_flush,
1497              "compaction_write_queue_cap", "100000");
1498    checkeq(100000, get_int_stat(h, h1, "ep_compaction_write_queue_cap"),
1499            "Expected compaction queue cap to be 100000");
1500    return SUCCESS;
1501}
1502
1503struct comp_thread_ctx {
1504    ENGINE_HANDLE *h;
1505    ENGINE_HANDLE_V1 *h1;
1506    uint16_t vbid;
1507    uint16_t db_file_id;
1508};
1509
1510extern "C" {
1511    static void compaction_thread(void *arg) {
1512        struct comp_thread_ctx *ctx = static_cast<comp_thread_ctx *>(arg);
1513        compact_db(ctx->h, ctx->h1, ctx->vbid, ctx->db_file_id, 0, 0, 0);
1514    }
1515}
1516
1517static enum test_result test_multiple_vb_compactions(ENGINE_HANDLE *h,
1518                                                     ENGINE_HANDLE_V1 *h1) {
1519    for (uint16_t i = 0; i < 4; ++i) {
1520        if (!set_vbucket_state(h, h1, i, vbucket_state_active)) {
1521            fprintf(stderr, "set state failed for vbucket %d.\n", i);
1522            return FAIL;
1523        }
1524        check(verify_vbucket_state(h, h1, i, vbucket_state_active),
1525              "VBucket state not active");
1526    }
1527
1528    std::vector<std::string> keys;
1529    for (int j = 0; j < 20000; ++j) {
1530        std::stringstream ss;
1531        ss << "key" << j;
1532        std::string key(ss.str());
1533        keys.push_back(key);
1534    }
1535
1536    int count = 0;
1537    std::vector<std::string>::iterator it;
1538    for (it = keys.begin(); it != keys.end(); ++it) {
1539        uint16_t vbid = count % 4;
1540        checkeq(ENGINE_SUCCESS,
1541                store(h,
1542                      h1,
1543                      NULL,
1544                      OPERATION_SET,
1545                      it->c_str(),
1546                      it->c_str(),
1547                      nullptr,
1548                      0,
1549                      vbid),
1550                "Failed to store a value");
1551        ++count;
1552    }
1553
1554    // Compact multiple vbuckets.
1555    const int n_threads = 4;
1556    cb_thread_t threads[n_threads];
1557    struct comp_thread_ctx ctx[n_threads];
1558
1559    const int num_shards = get_int_stat(h, h1, "ep_workload:num_shards",
1560                                        "workload");
1561
1562    for (int i = 0; i < n_threads; i++) {
1563        ctx[i].h = h;
1564        ctx[i].h1 = h1;
1565        ctx[i].vbid = static_cast<uint16_t>(i);
1566        ctx[i].db_file_id = ctx[i].vbid % num_shards;
1567        int r = cb_create_thread(&threads[i], compaction_thread, &ctx[i], 0);
1568        cb_assert(r == 0);
1569    }
1570
1571    for (int i = 0; i < n_threads; i++) {
1572        int r = cb_join_thread(threads[i]);
1573        cb_assert(r == 0);
1574    }
1575
1576    wait_for_stat_to_be(h, h1, "ep_pending_compactions", 0);
1577
1578    return SUCCESS;
1579}
1580
1581static enum test_result
1582test_multi_vb_compactions_with_workload(ENGINE_HANDLE *h,
1583                                        ENGINE_HANDLE_V1 *h1) {
1584    for (uint16_t i = 0; i < 4; ++i) {
1585        if (!set_vbucket_state(h, h1, i, vbucket_state_active)) {
1586            fprintf(stderr, "set state failed for vbucket %d.\n", i);
1587            return FAIL;
1588        }
1589        check(verify_vbucket_state(h, h1, i, vbucket_state_active),
1590              "VBucket state not active");
1591    }
1592
1593    std::vector<std::string> keys;
1594    for (int j = 0; j < 10000; ++j) {
1595        std::stringstream ss;
1596        ss << "key" << j;
1597        std::string key(ss.str());
1598        keys.push_back(key);
1599    }
1600
1601    int count = 0;
1602    std::vector<std::string>::iterator it;
1603    for (it = keys.begin(); it != keys.end(); ++it) {
1604        uint16_t vbid = count % 4;
1605        checkeq(ENGINE_SUCCESS,
1606                store(h,
1607                      h1,
1608                      NULL,
1609                      OPERATION_SET,
1610                      it->c_str(),
1611                      it->c_str(),
1612                      nullptr,
1613                      0,
1614                      vbid),
1615                "Failed to store a value");
1616        ++count;
1617    }
1618    wait_for_flusher_to_settle(h, h1);
1619
1620    for (int i = 0; i < 2; ++i) {
1621        count = 0;
1622        for (it = keys.begin(); it != keys.end(); ++it) {
1623            uint16_t vbid = count % 4;
1624            checkeq(cb::engine_errc::success,
1625                    get(h, h1, NULL, it->c_str(), vbid).first,
1626                    "Unable to get stored item");
1627            ++count;
1628        }
1629    }
1630    wait_for_stat_to_be(h, h1, "ep_workload_pattern", std::string{"read_heavy"});
1631
1632    // Compact multiple vbuckets.
1633    const int n_threads = 4;
1634    cb_thread_t threads[n_threads];
1635    struct comp_thread_ctx ctx[n_threads];
1636
1637    for (int i = 0; i < n_threads; i++) {
1638        ctx[i].h = h;
1639        ctx[i].h1 = h1;
1640        ctx[i].vbid = static_cast<uint16_t>(i);
1641        int r = cb_create_thread(&threads[i], compaction_thread, &ctx[i], 0);
1642        cb_assert(r == 0);
1643    }
1644
1645    for (int i = 0; i < n_threads; i++) {
1646        int r = cb_join_thread(threads[i]);
1647        cb_assert(r == 0);
1648    }
1649
1650    wait_for_stat_to_be(h, h1, "ep_pending_compactions", 0);
1651
1652    return SUCCESS;
1653}
1654
1655static enum test_result vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1656                                             const char* value = NULL) {
1657    check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1658          "Failed to set vbucket state.");
1659
1660    checkeq(ENGINE_NOT_MY_VBUCKET,
1661            vbucketDelete(h, h1, 2, value),
1662            "Expected NMVB");
1663
1664    check(set_vbucket_state(h, h1, 1, vbucket_state_dead),
1665          "Failed set set vbucket 1 state.");
1666
1667    checkeq(ENGINE_SUCCESS, vbucketDelete(h, h1, 1, value), "Expected success");
1668    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1669            "Expected failure deleting non-existent bucket.");
1670
1671    check(verify_vbucket_missing(h, h1, 1),
1672          "vbucket 0 was not missing after deleting it.");
1673
1674    return SUCCESS;
1675}
1676
1677static enum test_result test_vbucket_destroy_stats(ENGINE_HANDLE *h,
1678                                                   ENGINE_HANDLE_V1 *h1) {
1679
1680    int cacheSize = get_int_stat(h, h1, "ep_total_cache_size");
1681    int overhead = get_int_stat(h, h1, "ep_overhead");
1682    int nonResident = get_int_stat(h, h1, "ep_num_non_resident");
1683
1684    check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1685          "Failed to set vbucket state.");
1686
1687    std::vector<std::string> keys;
1688    for (int j = 0; j < 2000; ++j) {
1689        std::stringstream ss;
1690        ss << "key" << j;
1691        std::string key(ss.str());
1692        keys.push_back(key);
1693    }
1694
1695    int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
1696    std::vector<std::string>::iterator it;
1697    for (it = keys.begin(); it != keys.end(); ++it) {
1698        checkeq(ENGINE_SUCCESS,
1699                store(h,
1700                      h1,
1701                      NULL,
1702                      OPERATION_SET,
1703                      it->c_str(),
1704                      it->c_str(),
1705                      nullptr,
1706                      0,
1707                      1),
1708                "Failed to store a value");
1709    }
1710    wait_for_flusher_to_settle(h, h1);
1711    testHarness.time_travel(65);
1712    wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
1713
1714    check(set_vbucket_state(h, h1, 1, vbucket_state_dead),
1715          "Failed set set vbucket 1 state.");
1716
1717    int vbucketDel = get_int_stat(h, h1, "ep_vbucket_del");
1718    checkeq(ENGINE_SUCCESS, vbucketDelete(h, h1, 1), "Expected success");
1719    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS,
1720            last_status.load(),
1721            "Expected failure deleting non-existent bucket.");
1722
1723    check(verify_vbucket_missing(h, h1, 1),
1724          "vbucket 1 was not missing after deleting it.");
1725
1726    wait_for_stat_change(h, h1, "ep_vbucket_del", vbucketDel);
1727
1728    wait_for_stat_to_be(h, h1, "ep_total_cache_size", cacheSize);
1729    wait_for_stat_to_be(h, h1, "ep_overhead", overhead);
1730    wait_for_stat_to_be(h, h1, "ep_num_non_resident", nonResident);
1731
1732    return SUCCESS;
1733}
1734
1735static enum test_result vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1736                                                const char* value = NULL) {
1737    if (!isWarmupEnabled(h, h1)) {
1738        return SKIPPED;
1739    }
1740
1741    check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1742          "Failed to set vbucket state.");
1743
1744    // Store a value so the restart will try to resurrect it.
1745    checkeq(ENGINE_SUCCESS,
1746            store(h,
1747                  h1,
1748                  NULL,
1749                  OPERATION_SET,
1750                  "key",
1751                  "somevalue",
1752                  nullptr,
1753                  0,
1754                  1),
1755            "Failed to set a value");
1756    check_key_value(h, h1, "key", "somevalue", 9, 1);
1757
1758    // Reload to get a flush forced.
1759    testHarness.reload_engine(&h, &h1,
1760                              testHarness.engine_path,
1761                              testHarness.get_current_testcase()->cfg,
1762                              true, false);
1763    wait_for_warmup_complete(h, h1);
1764
1765    check(verify_vbucket_state(h, h1, 1, vbucket_state_active),
1766          "Bucket state was what it was initially, after restart.");
1767    check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1768          "Failed to set vbucket state.");
1769    check_key_value(h, h1, "key", "somevalue", 9, 1);
1770
1771    check(set_vbucket_state(h, h1, 1, vbucket_state_dead),
1772          "Failed set set vbucket 1 state.");
1773
1774    checkeq(ENGINE_SUCCESS, vbucketDelete(h, h1, 1, value), "Expected success");
1775    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1776            "Expected failure deleting non-existent bucket.");
1777
1778    check(verify_vbucket_missing(h, h1, 1),
1779          "vbucket 1 was not missing after deleting it.");
1780
1781    testHarness.reload_engine(&h, &h1,
1782                              testHarness.engine_path,
1783                              testHarness.get_current_testcase()->cfg,
1784                              true, false);
1785    wait_for_warmup_complete(h, h1);
1786
1787    if (verify_vbucket_state(h, h1, 1, vbucket_state_pending, true)) {
1788        std::cerr << "Bucket came up in pending state after delete." << std::endl;
1789        abort();
1790    }
1791
1792    check(verify_vbucket_missing(h, h1, 1),
1793          "vbucket 1 was not missing after restart.");
1794
1795    return SUCCESS;
1796}
1797
1798static enum test_result test_async_vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1799    return vbucket_destroy(h, h1);
1800}
1801
1802static enum test_result test_sync_vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1803    return vbucket_destroy(h, h1, "async=0");
1804}
1805
1806static enum test_result test_async_vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1807    return vbucket_destroy_restart(h, h1);
1808}
1809
1810static enum test_result test_sync_vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1811    return vbucket_destroy_restart(h, h1, "async=0");
1812}
1813
1814static enum test_result test_vb_set_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1815    return test_pending_vb_mutation(h, h1, OPERATION_SET);
1816}
1817
1818static enum test_result test_vb_add_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1819    return test_pending_vb_mutation(h, h1, OPERATION_ADD);
1820}
1821
1822static enum test_result test_vb_cas_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1823    return test_pending_vb_mutation(h, h1, OPERATION_CAS);
1824}
1825
1826static enum test_result test_vb_set_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1827    return test_replica_vb_mutation(h, h1, OPERATION_SET);
1828}
1829
1830static enum test_result test_vb_replace_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1831    return test_replica_vb_mutation(h, h1, OPERATION_REPLACE);
1832}
1833
1834static enum test_result test_vb_replace_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1835    return test_pending_vb_mutation(h, h1, OPERATION_REPLACE);
1836}
1837
1838static enum test_result test_vb_add_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1839    return test_replica_vb_mutation(h, h1, OPERATION_ADD);
1840}
1841
1842static enum test_result test_vb_cas_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1843    return test_replica_vb_mutation(h, h1, OPERATION_CAS);
1844}
1845
1846static enum test_result test_stats_seqno(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1847    check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1848          "Failed to set vbucket state.");
1849
1850    int num_keys = 100;
1851    for (int ii = 0; ii < num_keys; ++ii) {
1852        std::stringstream ss;
1853        ss << "key" << ii;
1854        checkeq(ENGINE_SUCCESS,
1855                store(h, h1, NULL, OPERATION_SET, ss.str().c_str(),
1856                      "value", NULL, 0, 0),
1857                "Failed to store an item.");
1858    }
1859    wait_for_flusher_to_settle(h, h1);
1860
1861    checkeq(100, get_int_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno"),
1862            "Invalid seqno");
1863
1864    if (isPersistentBucket(h, h1)) {
1865        checkeq(100,
1866                get_int_stat(h, h1, "vb_0:last_persisted_seqno", "vbucket-seqno"),
1867                "Unexpected last_persisted_seqno");
1868    }
1869    checkeq(0, get_int_stat(h, h1, "vb_1:high_seqno", "vbucket-seqno"),
1870            "Invalid seqno");
1871    checkeq(0, get_int_stat(h, h1, "vb_1:high_seqno", "vbucket-seqno 1"),
1872            "Invalid seqno");
1873    if (isPersistentBucket(h, h1)) {
1874        checkeq(0,
1875                get_int_stat(h, h1, "vb_1:last_persisted_seqno", "vbucket-seqno 1"),
1876                "Invalid last_persisted_seqno");
1877    }
1878
1879    uint64_t vb_uuid = get_ull_stat(h, h1, "vb_1:0:id", "failovers");
1880
1881    auto seqno_stats = get_all_stats(h, h1, "vbucket-seqno 1");
1882    checkeq(vb_uuid, uint64_t(std::stoull(seqno_stats.at("vb_1:uuid"))),
1883            "Invalid uuid");
1884
1885    checkeq(size_t(7), seqno_stats.size(), "Expected seven stats");
1886
1887    // Check invalid vbucket
1888    checkeq(ENGINE_NOT_MY_VBUCKET,
1889            get_stats(h, "vbucket-seqno 2"_ccb, add_stats),
1890            "Expected not my vbucket");
1891
1892    // Check bad vbucket parameter (not numeric)
1893    checkeq(ENGINE_EINVAL,
1894            get_stats(h, "vbucket-seqno tt2"_ccb, add_stats),
1895            "Expected invalid");
1896
1897    // Check extra spaces at the end
1898    checkeq(ENGINE_EINVAL,
1899            get_stats(h, "vbucket-seqno    "_ccb, add_stats),
1900            "Expected invalid");
1901
1902    return SUCCESS;
1903}
1904
1905static enum test_result test_stats_diskinfo(ENGINE_HANDLE *h,
1906                                            ENGINE_HANDLE_V1 *h1) {
1907    check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1908          "Failed to set vbucket state.");
1909
1910    int num_keys = 100;
1911    for (int ii = 0; ii < num_keys; ++ii) {
1912        std::stringstream ss;
1913        ss << "key" << ii;
1914        checkeq(ENGINE_SUCCESS,
1915                store(h, h1, NULL, OPERATION_SET, ss.str().c_str(),
1916                      "value", NULL, 0, 1),
1917                "Failed to store an item.");
1918    }
1919    wait_for_flusher_to_settle(h, h1);
1920
1921    size_t file_size = get_int_stat(h, h1, "ep_db_file_size", "diskinfo");
1922    size_t data_size = get_int_stat(h, h1, "ep_db_data_size", "diskinfo");
1923    check(file_size > 0, "DB file size should be greater than 0");
1924    check(data_size > 0, "DB data size should be greater than 0");
1925    check(file_size >= data_size, "DB file size should be >= DB data size");
1926    check(get_int_stat(h, h1, "vb_1:data_size", "diskinfo detail") > 0,
1927          "VB 1 data size should be greater than 0");
1928
1929    checkeq(ENGINE_EINVAL,
1930            get_stats(h, "diskinfo "_ccb, add_stats),
1931            "Expected invalid");
1932
1933    checkeq(ENGINE_EINVAL,
1934            get_stats(h, "diskinfo detai"_ccb, add_stats),
1935            "Expected invalid");
1936
1937    checkeq(ENGINE_EINVAL,
1938            get_stats(h, "diskinfo detaillll"_ccb, add_stats),
1939            "Expected invalid");
1940
1941    return SUCCESS;
1942}
1943
1944static enum test_result test_uuid_stats(ENGINE_HANDLE *h,
1945                                        ENGINE_HANDLE_V1 *h1)
1946{
1947    vals.clear();
1948    checkeq(ENGINE_SUCCESS,
1949            get_stats(h, "uuid"_ccb, add_stats),
1950            "Failed to get stats.");
1951    check(vals["uuid"] == "foobar", "Incorrect uuid");
1952    return SUCCESS;
1953}
1954
1955static enum test_result test_item_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1956    checkeq(ENGINE_SUCCESS,
1957            store(h, h1, NULL, OPERATION_SET, "key", "somevalue"),
1958            "Failed set.");
1959    wait_for_flusher_to_settle(h, h1);
1960    checkeq(ENGINE_SUCCESS,
1961            store(h, h1, NULL, OPERATION_SET, "key", "somevalueX"),
1962            "Failed set.");
1963    wait_for_flusher_to_settle(h, h1);
1964    checkeq(ENGINE_SUCCESS,
1965            store(h, h1, NULL, OPERATION_SET, "key1", "somevalueY"),
1966            "Failed set.");
1967    wait_for_flusher_to_settle(h, h1);
1968
1969    check_key_value(h, h1, "key", "somevalueX", 10);
1970    check_key_value(h, h1, "key1", "somevalueY", 10);
1971
1972    checkeq(ENGINE_SUCCESS, del(h, h1, "key1", 0, 0),
1973            "Failed remove with value.");
1974    wait_for_flusher_to_settle(h, h1);
1975
1976    checkeq(ENGINE_SUCCESS,
1977            store(h, h1, NULL, OPERATION_SET, "key1", "someothervalue"),
1978            "Failed set.");
1979    wait_for_flusher_to_settle(h, h1);
1980
1981    check_key_value(h, h1, "key1", "someothervalue", 14);
1982
1983    checkeq(3,
1984            get_int_stat(h, h1, "vb_active_ops_create"),
1985            "Expected 3 creations");
1986    checkeq(1,
1987            get_int_stat(h, h1, "vb_active_ops_update"),
1988            "Expected 1 updation");
1989    checkeq(1,
1990            get_int_stat(h, h1, "vb_active_ops_delete"),
1991            "Expected 1 deletion");
1992
1993    return SUCCESS;
1994}
1995
1996static enum test_result test_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1997    vals.clear();
1998    checkeq(ENGINE_SUCCESS,
1999            get_stats(h, {}, add_stats),
2000            "Failed to get stats.");
2001    check(vals.size() > 10, "Kind of expected more stats than that.");
2002
2003    return SUCCESS;
2004}
2005
2006static enum test_result test_mem_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2007    char value[2048];
2008    memset(value, 'b', sizeof(value));
2009    strcpy(value + sizeof(value) - 4, "\r\n");
2010    int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
2011    wait_for_persisted_value(h, h1, "key", value);
2012    testHarness.time_travel(65);
2013    if (isPersistentBucket(h, h1)) {
2014        wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
2015    }
2016
2017    if (isActiveCompressionEnabled(h, h1)) {
2018        wait_for_item_compressor_to_settle(h, h1);
2019    }
2020
2021    int mem_used = get_int_stat(h, h1, "mem_used");
2022    int cache_size = get_int_stat(h, h1, "ep_total_cache_size");
2023    int overhead = get_int_stat(h, h1, "ep_overhead");
2024    int value_size = get_int_stat(h, h1, "ep_value_size");
2025    check((mem_used - overhead) > cache_size,
2026          "ep_kv_size should be greater than the hashtable cache size due to "
2027          "the checkpoint overhead");
2028
2029    if (isPersistentBucket(h, h1)) {
2030        evict_key(h, h1, "key", 0, "Ejected.");
2031
2032        check(get_int_stat(h, h1, "ep_total_cache_size") <= cache_size,
2033              "Evict a value shouldn't increase the total cache size");
2034        check(get_int_stat(h, h1, "mem_used") < mem_used,
2035              "Expected mem_used to decrease when an item is evicted");
2036
2037        check_key_value(h, h1, "key", value, strlen(value), 0); // Load an item from disk again.
2038
2039        if (isActiveCompressionEnabled(h, h1)) {
2040            wait_for_item_compressor_to_settle(h, h1);
2041        }
2042
2043        check(get_int_stat(h, h1, "mem_used") >= mem_used,
2044              "Expected mem_used to remain the same after an item is loaded from disk");
2045        check(get_int_stat(h, h1, "ep_value_size") == value_size,
2046              "Expected ep_value_size to remain the same after item is "
2047              "loaded from disk");
2048    }
2049
2050    return SUCCESS;
2051}
2052
2053static enum test_result test_io_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2054    int exp_write_bytes;
2055    std::string backend = get_str_stat(h, h1, "ep_backend");
2056    if (backend == "couchdb") {
2057        exp_write_bytes = 22; /* TBD: Do not hard code the value */
2058    } else if (backend == "rocksdb") {
2059        // TODO RDB:
2060        return SKIPPED_UNDER_ROCKSDB;
2061    }
2062    else {
2063        return SKIPPED;
2064    }
2065
2066    reset_stats(h);
2067
2068    checkeq(0,
2069            get_int_stat(h, h1, "rw_0:io_bg_fetch_docs_read", "kvstore"),
2070            "Expected reset stats to set io_bg_fetch_docs_read to zero");
2071    checkeq(0, get_int_stat(h, h1, "rw_0:io_num_write", "kvstore"),
2072            "Expected reset stats to set io_num_write to zero");
2073    checkeq(0,
2074            get_int_stat(h, h1, "rw_0:io_bg_fetch_doc_bytes", "kvstore"),
2075            "Expected reset stats to set io_bg_fetch_doc_bytes to zero");
2076    checkeq(0, get_int_stat(h, h1, "rw_0:io_write_bytes", "kvstore"),
2077            "Expected reset stats to set io_write_bytes to zero");
2078
2079    const std::string key("a");
2080    const std::string value("b\r\n");
2081    wait_for_persisted_value(h, h1, key.c_str(), value.c_str());
2082    checkeq(0,
2083            get_int_stat(h, h1, "rw_0:io_bg_fetch_docs_read", "kvstore"),
2084            "Expected storing one value to not change the read counter");
2085    checkeq(0,
2086            get_int_stat(h, h1, "rw_0:io_bg_fetch_doc_bytes", "kvstore"),
2087            "Expected storing one value to not change the bgfetch doc bytes");
2088    checkeq(1, get_int_stat(h, h1, "rw_0:io_num_write", "kvstore"),
2089            "Expected storing the key to update the write counter");
2090    checkeq(exp_write_bytes,
2091            get_int_stat(h, h1, "rw_0:io_write_bytes", "kvstore"),
2092            "Expected storing the key to update the write bytes");
2093
2094    evict_key(h, h1, key.c_str(), 0, "Ejected.");
2095
2096    check_key_value(h, h1, "a", value.c_str(), value.size(), 0);
2097
2098    std::stringstream numReadStatStr;
2099    std::stringstream readBytesStatStr;
2100
2101    if (backend == "couchdb") {
2102        numReadStatStr << "ro_" << 0 << ":io_bg_fetch_docs_read";
2103        readBytesStatStr << "ro_" << 0 << ":io_bg_fetch_doc_bytes";
2104    } else {
2105        cb_assert(false);
2106    }
2107
2108    checkeq(1,
2109            get_int_stat(h, h1, numReadStatStr.str().c_str(), "kvstore"),
2110            "Expected reading the value back in to update the read counter");
2111
2112    const uint64_t exp_read_bytes =
2113            key.size() + value.size() +
2114            MetaData::getMetaDataSize(MetaData::Version::V1);
2115    checkeq(exp_read_bytes,
2116            get_stat<uint64_t>(
2117                    h, h1, readBytesStatStr.str().c_str(), "kvstore"),
2118            "Expected reading the value back in to update the read bytes");
2119
2120    // For read amplification, exact value depends on couchstore file layout,
2121    // but generally see a value of 2 here.
2122    checkge(get_float_stat(h, h1, "ep_bg_fetch_avg_read_amplification"),
2123            2.0f,
2124            "Expected sensible bgFetch read amplification value");
2125
2126    checkeq(1, get_int_stat(h, h1, "rw_0:io_num_write", "kvstore"),
2127            "Expected reading the value back in to not update the write counter");
2128    checkeq(exp_write_bytes,
2129            get_int_stat(h, h1, "rw_0:io_write_bytes", "kvstore"),
2130            "Expected reading the value back in to not update the write bytes");
2131
2132    return SUCCESS;
2133}
2134
2135static enum test_result test_vb_file_stats(ENGINE_HANDLE *h,
2136                                        ENGINE_HANDLE_V1 *h1) {
2137    wait_for_flusher_to_settle(h, h1);
2138    wait_for_stat_change(h, h1, "ep_db_data_size", 0);
2139
2140    int old_data_size = get_int_stat(h, h1, "ep_db_data_size");
2141    int old_file_size = get_int_stat(h, h1, "ep_db_file_size");
2142    check(old_file_size != 0, "Expected a non-zero value for ep_db_file_size");
2143
2144    // Write a value and test ...
2145    wait_for_persisted_value(h, h1, "a", "b\r\n");
2146    check(get_int_stat(h, h1, "ep_db_data_size") > old_data_size,
2147          "Expected the DB data size to increase");
2148    check(get_int_stat(h, h1, "ep_db_file_size") > old_file_size,
2149          "Expected the DB file size to increase");
2150
2151    check(get_int_stat(h, h1, "vb_0:db_data_size", "vbucket-details 0") > 0,
2152          "Expected the vbucket DB data size to non-zero");
2153    check(get_int_stat(h, h1, "vb_0:db_file_size", "vbucket-details 0") > 0,
2154          "Expected the vbucket DB file size to non-zero");
2155    return SUCCESS;
2156}
2157
2158static enum test_result test_vb_file_stats_after_warmup(ENGINE_HANDLE *h,
2159                                                        ENGINE_HANDLE_V1 *h1) {
2160    if (!isWarmupEnabled(h, h1)) {
2161        return SKIPPED;
2162    }
2163
2164    for (int i = 0; i < 100; ++i) {
2165        std::stringstream key;
2166        key << "key-" << i;
2167        checkeq(ENGINE_SUCCESS,
2168                store(h,
2169                      h1,
2170                      NULL,
2171                      OPERATION_SET,
2172                      key.str().c_str(),
2173                      "somevalue"),
2174                "Error setting.");
2175    }
2176    wait_for_flusher_to_settle(h, h1);
2177
2178    int fileSize = get_int_stat(h, h1, "vb_0:db_file_size", "vbucket-details 0");
2179    int spaceUsed = get_int_stat(h, h1, "vb_0:db_data_size", "vbucket-details 0");
2180
2181    // Restart the engine.
2182    testHarness.reload_engine(&h, &h1,
2183                              testHarness.engine_path,
2184                              testHarness.get_current_testcase()->cfg,
2185                              true, false);
2186    wait_for_warmup_complete(h, h1);
2187
2188    int newFileSize = get_int_stat(h, h1, "vb_0:db_file_size", "vbucket-details 0");
2189    int newSpaceUsed = get_int_stat(h, h1, "vb_0:db_data_size", "vbucket-details 0");
2190
2191    check((float)newFileSize >= 0.9 * fileSize, "Unexpected fileSize for vbucket");
2192    check((float)newSpaceUsed >= 0.9 * spaceUsed, "Unexpected spaceUsed for vbucket");
2193
2194    return SUCCESS;
2195}
2196
2197static enum test_result test_bg_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2198    reset_stats(h);
2199    wait_for_persisted_value(h, h1, "a", "b\r\n");
2200    evict_key(h, h1, "a", 0, "Ejected.");
2201    testHarness.time_travel(43);
2202    check_key_value(h, h1, "a", "b\r\n", 3, 0);
2203
2204    auto stats = get_all_stats(h, h1);
2205    checkeq(1, std::stoi(stats.at("ep_bg_num_samples")),
2206               "Expected one sample");
2207
2208    const char* bg_keys[] = { "ep_bg_min_wait",
2209                              "ep_bg_max_wait",
2210                              "ep_bg_wait_avg",
2211                              "ep_bg_min_load",
2212                              "ep_bg_max_load",
2213                              "ep_bg_load_avg"};
2214    for (const auto* key : bg_keys) {
2215        check(stats.find(key) != stats.end(),
2216              (std::string("Found no ") + key).c_str());
2217    }
2218
2219    evict_key(h, h1, "a", 0, "Ejected.");
2220    check_key_value(h, h1, "a", "b\r\n", 3, 0);
2221    check(get_int_stat(h, h1, "ep_bg_num_samples") == 2,
2222          "Expected one sample");
2223
2224    reset_stats(h);
2225    checkeq(0, get_int_stat(h, h1, "ep_bg_fetched"),
2226            "ep_bg_fetched is not reset to 0");
2227    return SUCCESS;
2228}
2229
2230static enum test_result test_bg_meta_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2231    reset_stats(h);
2232
2233    wait_for_persisted_value(h, h1, "k1", "v1");
2234    wait_for_persisted_value(h, h1, "k2", "v2");
2235
2236    evict_key(h, h1, "k1", 0, "Ejected.");
2237    checkeq(ENGINE_SUCCESS,
2238            del(h, h1, "k2", 0, 0), "Failed remove with value.");
2239    wait_for_flusher_to_settle(h, h1);
2240
2241    checkeq(0, get_int_stat(h, h1, "ep_bg_fetched"), "Expected bg_fetched to be 0");
2242    checkeq(0, get_int_stat(h, h1, "ep_bg_meta_fetched"), "Expected bg_meta_fetched to be 0");
2243
2244    check(get_meta(h, h1, "k2"), "Get meta failed");
2245    checkeq(0, get_int_stat(h, h1, "ep_bg_fetched"), "Expected bg_fetched to be 0");
2246    checkeq(1, get_int_stat(h, h1, "ep_bg_meta_fetched"), "Expected bg_meta_fetched to be 1");
2247
2248    checkeq(cb::engine_errc::success,
2249            get(h, h1, NULL, "k1", 0).first,
2250            "Missing key");
2251    checkeq(1, get_int_stat(h, h1, "ep_bg_fetched"), "Expected bg_fetched to be 1");
2252    checkeq(1, get_int_stat(h, h1, "ep_bg_meta_fetched"), "Expected bg_meta_fetched to be 1");
2253
2254    // store new key with some random metadata
2255    const size_t keylen = strlen("k3");
2256    ItemMetaData itemMeta;
2257    itemMeta.revSeqno = 10;
2258    itemMeta.cas = 0xdeadbeef;
2259    itemMeta.exptime = 0;
2260    itemMeta.flags = 0xdeadbeef;
2261
2262    add_with_meta(h, h1, "k3", keylen, NULL, 0, 0, &itemMeta);
2263    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Set meta failed");
2264
2265    check(get_meta(h, h1, "k2"), "Get meta failed");
2266    checkeq(1, get_int_stat(h, h1, "ep_bg_fetched"), "Expected bg_fetched to be 1");
2267    checkeq(1, get_int_stat(h, h1, "ep_bg_meta_fetched"),
2268            "Expected bg_meta_fetched to remain at 1");
2269
2270    return SUCCESS;
2271}
2272
2273static enum test_result test_key_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2274    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed set vbucket 1 state.");
2275
2276    // set (k1,v1) in vbucket 0
2277    checkeq(ENGINE_SUCCESS,
2278            store(h, h1, NULL, OPERATION_SET, "k1", "v1"),
2279            "Failed to store an item.");
2280
2281    // set (k2,v2) in vbucket 1
2282    checkeq(ENGINE_SUCCESS,
2283            store(h, h1, NULL, OPERATION_SET, "k2", "v2", nullptr, 0, 1),
2284            "Failed to store an item.");
2285
2286    const void *cookie = testHarness.create_cookie();
2287
2288    // stat for key "k1" and vbucket "0"
2289    const char *statkey1 = "key k1 0";
2290    checkeq(ENGINE_SUCCESS,
2291            h1->get_stats(h, cookie, {statkey1, strlen(statkey1)}, add_stats),
2292            "Failed to get stats.");
2293    check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2294    check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2295    check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2296    check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2297    check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2298
2299    // stat for key "k2" and vbucket "1"
2300    const char *statkey2 = "key k2 1";
2301    checkeq(ENGINE_SUCCESS,
2302            h1->get_stats(h, cookie, {statkey2, strlen(statkey2)}, add_stats),
2303            "Failed to get stats.");
2304    check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2305    check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2306    check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2307    check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2308    check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2309
2310    testHarness.destroy_cookie(cookie);
2311    return SUCCESS;
2312}
2313
2314static enum test_result test_vkey_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2315    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed set vbucket 1 state.");
2316    check(set_vbucket_state(h, h1, 2, vbucket_state_active), "Failed set vbucket 2 state.");
2317    check(set_vbucket_state(h, h1, 3, vbucket_state_active), "Failed set vbucket 3 state.");
2318    check(set_vbucket_state(h, h1, 4, vbucket_state_active), "Failed set vbucket 4 state.");
2319
2320    wait_for_persisted_value(h, h1, "k1", "v1");
2321    wait_for_persisted_value(h, h1, "k2", "v2", 1);
2322    wait_for_persisted_value(h, h1, "k3", "v3", 2);
2323    wait_for_persisted_value(h, h1, "k4", "v4", 3);
2324    wait_for_persisted_value(h, h1, "k5", "v5", 4);
2325
2326    check(set_vbucket_state(h, h1, 2, vbucket_state_replica), "Failed to set VB2 state.");
2327    check(set_vbucket_state(h, h1, 3, vbucket_state_pending), "Failed to set VB3 state.");
2328    check(set_vbucket_state(h, h1, 4, vbucket_state_dead), "Failed to set VB4 state.");
2329
2330    const void *cookie = testHarness.create_cookie();
2331
2332    // stat for key "k1" and vbucket "0"
2333    const char *statkey1 = "vkey k1 0";
2334    checkeq(ENGINE_SUCCESS,
2335            h1->get_stats(h, cookie, {statkey1, strlen(statkey1)}, add_stats),
2336            "Failed to get stats.");
2337    check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2338    check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2339    check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2340    check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2341    check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2342    check(vals.find("key_valid") != vals.end(), "Found no key_valid");
2343
2344    // stat for key "k2" and vbucket "1"
2345    const char *statkey2 = "vkey k2 1";
2346    checkeq(ENGINE_SUCCESS,
2347            h1->get_stats(h, cookie, {statkey2, strlen(statkey2)}, add_stats),
2348            "Failed to get stats.");
2349    check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2350    check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2351    check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2352    check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2353    check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2354    check(vals.find("key_valid") != vals.end(), "Found no key_valid");
2355
2356    // stat for key "k3" and vbucket "2"
2357    const char *statkey3 = "vkey k3 2";
2358    checkeq(ENGINE_SUCCESS,
2359            h1->get_stats(h, cookie, {statkey3, strlen(statkey3)}, add_stats),
2360            "Failed to get stats.");
2361    check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2362    check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2363    check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2364    check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2365    check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2366    check(vals.find("key_valid") != vals.end(), "Found no key_valid");
2367
2368    // stat for key "k4" and vbucket "3"
2369    const char *statkey4 = "vkey k4 3";
2370    checkeq(ENGINE_SUCCESS,
2371            h1->get_stats(h, cookie, {statkey4, strlen(statkey4)}, add_stats),
2372            "Failed to get stats.");
2373    check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2374    check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2375    check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2376    check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2377    check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2378    check(vals.find("key_valid") != vals.end(), "Found no key_valid");
2379
2380    // stat for key "k5" and vbucket "4"
2381    const char *statkey5 = "vkey k5 4";
2382    checkeq(ENGINE_SUCCESS,
2383            h1->get_stats(h, cookie, {statkey5, strlen(statkey5)}, add_stats),
2384            "Failed to get stats.");
2385    check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2386    check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2387    check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2388    check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2389    check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2390    check(vals.find("key_valid") != vals.end(), "Found no key_valid");
2391
2392    testHarness.destroy_cookie(cookie);
2393    return SUCCESS;
2394}
2395
2396static enum test_result test_warmup_conf(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2397    if (!isWarmupEnabled(h, h1)) {
2398        return SKIPPED;
2399    }
2400
2401    checkeq(100, get_int_stat(h, h1, "ep_warmup_min_items_threshold"),
2402            "Incorrect initial warmup min items threshold.");
2403    checkeq(100, get_int_stat(h, h1, "ep_warmup_min_memory_threshold"),
2404            "Incorrect initial warmup min memory threshold.");
2405
2406    check(!set_param(h, h1, protocol_binary_engine_param_flush,
2407                     "warmup_min_items_threshold", "a"),
2408          "Set warmup_min_items_threshold should have failed");
2409    check(!set_param(h, h1, protocol_binary_engine_param_flush,
2410                     "warmup_min_items_threshold", "a"),
2411          "Set warmup_min_memory_threshold should have failed");
2412
2413    check(set_param(h, h1, protocol_binary_engine_param_flush,
2414                    "warmup_min_items_threshold", "80"),
2415          "Set warmup_min_items_threshold should have worked");
2416    check(set_param(h, h1, protocol_binary_engine_param_flush,
2417                    "warmup_min_memory_threshold", "80"),
2418          "Set warmup_min_memory_threshold should have worked");
2419
2420    checkeq(80, get_int_stat(h, h1, "ep_warmup_min_items_threshold"),
2421            "Incorrect smaller warmup min items threshold.");
2422    checkeq(80, get_int_stat(h, h1, "ep_warmup_min_memory_threshold"),
2423            "Incorrect smaller warmup min memory threshold.");
2424
2425    for (int i = 0; i < 100; ++i) {
2426        std::stringstream key;
2427        key << "key-" << i;
2428        checkeq(ENGINE_SUCCESS,
2429                store(h,
2430                      h1,
2431                      NULL,
2432                      OPERATION_SET,
2433                      key.str().c_str(),
2434                      "somevalue"),
2435                "Error setting.");
2436    }
2437
2438    // Restart the server.
2439    std::string config(testHarness.get_current_testcase()->cfg);
2440    config = config + "warmup_min_memory_threshold=0";
2441    testHarness.reload_engine(&h, &h1,
2442                              testHarness.engine_path,
2443                              config.c_str(),
2444                              true, false);
2445    wait_for_warmup_complete(h, h1);
2446
2447    const std::string eviction_policy = get_str_stat(h, h1, "ep_item_eviction_policy");
2448    if (eviction_policy == "value_only") {
2449        checkeq(100, get_int_stat(h, h1, "ep_warmup_key_count", "warmup"),
2450                "Expected 100 keys loaded after warmup");
2451    } else { // Full eviction mode
2452        checkeq(0, get_int_stat(h, h1, "ep_warmup_key_count", "warmup"),
2453                "Expected 0 keys loaded after warmup");
2454    }
2455
2456    checkeq(0, get_int_stat(h, h1, "ep_warmup_value_count", "warmup"),
2457            "Expected 0 values loaded after warmup");
2458
2459    return SUCCESS;
2460}
2461
2462// Test that all the configuration parameters associated with the ItemPager,
2463// can be set.
2464static enum test_result test_itempager_conf(ENGINE_HANDLE* h,
2465                                            ENGINE_HANDLE_V1* h1) {
2466    check(set_param(h,
2467                    h1,
2468                    protocol_binary_engine_param_flush,
2469                    "pager_active_vb_pcnt",
2470                    "50"),
2471          "Setting pager_active_vb_pcnt should have worked");
2472    checkeq(50,
2473            get_int_stat(h, h1, "ep_pager_active_vb_pcnt"),
2474            "pager_active_vb_pcnt did not get set to the correct value");
2475
2476    check(set_param(h,
2477                    h1,
2478                    protocol_binary_engine_param_flush,
2479                    "pager_sleep_time_ms",
2480                    "1000"),
2481          "Setting pager_sleep_time_ms should have worked");
2482    checkeq(1000,
2483            get_int_stat(h, h1, "ep_pager_sleep_time_ms"),
2484            "pager_sleep_time_ms did not get set to the correct value");
2485
2486    check(set_param(h,
2487                    h1,
2488                    protocol_binary_engine_param_flush,
2489                    "ht_eviction_policy",
2490                    "2-bit_lru"),
2491          "Setting ht_eviction_policy should have worked");
2492    checkeq(std::string("2-bit_lru"),
2493            get_str_stat(h, h1, "ep_ht_eviction_policy"),
2494            "ht_eviction_policy did not get set to the correct value");
2495
2496    check(set_param(h,
2497                    h1,
2498                    protocol_binary_engine_param_flush,
2499                    "item_eviction_age_percentage",
2500                    "100"),
2501          "Set item_eviction_age_percentage should have worked");
2502    checkeq(100,
2503            get_int_stat(h, h1, "ep_item_eviction_age_percentage"),
2504            "item_eviction_age_percentage did not get set to the correct "
2505            "value");
2506
2507    check(set_param(h,
2508                    h1,
2509                    protocol_binary_engine_param_flush,
2510                    "item_eviction_freq_counter_age_threshold",
2511                    "10"),
2512          "Set item_eviction_freq_counter_age_threshold should have worked");
2513    checkeq(10,
2514            get_int_stat(h, h1, "ep_item_eviction_freq_counter_age_threshold"),
2515            "item_eviction_freq_counter_age_threshold did not get set to the "
2516            "correct value");
2517
2518    check(set_param(h,
2519                    h1,
2520                    protocol_binary_engine_param_flush,
2521                    "item_freq_decayer_chunk_duration",
2522                    "1000"),
2523          "Set item_freq_decayer_chunk_duration should have worked");
2524    checkeq(1000,
2525            get_int_stat(h, h1, "ep_item_freq_decayer_chunk_duration"),
2526            "item_freq_decayer_chunk_duration did not get set to the correct "
2527            "value");
2528
2529    check(set_param(h,
2530                    h1,
2531                    protocol_binary_engine_param_flush,
2532                    "item_freq_decayer_percent",
2533                    "100"),
2534          "Set item_freq_decayer_percent should have worked");
2535    checkeq(100,
2536            get_int_stat(h, h1, "ep_item_freq_decayer_percent"),
2537            "item_freq_decayer_percent did not get set to the correct value");
2538
2539    return SUCCESS;
2540}
2541
2542static enum test_result test_bloomfilter_conf(ENGINE_HANDLE *h,
2543                                              ENGINE_HANDLE_V1 *h1) {
2544
2545    if (get_bool_stat(h, h1, "ep_bfilter_enabled") == false) {
2546        check(set_param(h, h1, protocol_binary_engine_param_flush,
2547                        "bfilter_enabled", "true"),
2548              "Set bloomfilter_enabled should have worked");
2549    }
2550    check(get_bool_stat(h, h1, "ep_bfilter_enabled"),
2551          "Bloom filter wasn't enabled");
2552
2553    check(get_float_stat(h, h1, "ep_bfilter_residency_threshold") == (float)0.1,
2554          "Incorrect initial bfilter_residency_threshold.");
2555
2556    check(set_param(h, h1, protocol_binary_engine_param_flush,
2557          "bfilter_enabled", "false"),
2558          "Set bloomfilter_enabled should have worked.");
2559    check(set_param(h, h1, protocol_binary_engine_param_flush,
2560          "bfilter_residency_threshold", "0.15"),
2561          "Set bfilter_residency_threshold should have worked.");
2562
2563    check(get_bool_stat(h, h1, "ep_bfilter_enabled") == false,
2564          "Bloom filter should have been disabled.");
2565    check(get_float_stat(h, h1, "ep_bfilter_residency_threshold") == (float)0.15,
2566          "Incorrect bfilter_residency_threshold.");
2567
2568    return SUCCESS;
2569}
2570
2571static enum test_result test_bloomfilters(ENGINE_HANDLE *h,
2572                                          ENGINE_HANDLE_V1 *h1) {
2573
2574    if (get_bool_stat(h, h1, "ep_bfilter_enabled") == false) {
2575        check(set_param(h, h1, protocol_binary_engine_param_flush,
2576                    "bfilter_enabled", "true"),
2577                "Set bloomfilter_enabled should have worked");
2578    }
2579    check(get_bool_stat(h, h1, "ep_bfilter_enabled"),
2580            "Bloom filter wasn't enabled");
2581
2582    // Key is only present if bgOperations is non-zero.
2583    int num_read_attempts = get_int_stat_or_default(h, h1, 0,
2584                                                    "ep_bg_num_samples");
2585
2586    // Ensure vbucket's bloom filter is enabled
2587    checkeq(std::string("ENABLED"),
2588            get_str_stat(h, h1, "vb_0:bloom_filter", "vbucket-details 0"),
2589            "Vbucket 0's bloom filter wasn't enabled upon setup!");
2590
2591    int i;
2592
2593    // Insert 10 items.
2594    for (i = 0; i < 10; ++i) {
2595        std::stringstream key;
2596        key << "key-" << i;
2597        checkeq(ENGINE_SUCCESS,
2598                store(h,
2599                      h1,
2600                      NULL,
2601                      OPERATION_SET,
2602                      key.str().c_str(),
2603                      "somevalue"),
2604                "Error setting.");
2605    }
2606    wait_for_flusher_to_settle(h, h1);
2607
2608    // Evict all 10 items.
2609    for (i = 0; i < 10; ++i) {
2610        std::stringstream key;
2611        key << "key-" << i;
2612        evict_key(h, h1, key.str().c_str(), 0, "Ejected.");
2613    }
2614    wait_for_flusher_to_settle(h, h1);
2615
2616    // Ensure 10 items are non-resident.
2617    cb_assert(10 == get_int_stat(h, h1, "ep_num_non_resident"));
2618
2619    // Issue delete on first 5 items.
2620    for (i = 0; i < 5; ++i) {
2621        std::stringstream key;
2622        key << "key-" << i;
2623        checkeq(ENGINE_SUCCESS,
2624                del(h, h1, key.str().c_str(), 0, 0),
2625                "Failed remove with value.");
2626    }
2627    wait_for_flusher_to_settle(h, h1);
2628
2629    // Ensure that there are 5 non-resident items
2630    cb_assert(5 == get_int_stat(h, h1, "ep_num_non_resident"));
2631    cb_assert(5 == get_int_stat(h, h1, "curr_items"));
2632
2633    checkeq(ENGINE_SUCCESS,
2634            get_stats(h, {}, add_stats),
2635            "Failed to get stats.");
2636    std::string eviction_policy = vals.find("ep_item_eviction_policy")->second;
2637
2638    useconds_t sleepTime = 128;
2639
2640    if (eviction_policy == "value_only") {  // VALUE-ONLY EVICTION MODE
2641
2642        checkeq(5,
2643                get_int_stat(h, h1, "vb_0:bloom_filter_key_count",
2644                             "vbucket-details 0"),
2645                "Unexpected no. of keys in bloom filter");
2646
2647        checkeq(num_read_attempts,
2648                get_int_stat_or_default(h, h1, 0, "ep_bg_num_samples"),
2649                "Expected bgFetch attempts to remain unchanged");
2650
2651        for (i = 0; i < 5; ++i) {
2652            std::stringstream key;
2653            key << "key-" << i;
2654            check(get_meta(h, h1, key.str().c_str()), "Get meta failed");
2655        }
2656
2657        // GetMeta would cause bgFetches as bloomfilter contains
2658        // the deleted items.
2659        checkeq(num_read_attempts + 5,
2660                get_int_stat(h, h1, "ep_bg_num_samples"),
2661                "Expected bgFetch attempts to increase by five");
2662
2663        // Run compaction, with drop_deletes
2664        compact_db(h, h1, 0, 0, 15, 15, 1);
2665        while (get_int_stat(h, h1, "ep_pending_compactions") != 0) {
2666            decayingSleep(&sleepTime);
2667        }
2668
2669        for (i = 0; i < 5; ++i) {
2670            std::stringstream key;
2671            key << "key-" << i;
2672            check(get_meta(h, h1, key.str().c_str()), "Get meta failed");
2673        }
2674        checkeq(num_read_attempts + 5,
2675                get_int_stat(h, h1, "ep_bg_num_samples"),
2676                "Expected bgFetch attempts to stay as before");
2677
2678    } else {                                // FULL EVICTION MODE
2679
2680        checkeq(10,
2681                get_int_stat(h, h1, "vb_0:bloom_filter_key_count",
2682                             "vbucket-details 0"),
2683                "Unexpected no. of keys in bloom filter");
2684
2685
2686        // Because of issuing deletes on non-resident items
2687        checkeq(num_read_attempts + 5,
2688                get_int_stat(h, h1, "ep_bg_num_samples"),
2689                "Expected bgFetch attempts to increase by five, after deletes");
2690
2691        // Run compaction, with drop_deletes, to exclude deleted items
2692        // from bloomfilter.
2693        compact_db(h, h1, 0, 0, 15, 15, 1);
2694        while (get_int_stat(h, h1, "ep_pending_compactions") != 0) {
2695            decayingSleep(&sleepTime);
2696        }
2697
2698        for (i = 0; i < 5; i++) {
2699            std::stringstream key;
2700            key << "key-" << i;
2701            checkeq(cb::engine_errc::no_such_key,
2702                    get(h, h1, NULL, key.str(), 0).first,
2703                    "Unable to get stored item");
2704        }
2705        // + 6 because last delete is not purged by the compactor
2706        checkeq(num_read_attempts + 6,
2707                get_int_stat(h, h1, "ep_bg_num_samples"),
2708                "Expected bgFetch attempts to stay as before");
2709    }
2710
2711    return SUCCESS;
2712}
2713
2714static enum test_result test_bloomfilters_with_store_apis(ENGINE_HANDLE *h,
2715                                                          ENGINE_HANDLE_V1 *h1) {
2716    if (get_bool_stat(h, h1, "ep_bfilter_enabled") == false) {
2717        check(set_param(h, h1, protocol_binary_engine_param_flush,
2718                    "bfilter_enabled", "true"),
2719                "Set bloomfilter_enabled should have worked");
2720    }
2721    check(get_bool_stat(h, h1, "ep_bfilter_enabled"),
2722            "Bloom filter wasn't enabled");
2723
2724    int num_read_attempts = get_int_stat_or_default(h, h1, 0,
2725                                                    "ep_bg_num_samples");
2726
2727    // Ensure vbucket's bloom filter is enabled
2728    checkeq(std::string("ENABLED"),
2729            get_str_stat(h, h1, "vb_0:bloom_filter", "vbucket-details 0"),
2730            "Vbucket 0's bloom filter wasn't enabled upon setup!");
2731
2732    for (int i = 0; i < 1000; i++) {
2733        std::stringstream key;
2734        key << "key-" << i;
2735        check(!get_meta(h, h1, key.str().c_str()),
2736                "Get meta should fail.");
2737    }
2738
2739    checkeq(num_read_attempts,
2740            get_int_stat_or_default(h, h1, 0, "ep_bg_num_samples"),
2741            "Expected no bgFetch attempts");
2742
2743    checkeq(ENGINE_SUCCESS,
2744            get_stats(h, {}, add_stats),
2745            "Failed to get stats.");
2746    std::string eviction_policy = vals.find("ep_item_eviction_policy")->second;
2747
2748    if (eviction_policy == "full_eviction") {  // FULL EVICTION MODE
2749        // Set with Meta
2750        int j;
2751        for (j = 0; j < 10; j++) {
2752            uint64_t cas_for_set = last_cas;
2753            // init some random metadata
2754            ItemMetaData itm_meta;
2755            itm_meta.revSeqno = 10;
2756            itm_meta.cas = 0xdeadbeef;
2757            itm_meta.exptime = time(NULL) + 300;
2758            itm_meta.flags = 0xdeadbeef;
2759
2760            std::stringstream key;
2761            key << "swm-" << j;
2762            set_with_meta(h, h1, key.str().c_str(), key.str().length(),
2763                          "somevalue", 9, 0, &itm_meta, cas_for_set);
2764        }
2765
2766        checkeq(num_read_attempts,
2767                get_int_stat_or_default(h, h1, 0, "ep_bg_num_samples"),
2768                "Expected no bgFetch attempts");
2769
2770        // Add
2771        for (j = 0; j < 10; j++) {
2772            std::stringstream key;
2773            key << "add-" << j;
2774
2775            checkeq(ENGINE_SUCCESS,
2776                    store(h,
2777                          h1,
2778                          NULL,
2779                          OPERATION_ADD,
2780                          key.str().c_str(),
2781                          "newvalue"),
2782                    "Failed to add value again.");
2783        }
2784
2785        checkeq(num_read_attempts,
2786                get_int_stat_or_default(h, h1, 0, "ep_bg_num_samples"),
2787                "Expected no bgFetch attempts");
2788
2789        // Delete
2790        for (j = 0; j < 10; j++) {
2791            std::stringstream key;
2792            key << "del-" << j;
2793            checkeq(ENGINE_KEY_ENOENT,
2794                    del(h, h1, key.str().c_str(), 0, 0),
2795                    "Failed remove with value.");
2796        }
2797
2798        checkeq(num_read_attempts,
2799                get_int_stat_or_default(h, h1, 0, "ep_bg_num_samples"),
2800                "Expected no bgFetch attempts");
2801
2802    }
2803
2804    return SUCCESS;
2805}
2806
2807static enum test_result test_bloomfilter_delete_plus_set_scenario(
2808                                       ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2809    if (get_bool_stat(h, h1, "ep_bfilter_enabled") == false) {
2810        check(set_param(h, h1, protocol_binary_engine_param_flush,
2811                    "bfilter_enabled", "true"),
2812                "Set bloomfilter_enabled should have worked");
2813    }
2814    check(get_bool_stat(h, h1, "ep_bfilter_enabled"),
2815            "Bloom filter wasn't enabled");
2816
2817    // Ensure vbucket's bloom filter is enabled
2818    checkeq(std::string("ENABLED"),
2819            get_str_stat(h, h1, "vb_0:bloom_filter", "vbucket-details 0"),
2820            "Vbucket 0's bloom filter wasn't enabled upon setup!");
2821
2822    checkeq(ENGINE_SUCCESS,
2823            store(h, h1, NULL, OPERATION_SET, "k1", "v1"),
2824            "Failed to fail to store an item.");
2825
2826    wait_for_flusher_to_settle(h, h1);
2827    int num_writes = get_int_stat(h, h1, "rw_0:io_num_write", "kvstore");
2828    int num_persisted = get_int_stat(h, h1, "ep_total_persisted");
2829    cb_assert(num_writes == 1 && num_persisted == 1);
2830
2831    checkeq(ENGINE_SUCCESS,
2832            del(h, h1, "k1", 0, 0), "Failed remove with value.");
2833    stop_persistence(h, h1);
2834    checkeq(ENGINE_SUCCESS,
2835            store(h, h1, NULL, OPERATION_SET, "k1", "v2", nullptr, 0, 0),
2836            "Failed to fail to store an item.");
2837    int key_count = get_int_stat(h, h1, "vb_0:bloom_filter_key_count",
2838                                 "vbucket-details 0");
2839
2840    if (key_count == 0) {
2841        check(get_int_stat(h, h1, "rw_0:io_num_write", "kvstore") <= 2,
2842                "Unexpected number of writes");
2843        start_persistence(h, h1);
2844        wait_for_flusher_to_settle(h, h1);
2845        checkeq(0, get_int_stat(h, h1, "vb_0:bloom_filter_key_count",
2846                                "vbucket-details 0"),
2847                "Unexpected number of keys in bloomfilter");
2848    } else {
2849        cb_assert(key_count == 1);
2850        checkeq(2, get_int_stat(h, h1, "rw_0:io_num_write", "kvstore"),
2851                "Unexpected number of writes");
2852        start_persistence(h, h1);
2853        wait_for_flusher_to_settle(h, h1);
2854        checkeq(1, get_int_stat(h, h1, "vb_0:bloom_filter_key_count",
2855                                "vbucket-details 0"),
2856                "Unexpected number of keys in bloomfilter");
2857    }
2858
2859    return SUCCESS;
2860}
2861
2862static enum test_result test_datatype(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2863    const void *cookie = testHarness.create_cookie();
2864    testHarness.set_datatype_support(cookie, true);
2865
2866    item *itm = NULL;
2867    const std::string key("{\"foo\":\"bar\"}");
2868    const protocol_binary_datatype_t datatype = PROTOCOL_BINARY_DATATYPE_JSON;
2869    uint64_t cas = 0;
2870    std::string value("x");
2871    checkeq(ENGINE_SUCCESS,
2872            storeCasOut(h, h1, NULL, 0, key, value, datatype, itm, cas),
2873            "Expected set to succeed");
2874
2875    auto ret = get(h, h1, cookie, key, 0);
2876    checkeq(cb::engine_errc::success, ret.first, "Unable to get stored item");
2877
2878    item_info info;
2879    h1->get_item_info(h, ret.second.get(), &info);
2880    checkeq(static_cast<uint8_t>(PROTOCOL_BINARY_DATATYPE_JSON),
2881            info.datatype, "Invalid datatype");
2882
2883    const char* key1 = "foo";
2884    const char* val1 = "{\"foo1\":\"bar1\"}";
2885    ItemMetaData itm_meta;
2886    itm_meta.revSeqno = 10;
2887    itm_meta.cas = info.cas;
2888    itm_meta.exptime = info.exptime;
2889    itm_meta.flags = info.flags;
2890    set_with_meta(h, h1, key1, strlen(key1), val1, strlen(val1), 0, &itm_meta,
2891                  last_cas, 0, info.datatype, cookie);
2892
2893    ret = get(h, h1, cookie, key1, 0);
2894    checkeq(cb::engine_errc::success, ret.first, "Unable to get stored item");
2895
2896    h1->get_item_info(h, ret.second.get(), &info);
2897    checkeq(static_cast<uint8_t>(PROTOCOL_BINARY_DATATYPE_JSON),
2898            info.datatype, "Invalid datatype, when setWithMeta");
2899
2900    testHarness.destroy_cookie(cookie);
2901    return SUCCESS;
2902}
2903
2904static enum test_result test_datatype_with_unknown_command(ENGINE_HANDLE *h,
2905                                                           ENGINE_HANDLE_V1 *h1) {
2906    const void *cookie = testHarness.create_cookie();
2907    testHarness.set_datatype_support(cookie, true);
2908    const char* key = "foo";
2909    const char* val = "{\"foo\":\"bar\"}";
2910    uint8_t datatype = PROTOCOL_BINARY_DATATYPE_JSON;
2911
2912    ItemMetaData itm_meta;
2913    itm_meta.revSeqno = 10;
2914    itm_meta.cas = 0x1;
2915    itm_meta.exptime = 0;
2916    itm_meta.flags = 0;
2917
2918    //SET_WITH_META
2919    set_with_meta(h, h1, key, strlen(key), val, strlen(val), 0, &itm_meta,
2920                  0, 0, datatype, cookie);
2921
2922    auto ret = get(h, h1, cookie, key, 0);
2923    checkeq(cb::engine_errc::success, ret.first, "Unable to get stored item");
2924
2925    item_info info;
2926    h1->get_item_info(h, ret.second.get(), &info);
2927    checkeq(static_cast<uint8_t>(PROTOCOL_BINARY_DATATYPE_JSON),
2928            info.datatype, "Invalid datatype, when setWithMeta");
2929
2930    //SET_RETURN_META
2931    checkeq(ENGINE_SUCCESS,
2932            set_ret_meta(h,
2933                         h1,
2934                         "foo1",
2935                         4,
2936                         val,
2937                         strlen(val),
2938                         0,
2939                         0,
2940                         0,
2941                         0,
2942                         datatype,
2943                         cookie),
2944            "Expected success");
2945    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
2946            "Expected set returing meta to succeed");
2947    checkeq(static_cast<uint8_t>(PROTOCOL_BINARY_DATATYPE_JSON),
2948            last_datatype.load(), "Invalid datatype, when set_return_meta");
2949
2950    testHarness.destroy_cookie(cookie);
2951    return SUCCESS;
2952}
2953
2954static enum test_result test_session_cas_validation(ENGINE_HANDLE *h,
2955                                                    ENGINE_HANDLE_V1 *h1) {
2956    //Testing PROTOCOL_BINARY_CMD_SET_VBUCKET..
2957    char ext[4];
2958    protocol_binary_request_header *pkt;
2959    vbucket_state_t state = vbucket_state_active;
2960    uint32_t val = static_cast<uint32_t>(state);
2961    val = htonl(val);
2962    memcpy(ext, (char*)&val, sizeof(val));
2963
2964    uint64_t cas = 0x0101010101010101;
2965    pkt = createPacket(PROTOCOL_BINARY_CMD_SET_VBUCKET, 0, cas, ext, 4);
2966    checkeq(ENGINE_SUCCESS,
2967            h1->unknown_command(h, NULL, pkt, add_response, testHarness.doc_namespace),
2968            "SET_VBUCKET command failed");
2969    cb_free(pkt);
2970    cb_assert(last_status == PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS);
2971
2972    cas = 0x0102030405060708;
2973    pkt = createPacket(PROTOCOL_BINARY_CMD_SET_VBUCKET, 0, cas, ext, 4);
2974    checkeq(ENGINE_SUCCESS,
2975            h1->unknown_command(h, NULL, pkt, add_response, testHarness.doc_namespace),
2976            "SET_VBUCKET command failed");
2977    cb_free(pkt);
2978    cb_assert(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS);
2979
2980    return SUCCESS;
2981}
2982
2983static enum test_result test_access_scanner_settings(ENGINE_HANDLE *h,
2984                                                     ENGINE_HANDLE_V1 *h1) {
2985    if (!isWarmupEnabled(h, h1)) {
2986        // Access scanner n/a without warmup.
2987        return SKIPPED;
2988    }
2989
2990    // Create a unique access log path by combining with the db path.
2991    checkeq(ENGINE_SUCCESS,
2992            get_stats(h, {}, add_stats),
2993            "Failed to get stats.");
2994    std::string dbname = vals.find("ep_dbname")->second;
2995
2996    const auto alog_path = std::string("alog_path=") + dbname +
2997            DIRECTORY_SEPARATOR_CHARACTER + "access.log";
2998    std::string newconfig = std::string(testHarness.get_current_testcase()->cfg)
2999                            + alog_path;
3000
3001    testHarness.reload_engine(&h, &h1,
3002                              testHarness.engine_path,
3003                              newconfig.c_str(),
3004                              true, false);
3005    wait_for_warmup_complete(h, h1);
3006
3007    std::string err_msg;
3008    // Check access scanner is enabled and alog_task_time is at default
3009    checkeq(true, get_bool_stat(h, h1, "ep_access_scanner_enabled"),
3010            "Expected access scanner to be enabled");
3011    cb_assert(get_int_stat(h, h1, "ep_alog_task_time") == 2);
3012
3013    // Ensure access_scanner_task_time is what its expected to be.
3014    // Need to wait until the AccessScanner task has been setup.
3015    wait_for_stat_change(h, h1, "ep_access_scanner_task_time",
3016                         std::string{"NOT_SCHEDULED"});
3017
3018    std::string str = get_str_stat(h, h1, "ep_access_scanner_task_time");
3019    std::string expected_time = "02:00";
3020    err_msg.assign("Initial time incorrect, expect: " +
3021                   expected_time + ", actual: " + str.substr(11, 5));
3022    checkeq(0, str.substr(11, 5).compare(expected_time), err_msg.c_str());
3023
3024    // Update alog_task_time and ensure the update is successful
3025    expected_time = "05:00";
3026
3027    // [MB-24422] we need to set this multiple times as the change listeners
3028    //  may not have been initialized at the time of call
3029    repeat_till_true([&]() {
3030        set_param(h,
3031                  h1,
3032                  protocol_binary_engine_param_flush,
3033                  "alog_task_time",
3034                  "5");
3035        str = get_str_stat(h, h1, "ep_access_scanner_task_time");
3036        return (0 == str.substr(11, 5).compare(expected_time));
3037    });
3038
3039    err_msg.assign("Updated time incorrect, expect: " +
3040                   expected_time + ", actual: " + str.substr(11, 5));
3041    checkeq(0, str.substr(11, 5).compare(expected_time), err_msg.c_str());
3042
3043    // Update alog_sleep_time by 10 mins and ensure the update is successful.
3044    const std::chrono::minutes update_by{10};
3045    std::string targetTaskTime1{make_time_string(std::chrono::system_clock::now() +
3046                                                 update_by)};
3047
3048    set_param(h, h1, protocol_binary_engine_param_flush, "alog_sleep_time",
3049              std::to_string(update_by.count()).c_str());
3050    str = get_str_stat(h, h1, "ep_access_scanner_task_time");
3051
3052    // Recalculate now() + 10mins as upper bound on when the task should be
3053    // scheduled.
3054    std::string targetTaskTime2{make_time_string(std::chrono::system_clock::now() +
3055                                                 update_by)};
3056
3057    // ep_access_scanner_task_time should fall within the range of
3058    // targetTaskTime1 and targetTaskTime2
3059    err_msg.assign("Unexpected task time range, expect: " +
3060                   targetTaskTime1 + " <= " + str + " <= " + targetTaskTime2);
3061    check(targetTaskTime1 <= str, err_msg.c_str());
3062    check(str <= targetTaskTime2, err_msg.c_str());
3063
3064    return SUCCESS;
3065}
3066
3067static enum test_result test_access_scanner(ENGINE_HANDLE *h,
3068                                            ENGINE_HANDLE_V1 *h1) {
3069    if (!isWarmupEnabled(h, h1)) {
3070        // Access scanner not applicable without warmup.
3071        return SKIPPED;
3072    }
3073
3074    // Create a unique access log path by combining with the db path.
3075    checkeq(ENGINE_SUCCESS,
3076            get_stats(h, {}, add_stats),
3077            "Failed to get stats.");
3078    const auto dbname = vals.find("ep_dbname")->second;
3079
3080    const auto alog_path = std::string("alog_path=") + dbname +
3081            DIRECTORY_SEPARATOR_CHARACTER + "access.log";
3082
3083    /* We do not want the access scanner task to be running while we initiate it
3084       explicitly below. Hence set the alog_task_time to about 1 ~ 2 hours
3085       from now */
3086    const time_t now = time(nullptr);
3087    struct tm tm_now;
3088    cb_gmtime_r(&now, &tm_now);
3089    const auto two_hours_hence = (tm_now.tm_hour + 2) % 24;
3090
3091    const auto alog_task_time = std::string("alog_task_time=") +
3092            std::to_string(two_hours_hence);
3093
3094    const auto newconfig = std::string(testHarness.get_current_testcase()->cfg)
3095                           + alog_path + ";" + alog_task_time;
3096
3097    testHarness.reload_engine(&h, &h1,
3098                              testHarness.engine_path,
3099                              newconfig.c_str(),
3100                              true, false);
3101    wait_for_warmup_complete(h, h1);
3102
3103    /* Check that alog_task_time was correctly updated. */
3104    checkeq(get_int_stat(h, h1, "ep_alog_task_time"),
3105            two_hours_hence,
3106            "Failed to set alog_task_time to 2 hours in the future");
3107
3108    checkeq(ENGINE_SUCCESS,
3109            get_stats(h, {}, add_stats),
3110            "Failed to get stats.");
3111    std::string name = vals.find("ep_alog_path")->second;
3112
3113    /* Check access scanner is enabled */
3114    checkeq(true, get_bool_stat(h, h1, "ep_access_scanner_enabled"),
3115            "Access scanner task not enabled by default. Check test config");
3116
3117    const int num_shards = get_int_stat(h, h1, "ep_workload:num_shards",
3118                                        "workload");
3119    name = name + ".0";
3120    std::string prev(name + ".old");
3121
3122    /* Get the resident ratio down to below 95% - point at which access.log
3123     * generation occurs.
3124     */
3125    int num_items = 0;
3126    // Size chosen to create ~2000 items (i.e. 2x more than we sanity-check below)
3127    // with the given max_size for this test.
3128    const std::string value(2000, 'x');
3129    while (true) {
3130        // Gathering stats on every store is expensive, just check every 100 iterations
3131        if ((num_items % 100) == 0) {
3132            if (get_int_stat(h, h1, "vb_active_perc_mem_resident") < 94) {
3133                break;
3134            }
3135        }
3136
3137        std::string key("key" + std::to_string(num_items));
3138        ENGINE_ERROR_CODE ret =
3139                store(h, h1, NULL, OPERATION_SET, key.c_str(), value.c_str());
3140        switch (ret) {
3141            case ENGINE_SUCCESS:
3142                num_items++;
3143                break;
3144
3145            case ENGINE_ENOMEM:
3146            case ENGINE_TMPFAIL:
3147                // Returned when at high watermark; simply retry the op.
3148                break;
3149
3150            default:
3151                fprintf(stderr, "test_access_scanner: Unexpected result from store(): %d\n",
3152                        ret);
3153                abort();
3154        }
3155
3156    }
3157
3158    // Sanity check - ensure we have enough vBucket quota (max_size)
3159    // such that we have 1000 items - enough to give us 0.1%
3160    // granuarity in any residency calculations. */
3161    if (num_items < 1000) {
3162        std::cerr << "Error: test_access_scanner: "
3163            "expected at least 1000 items after filling vbucket, "
3164            "but only have " << num_items << ". "
3165            "Check max_size setting for test." << std::endl;
3166        return FAIL;
3167    }
3168
3169    wait_for_flusher_to_settle(h, h1);
3170    verify_curr_items(h, h1, num_items, "Wrong number of items");
3171    int num_non_resident = get_int_stat(h, h1, "vb_active_num_non_resident");
3172    checkge(num_non_resident, num_items * 6 / 100,
3173            "Expected num_non_resident to be at least 6% of total items");
3174
3175    /* Run access scanner task once and expect it to generate access log */
3176    check(set_param(h, h1, protocol_binary_engine_param_flush,
3177                    "access_scanner_run", "true"),
3178          "Failed to trigger access scanner");
3179
3180    // Wait for the number of runs to equal the number of shards.
3181    wait_for_stat_to_be(h, h1, "ep_num_access_scanner_runs", num_shards);
3182
3183    /* This time since resident ratio is < 95% access log should be generated */
3184    checkeq(0, access(name.c_str(), F_OK),
3185            (std::string("access log file (") + name +
3186             ") should exist (got errno:" + std::to_string(errno)).c_str());
3187
3188    /* Increase resident ratio by deleting items */
3189    checkeq(ENGINE_SUCCESS, vbucketDelete(h, h1, 0), "Expected success");
3190    check(set_vbucket_state(h, h1, 0, vbucket_state_active),
3191          "Failed to set VB0 state.");
3192
3193    /* Run access scanner task once */
3194    const int access_scanner_skips =
3195            get_int_stat(h, h1, "ep_num_access_scanner_skips");
3196    check(set_param(h, h1, protocol_binary_engine_param_flush,
3197                    "access_scanner_run", "true"),
3198          "Failed to trigger access scanner");
3199    wait_for_stat_to_be(h, h1, "ep_num_access_scanner_skips",
3200                        access_scanner_skips + num_shards);
3201
3202    /* Access log files should be removed because resident ratio > 95% */
3203    checkeq(-1, access(prev.c_str(), F_OK),
3204            ".old access log file should not exist");
3205    checkeq(-1, access(name.c_str(), F_OK), "access log file should not exist");
3206
3207    return SUCCESS;
3208}
3209
3210static enum test_result test_set_param_message(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
3211    set_param(h, h1, protocol_binary_engine_param_flush, "alog_task_time", "50");
3212
3213    checkeq(PROTOCOL_BINARY_RESPONSE_EINVAL, last_status.load(),
3214        "Expected an invalid value error for an out of bounds alog_task_time");
3215    check(std::string("Validation Error").compare(last_body), "Expected a "
3216            "validation error in the response body");
3217    return SUCCESS;
3218}
3219
3220static enum test_result test_warmup_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
3221    if (!isWarmupEnabled(h, h1)) {
3222        return SKIPPED;
3223    }
3224
3225    check(set_vbucket_state(h, h1, 0, vbucket_state_active), "Failed to set VB0 state.");
3226    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set VB1 state.");
3227
3228    for (int i = 0; i < 5000; ++i) {
3229        std::stringstream key;
3230        key << "key-" << i;
3231        checkeq(ENGINE_SUCCESS,
3232                store(h,
3233                      h1,
3234                      NULL,
3235                      OPERATION_SET,
3236                      key.str().c_str(),
3237                      "somevalue"),
3238                "Error setting.");
3239    }
3240
3241    // Restart the server.
3242    testHarness.reload_engine(&h, &h1,
3243                              testHarness.engine_path,
3244                              testHarness.get_current_testcase()->cfg,
3245                              true, false);
3246
3247    wait_for_warmup_complete(h, h1);
3248
3249    const auto warmup_stats = get_all_stats(h, h1, "warmup");
3250
3251    // Check all expected warmup stats exists.
3252    const char* warmup_keys[] = { "ep_warmup_thread",
3253                                  "ep_warmup_value_count",
3254                                  "ep_warmup_key_count",
3255                                  "ep_warmup_dups",
3256                                  "ep_warmup_oom",
3257                                  "ep_warmup_time"};
3258    for (const auto* key : warmup_keys) {
3259        check(warmup_stats.find(key) != warmup_stats.end(),
3260              (std::string("Found no ") + key).c_str());
3261    }
3262
3263    std::string warmup_time = warmup_stats.at("ep_warmup_time");
3264    cb_assert(std::stoi(warmup_time) > 0);
3265
3266    const auto prev_vb_stats = get_all_stats(h, h1, "prev-vbucket");
3267
3268    check(prev_vb_stats.find("vb_0") != prev_vb_stats.end(),
3269          "Found no previous state for VB0");
3270    check(prev_vb_stats.find("vb_1") != prev_vb_stats.end(),
3271          "Found no previous state for VB1");
3272
3273    checkeq(std::string("active"), prev_vb_stats.at("vb_0"),
3274            "Unexpected stats for vb 0");
3275    checkeq(std::string("replica"), prev_vb_stats.at("vb_1"),
3276            "Unexpected stats for vb 1");
3277
3278    const auto vb_details_stats = get_all_stats(h, h1, "vbucket-details");
3279    checkeq(5000, std::stoi(vb_details_stats.at("vb_0:num_items")),
3280            "Unexpected item count for vb 0");
3281    checkeq(0, std::stoi(vb_details_stats.at("vb_1:num_items")),
3282            "Unexpected item count for vb 1");
3283
3284    return SUCCESS;
3285}
3286
3287static enum test_result test_warmup_with_threshold(ENGINE_HANDLE *h,
3288                                                   ENGINE_HANDLE_V1 *h1) {
3289    if (!isWarmupEnabled(h, h1)) {
3290        return SKIPPED;
3291    }
3292
3293    check(set_vbucket_state(h, h1, 0, vbucket_state_active), "Failed set vbucket 1 state.");
3294    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed set vbucket 2 state.");
3295    check(set_vbucket_state(h, h1, 2, vbucket_state_active), "Failed set vbucket 3 state.");
3296    check(set_vbucket_state(h, h1, 3, vbucket_state_active), "Failed set vbucket 4 state.");
3297
3298    for (int i = 0; i < 10000; ++i) {
3299        std::stringstream key;
3300        key << "key+" << i;
3301        checkeq(ENGINE_SUCCESS,
3302                store(h,
3303                      h1,
3304                      NULL,
3305                      OPERATION_SET,
3306                      key.str().c_str(),
3307                      "somevalue",
3308                      nullptr,
3309                      0,
3310                      (i % 4)),
3311                "Error setting.");
3312    }
3313
3314    // Restart the server.
3315    testHarness.reload_engine(&h, &h1,
3316                              testHarness.engine_path,
3317                              testHarness.get_current_testcase()->cfg,
3318                              true, false);
3319
3320    wait_for_warmup_complete(h, h1);
3321
3322    checkeq(1,
3323            get_int_stat(h, h1, "ep_warmup_min_item_threshold", "warmup"),
3324            "Unable to set warmup_min_item_threshold to 1%");
3325
3326    const std::string policy = get_str_stat(h, h1, "ep_item_eviction_policy");
3327
3328    if (policy == "full_eviction") {
3329        checkeq(get_int_stat(h, h1, "ep_warmup_key_count", "warmup"),
3330                get_int_stat(h, h1, "ep_warmup_value_count", "warmup"),
3331                "Warmed up key count didn't match warmed up value count");
3332    } else {
3333        checkeq(10000, get_int_stat(h, h1, "ep_warmup_key_count", "warmup"),
3334                "Warmup didn't warmup all keys");
3335    }
3336    check(get_int_stat(h, h1, "ep_warmup_value_count", "warmup") <= 110,
3337            "Warmed up value count found to be greater than 1%");
3338
3339    cb_assert(get_int_stat(h, h1, "ep_warmup_time", "warmup") > 0);
3340
3341    return SUCCESS;
3342}
3343
3344#if 0
3345// Comment out the entire test since the hack gave warnings on win32
3346static enum test_result test_warmup_accesslog(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
3347#ifdef __APPLE__
3348    /* I'm getting a weird link error from clang.. disable the test until I
3349    ** understand why
3350    */
3351    return SKIPPED;
3352#else
3353    item *it = NULL;
3354
3355    int n_items_to_store1 = 10;
3356    for (int i = 0; i < n_items_to_store1; ++i) {
3357        std::stringstream key;
3358        key << "key-" << i;
3359        const char* keystr = key.str().c_str();
3360        checkeq(ENGINE_SUCCESS,
3361                store(h, h1, NULL, OPERATION_SET, keystr, "somevalue", &it, 0, 0),
3362                "Error setting.");
3363        h1->release(h, NULL, it);
3364    }
3365
3366    wait_for_flusher_to_settle(h, h1);
3367
3368    int n_items_to_access = 10;
3369    for (int i = 0; i < n_items_to_access; ++i) {
3370        std::stringstream key;
3371        key << "key-" << i;
3372        const char* keystr = key.str().c_str();
3373        checkeq(ENGINE_SUCCESS,
3374                get(h, h1, NULL, &it, keystr, 0),
3375                "Error getting.");
3376        h1->release(h, NULL, it);
3377    }
3378
3379    // sleep so that scanner task can have timew to generate access log
3380    sleep(61);
3381
3382    // store additional items
3383    int n_items_to_store2 = 10;
3384    for (int i = 0; i < n_items_to_store2; ++i) {
3385        std::stringstream key;
3386        key << "key2-" << i;
3387        const char* keystr = key.str().c_str();
3388        checkeq(ENGINE_SUCCESS,
3389                store(h, h1, NULL, OPERATION_SET, keystr, "somevalue", &it, 0, 0),
3390                "Error setting.");
3391        h1->release(h, NULL, it);
3392    }
3393
3394    // Restart the server.
3395    testHarness.reload_engine(&h, &h1,
3396                              testHarness.engine_path,
3397                              testHarness.get_current_testcase()->cfg,
3398                              true, false);
3399
3400    wait_for_warmup_complete(h, h1);
3401    // n_items_to_access items should be loaded from access log first
3402    // but we continue to load until we hit 75% item watermark
3403
3404    int warmedup = get_int_stat(h, h1, "ep_warmup_value_count", "warmup");
3405    //    std::cout << "ep_warmup_value_count = " << warmedup << std::endl;
3406    int expected = (n_items_to_store1 + n_items_to_store2) * 0.75 + 1;
3407
3408    check(warmedup == expected, "Expected 16 items to be resident");
3409    return SUCCESS;
3410#endif
3411}
3412#endif
3413
3414static enum test_result test_warmup_oom(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
3415    if (!isWarmupEnabled(h, h1)) {
3416        return SKIPPED;
3417    }
3418
3419    write_items(h, h1, 20000, 0, "superlongnameofkey1234567890123456789012345678902");
3420
3421    wait_for_flusher_to_settle(h, h1);
3422
3423    std::string config(testHarness.get_current_testcase()->cfg);
3424    config = config + "max_size=2097152;item_eviction_policy=value_only";
3425
3426    testHarness.reload_engine(&h, &h1,
3427                              testHarness.engine_path,
3428                              config.c_str(),
3429                              true, false);
3430
3431    wait_for_warmup_complete(h, h1);
3432
3433    protocol_binary_request_header *pkt = createPacket(PROTOCOL_BINARY_CMD_ENABLE_TRAFFIC);
3434    checkeq(ENGINE_SUCCESS,
3435            h1->unknown_command(h, NULL, pkt, add_response, testHarness.doc_namespace),
3436            "Failed to send data traffic command to the services");
3437    checkeq(PROTOCOL_BINARY_RESPONSE_ENOMEM, last_status.load(),
3438            "Data traffic command should have failed with enomem");
3439    cb_free(pkt);
3440
3441    return SUCCESS;
3442}
3443
3444static enum test_result test_cbd_225(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
3445
3446    // get engine startup token
3447    time_t token1 = get_int_stat(h, h1, "ep_startup_time");
3448    check(token1 != 0, "Expected non-zero startup token");
3449
3450    // store some random data
3451    checkeq(ENGINE_SUCCESS,
3452            store(h, h1, NULL, OPERATION_SET, "k1", "v1"),
3453            "Failed to fail to store an item.");
3454    checkeq(ENGINE_SUCCESS,
3455            store(h, h1, NULL, OPERATION_SET, "k2", "v2"),
3456            "Failed to fail to store an item.");
3457    wait_for_flusher_to_settle(h, h1);
3458
3459    // check token again, which should be the same as before
3460    time_t token2 = get_int_stat(h, h1, "ep_startup_time");
3461    check(token2 == token1, "Expected the same startup token");
3462
3463    // reload the engine
3464    testHarness.time_travel(10);
3465    testHarness.reload_engine(&h, &h1,
3466                              testHarness.engine_path,
3467                              testHarness.get_current_testcase()->cfg,
3468                              true, false);
3469    wait_for_warmup_complete(h, h1);
3470
3471    // check token, this time we should get a different one
3472    time_t token3 = get_int_stat(h, h1, "ep_startup_time");
3473    check(token3 != token1, "Expected a different startup token");
3474
3475    return SUCCESS;
3476}
3477
3478static enum test_result test_workload_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
3479    const void* cookie = testHarness.create_cookie();
3480    checkeq(ENGINE_SUCCESS,
3481            h1->get_stats(h, cookie, "workload"_ccb, add_stats),
3482            "Falied to get workload stats");
3483    testHarness.destroy_cookie(cookie);
3484    int num_read_threads = get_int_stat(h, h1, "ep_workload:num_readers",
3485                                               "workload");
3486    int num_write_threads = get_int_stat(h, h1, "ep_workload:num_writers",
3487                                                "workload");
3488    int num_auxio_threads = get_int_stat(h, h1, "ep_workload:num_auxio",
3489                                                "workload");
3490    int num_nonio_threads = get_int_stat(h, h1, "ep_workload:num_nonio",
3491                                                "workload");
3492    int max_read_threads = get_int_stat(h, h1, "ep_workload:max_readers",
3493                                               "workload");