xref: /4.6.4/ep-engine/tests/ep_testsuite.cc (revision d7e17916)
1/* -*- MODE: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/*
3 *     Copyright 2010 Couchbase, Inc
4 *
5 *   Licensed under the Apache License, Version 2.0 (the "License");
6 *   you may not use this file except in compliance with the License.
7 *   You may obtain a copy of the License at
8 *
9 *       http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *   Unless required by applicable law or agreed to in writing, software
12 *   distributed under the License is distributed on an "AS IS" BASIS,
13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *   See the License for the specific language governing permissions and
15 *   limitations under the License.
16 */
17
18// Usage: (to run just a single test case)
19// make engine_tests EP_TEST_NUM=3
20
21#include "config.h"
22
23#include <stdio.h>
24#include <stdlib.h>
25#include <string.h>
26
27#include <condition_variable>
28#include <cstdlib>
29#include <chrono>
30#include <iostream>
31#include <iomanip>
32#include <map>
33#include <mutex>
34#include <set>
35#include <sstream>
36#include <string>
37#include <thread>
38#include <unordered_map>
39#include <unordered_set>
40#include <vector>
41
42#include "atomic.h"
43#include "compress.h"
44#include "ep_test_apis.h"
45
46#include "ep_testsuite_common.h"
47#include "locks.h"
48#include <libcouchstore/couch_db.h>
49#include "mock/mock_dcp.h"
50#include "mutex.h"
51#include <memcached/engine.h>
52#include <memcached/engine_testapp.h>
53#include <platform/cb_malloc.h>
54#include <platform/dirutils.h>
55#include <JSON_checker.h>
56
57#ifdef linux
58/* /usr/include/netinet/in.h defines macros from ntohs() to _bswap_nn to
59 * optimize the conversion functions, but the prototypes generate warnings
60 * from gcc. The conversion methods isn't the bottleneck for my app, so
61 * just remove the warnings by undef'ing the optimization ..
62 */
63#undef ntohs
64#undef ntohl
65#undef htons
66#undef htonl
67#endif
68
69// ptr_fun don't like the extern "C" thing for unlock cookie.. cast it
70// away ;)
71typedef void (*UNLOCK_COOKIE_T)(const void *cookie);
72
73#define MULTI_DISPATCHER_CONFIG \
74    "ht_size=129;ht_locks=3;chk_remover_stime=1;chk_period=60"
75
76class ThreadData {
77public:
78    ThreadData(ENGINE_HANDLE *eh, ENGINE_HANDLE_V1 *ehv1,
79               int e=0) : h(eh), h1(ehv1), extra(e) {}
80    ENGINE_HANDLE    *h;
81    ENGINE_HANDLE_V1 *h1;
82    int               extra;
83};
84
85
86static void check_observe_seqno(bool failover, uint8_t format_type, uint16_t vb_id,
87                                uint64_t vb_uuid, uint64_t last_persisted_seqno,
88                                uint64_t current_seqno, uint64_t failover_vbuuid = 0,
89                                uint64_t failover_seqno = 0) {
90    uint8_t  recv_format_type;
91    uint16_t recv_vb_id;
92    uint64_t recv_vb_uuid;
93    uint64_t recv_last_persisted_seqno;
94    uint64_t recv_current_seqno;
95    uint64_t recv_failover_vbuuid;
96    uint64_t recv_failover_seqno;
97
98    memcpy(&recv_format_type, last_body.data(), sizeof(uint8_t));
99    checkeq(format_type, recv_format_type, "Wrong format type in result");
100    memcpy(&recv_vb_id, last_body.data() + 1, sizeof(uint16_t));
101    checkeq(vb_id, ntohs(recv_vb_id), "Wrong vbucket id in result");
102    memcpy(&recv_vb_uuid, last_body.data() + 3, sizeof(uint64_t));
103    checkeq(vb_uuid, ntohll(recv_vb_uuid), "Wrong vbucket uuid in result");
104    memcpy(&recv_last_persisted_seqno, last_body.data() + 11, sizeof(uint64_t));
105    checkeq(last_persisted_seqno, ntohll(recv_last_persisted_seqno),
106            "Wrong persisted seqno in result");
107    memcpy(&recv_current_seqno, last_body.data() + 19, sizeof(uint64_t));
108    checkeq(current_seqno, ntohll(recv_current_seqno), "Wrong current seqno in result");
109
110    if (failover) {
111        memcpy(&recv_failover_vbuuid, last_body.data() + 27, sizeof(uint64_t));
112        checkeq(failover_vbuuid, ntohll(recv_failover_vbuuid),
113                "Wrong failover uuid in result");
114        memcpy(&recv_failover_seqno, last_body.data() + 35, sizeof(uint64_t));
115        checkeq(failover_seqno, ntohll(recv_failover_seqno),
116                "Wrong failover seqno in result");
117    }
118}
119
120static enum test_result test_replace_with_eviction(ENGINE_HANDLE *h,
121                                                   ENGINE_HANDLE_V1 *h1) {
122    item *i = NULL;
123    checkeq(ENGINE_SUCCESS,
124            store(h, h1, NULL, OPERATION_SET,"key", "somevalue", &i),
125            "Failed to set value.");
126    h1->release(h, NULL, i);
127    wait_for_flusher_to_settle(h, h1);
128    evict_key(h, h1, "key");
129    int numBgFetched = get_int_stat(h, h1, "ep_bg_fetched");
130
131    checkeq(ENGINE_SUCCESS,
132            store(h, h1, NULL, OPERATION_REPLACE,"key", "somevalue1", &i),
133            "Failed to replace existing value.");
134
135    checkeq(ENGINE_SUCCESS,
136            h1->get_stats(h, NULL, NULL, 0, add_stats),
137            "Failed to get stats.");
138    std::string eviction_policy = vals.find("ep_item_eviction_policy")->second;
139    if (eviction_policy == "full_eviction") {
140        numBgFetched++;
141    }
142
143    checkeq(numBgFetched,
144            get_int_stat(h, h1, "ep_bg_fetched"),
145            "Bg fetched value didn't match");
146
147    h1->release(h, NULL, i);
148    check_key_value(h, h1, "key", "somevalue1", 10);
149    return SUCCESS;
150}
151
152static enum test_result test_wrong_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
153                                               ENGINE_STORE_OPERATION op) {
154    item *i = NULL;
155    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
156    uint64_t cas = 11;
157    if (op == OPERATION_ADD) {
158        // Add operation with cas != 0 doesn't make sense
159        cas = 0;
160    }
161    checkeq(ENGINE_NOT_MY_VBUCKET,
162            store(h, h1, NULL, op, "key", "somevalue", &i, cas, 1),
163            "Expected not_my_vbucket");
164    h1->release(h, NULL, i);
165    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
166    return SUCCESS;
167}
168
169static enum test_result test_pending_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
170                                                 ENGINE_STORE_OPERATION op) {
171    const void *cookie = testHarness.create_cookie();
172    testHarness.set_ewouldblock_handling(cookie, false);
173    item *i = NULL;
174    check(set_vbucket_state(h, h1, 1, vbucket_state_pending),
175          "Failed to set vbucket state.");
176    check(verify_vbucket_state(h, h1, 1, vbucket_state_pending),
177          "Bucket state was not set to pending.");
178    uint64_t cas = 11;
179    if (op == OPERATION_ADD) {
180        // Add operation with cas != 0 doesn't make sense..
181        cas = 0;
182    }
183    checkeq(ENGINE_EWOULDBLOCK,
184            store(h, h1, cookie, op, "key", "somevalue", &i, cas, 1),
185            "Expected ewouldblock");
186    h1->release(h, NULL, i);
187    testHarness.destroy_cookie(cookie);
188    return SUCCESS;
189}
190
191static enum test_result test_replica_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
192                                                 ENGINE_STORE_OPERATION op) {
193    item *i = NULL;
194    check(set_vbucket_state(h, h1, 1, vbucket_state_replica),
195          "Failed to set vbucket state.");
196    check(verify_vbucket_state(h, h1, 1, vbucket_state_replica),
197          "Bucket state was not set to replica.");
198    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
199
200    uint64_t cas = 11;
201    if (op == OPERATION_ADD) {
202        // performing add with a CAS != 0 doesn't make sense...
203        cas = 0;
204    }
205    checkeq(ENGINE_NOT_MY_VBUCKET,
206            store(h, h1, NULL, op, "key", "somevalue", &i, cas, 1),
207            "Expected not my vbucket");
208    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
209    h1->release(h, NULL, i);
210    return SUCCESS;
211}
212
213//
214// ----------------------------------------------------------------------
215// The actual tests are below.
216// ----------------------------------------------------------------------
217//
218
219static int checkCurrItemsAfterShutdown(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
220                                       int numItems2Load, bool shutdownForce) {
221    std::vector<std::string> keys;
222    for (int index = 0; index < numItems2Load; ++index) {
223        std::stringstream s;
224        s << "keys_2_load-" << index;
225        std::string key(s.str());
226        keys.push_back(key);
227    }
228
229    checkeq(0, get_int_stat(h, h1, "ep_total_persisted"),
230            "Expected ep_total_persisted equals 0");
231    checkeq(0, get_int_stat(h, h1, "curr_items"),
232            "Expected curr_items equals 0");
233
234    // stop flusher before loading new items
235    protocol_binary_request_header *pkt = createPacket(PROTOCOL_BINARY_CMD_STOP_PERSISTENCE);
236    checkeq(ENGINE_SUCCESS,
237            h1->unknown_command(h, NULL, pkt, add_response),
238            "CMD_STOP_PERSISTENCE failed!");
239    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS,
240            last_status.load(),
241            "Failed to stop persistence!");
242    cb_free(pkt);
243
244    std::vector<std::string>::iterator itr;
245    for (itr = keys.begin(); itr != keys.end(); ++itr) {
246        item *i;
247        checkeq(ENGINE_SUCCESS,
248                store(h, h1, NULL, OPERATION_SET, itr->c_str(), "oracle", &i, 0, 0),
249                "Failed to store a value");
250        h1->release(h, NULL, i);
251    }
252
253    checkeq(0, get_int_stat(h, h1, "ep_total_persisted"),
254            "Incorrect ep_total_persisted, expected 0");
255    std::stringstream ss;
256    ss << "Incorrect curr_items, expected " << numItems2Load;
257    std::string errmsg(ss.str());
258    checkeq(numItems2Load, get_int_stat(h, h1, "curr_items"),
259            errmsg.c_str());
260
261    // resume flusher before shutdown + warmup
262    pkt = createPacket(PROTOCOL_BINARY_CMD_START_PERSISTENCE);
263    checkeq(ENGINE_SUCCESS, h1->unknown_command(h, NULL, pkt, add_response),
264            "CMD_START_PERSISTENCE failed!");
265    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
266          "Failed to start persistence!");
267    cb_free(pkt);
268
269    // shutdown engine force and restart
270    testHarness.reload_engine(&h, &h1,
271                              testHarness.engine_path,
272                              testHarness.get_current_testcase()->cfg,
273                              true, shutdownForce);
274    wait_for_warmup_complete(h, h1);
275    return get_int_stat(h, h1, "curr_items");
276}
277
278static enum test_result test_flush_shutdown_force(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
279    int numItems2load = 3000;
280    bool shutdownForce = true;
281    int currItems = checkCurrItemsAfterShutdown(h, h1, numItems2load, shutdownForce);
282    check (currItems <= numItems2load,
283           "Number of curr items should be <= 3000, unless previous "
284           "shutdown force had to wait for the flusher");
285    return SUCCESS;
286}
287
288static enum test_result test_flush_shutdown_noforce(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
289    int numItems2load = 3000;
290    bool shutdownForce = false;
291    int currItems = checkCurrItemsAfterShutdown(h, h1, numItems2load, shutdownForce);
292    check (currItems == numItems2load,
293           "Number of curr items should be equal to 3000, unless previous "
294           "shutdown did not wait for the flusher");
295    return SUCCESS;
296}
297
298static enum test_result test_flush_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
299    item *i = NULL;
300    // First try to delete something we know to not be there.
301    checkeq(ENGINE_KEY_ENOENT, del(h, h1, "key", 0, 0),
302            "Failed to fail initial delete.");
303    checkeq(ENGINE_SUCCESS,
304            store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
305            "Failed set.");
306    h1->release(h, NULL, i);
307    check_key_value(h, h1, "key", "somevalue", 9);
308
309    // Restart once to ensure written to disk.
310    testHarness.reload_engine(&h, &h1,
311                              testHarness.engine_path,
312                              testHarness.get_current_testcase()->cfg,
313                              true, false);
314    wait_for_warmup_complete(h, h1);
315
316    // Read value from disk.
317    check_key_value(h, h1, "key", "somevalue", 9);
318
319    // Flush
320    set_degraded_mode(h, h1, NULL, true);
321    checkeq(ENGINE_SUCCESS, h1->flush(h, NULL, 0),
322            "Failed to flush");
323    set_degraded_mode(h, h1, NULL, false);
324
325    checkeq(ENGINE_SUCCESS,
326            store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i),
327            "Failed post-flush set.");
328    h1->release(h, NULL, i);
329    check_key_value(h, h1, "key2", "somevalue", 9);
330
331    // Restart again, ensure written to disk.
332    testHarness.reload_engine(&h, &h1,
333                              testHarness.engine_path,
334                              testHarness.get_current_testcase()->cfg,
335                              true, false);
336    wait_for_warmup_complete(h, h1);
337
338    checkeq(ENGINE_SUCCESS,
339            store(h, h1, NULL, OPERATION_SET, "key3", "somevalue", &i),
340            "Failed post-flush, post-restart set.");
341    h1->release(h, NULL, i);
342    check_key_value(h, h1, "key3", "somevalue", 9);
343
344    // Read value again, should not be there.
345    checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"),
346            "Expected missing key");
347    return SUCCESS;
348}
349
350static enum test_result test_shutdown_snapshot_range(ENGINE_HANDLE *h,
351                                                     ENGINE_HANDLE_V1 *h1) {
352    const int num_items = 100;
353    for (int j = 0; j < num_items; ++j) {
354        item *i = NULL;
355        std::stringstream ss;
356        ss << "key" << j;
357        checkeq(ENGINE_SUCCESS,
358                store(h, h1, NULL, OPERATION_SET, ss.str().c_str(), "data", &i),
359                "Failed to store a value");
360        h1->release(h, NULL, i);
361    }
362
363    wait_for_flusher_to_settle(h, h1);
364    int end = get_int_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno");
365
366    /* change vb state to replica before restarting (as it happens in graceful
367       failover)*/
368    check(set_vbucket_state(h, h1, 0, vbucket_state_replica),
369          "Failed set vbucket 0 to replica state.");
370
371    /* trigger persist vb state task */
372    check(set_param(h, h1, protocol_binary_engine_param_flush,
373                    "vb_state_persist_run", "0"),
374          "Failed to trigger vb state persist");
375
376    /* restart the engine */
377    testHarness.reload_engine(&h, &h1,
378                              testHarness.engine_path,
379                              testHarness.get_current_testcase()->cfg,
380                              true, false);
381    wait_for_warmup_complete(h, h1);
382
383    /* Check if snapshot range is persisted correctly */
384    checkeq(end, get_int_stat(h, h1, "vb_0:last_persisted_snap_start",
385                              "vbucket-seqno"),
386            "Wrong snapshot start persisted");
387    checkeq(end, get_int_stat(h, h1, "vb_0:last_persisted_snap_end",
388                                    "vbucket-seqno"),
389            "Wrong snapshot end persisted");
390
391    return SUCCESS;
392}
393
394static enum test_result test_flush_multiv_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
395    item *i = NULL;
396    check(set_vbucket_state(h, h1, 2, vbucket_state_active),
397          "Failed to set vbucket state.");
398    checkeq(ENGINE_SUCCESS,
399            store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
400            "Failed set.");
401    h1->release(h, NULL, i);
402    checkeq(ENGINE_SUCCESS,
403            store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i, 0, 2),
404            "Failed set in vb2.");
405    h1->release(h, NULL, i);
406
407    // Restart once to ensure written to disk.
408    testHarness.reload_engine(&h, &h1,
409                              testHarness.engine_path,
410                              testHarness.get_current_testcase()->cfg,
411                              true, false);
412    wait_for_warmup_complete(h, h1);
413
414    // Read value from disk.
415    check_key_value(h, h1, "key", "somevalue", 9);
416
417    // Flush
418    set_degraded_mode(h, h1, NULL, true);
419    checkeq(ENGINE_SUCCESS, h1->flush(h, NULL, 0),
420            "Failed to flush");
421    set_degraded_mode(h, h1, NULL, false);
422
423    // Restart again, ensure written to disk.
424    testHarness.reload_engine(&h, &h1,
425                              testHarness.engine_path,
426                              testHarness.get_current_testcase()->cfg,
427                              true, false);
428    wait_for_warmup_complete(h, h1);
429
430    // Read value again, should not be there.
431    checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"), "Expected missing key");
432    check(verify_vbucket_missing(h, h1, 2), "Bucket 2 came back.");
433    return SUCCESS;
434}
435
436static enum test_result test_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
437    item *i = NULL;
438    static const char val[] = "somevalue";
439    ENGINE_ERROR_CODE ret = store(h, h1, NULL, OPERATION_SET, "key", val, &i);
440    checkeq(ENGINE_SUCCESS, ret, "Failed set.");
441    h1->release(h, NULL, i);
442
443    testHarness.reload_engine(&h, &h1,
444                              testHarness.engine_path,
445                              testHarness.get_current_testcase()->cfg,
446                              true, false);
447    wait_for_warmup_complete(h, h1);
448    check_key_value(h, h1, "key", val, strlen(val));
449    return SUCCESS;
450}
451
452static enum test_result test_restart_session_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
453    const void* cookie = createTapConn(h, h1, "tap_client_thread");
454    testHarness.unlock_cookie(cookie);
455    testHarness.destroy_cookie(cookie);
456
457    testHarness.reload_engine(&h, &h1,
458                              testHarness.engine_path,
459                              testHarness.get_current_testcase()->cfg,
460                              true, false);
461    wait_for_warmup_complete(h, h1);
462    cookie = createTapConn(h, h1, "tap_client_thread");
463
464    checkeq(ENGINE_SUCCESS, h1->get_stats(h, NULL, "tap", 3, add_stats),
465            "Failed to get stats.");
466    std::string val = vals["eq_tapq:tap_client_thread:backfill_completed"];
467    checkeq(0, strcmp(val.c_str(), "true"), "Don't expect the backfill upon restart");
468    testHarness.unlock_cookie(cookie);
469    testHarness.destroy_cookie(cookie);
470    return SUCCESS;
471}
472
473static enum test_result test_specialKeys(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
474    item *i = NULL;
475    ENGINE_ERROR_CODE ret;
476
477    // Simplified Chinese "Couchbase"
478    static const char key0[] = "沙发数据库";
479    static const char val0[] = "some Chinese value";
480    check((ret = store(h, h1, NULL, OPERATION_SET, key0, val0, &i)) == ENGINE_SUCCESS,
481          "Failed set Chinese key");
482    check_key_value(h, h1, key0, val0, strlen(val0));
483    h1->release(h, NULL, i);
484    // Traditional Chinese "Couchbase"
485    static const char key1[] = "沙發數據庫";
486    static const char val1[] = "some Traditional Chinese value";
487    check((ret = store(h, h1, NULL, OPERATION_SET, key1, val1, &i)) == ENGINE_SUCCESS,
488          "Failed set Traditional Chinese key");
489    h1->release(h, NULL, i);
490    // Korean "couch potato"
491    static const char key2[] = "쇼파감자";
492    static const char val2[] = "some Korean value";
493    check((ret = store(h, h1, NULL, OPERATION_SET, key2, val2, &i)) == ENGINE_SUCCESS,
494          "Failed set Korean key");
495    h1->release(h, NULL, i);
496    // Russian "couch potato"
497    static const char key3[] = "лодырь, лентяй";
498    static const char val3[] = "some Russian value";
499    check((ret = store(h, h1, NULL, OPERATION_SET, key3, val3, &i)) == ENGINE_SUCCESS,
500          "Failed set Russian key");
501    h1->release(h, NULL, i);
502    // Japanese "couch potato"
503    static const char key4[] = "カウチポテト";
504    static const char val4[] = "some Japanese value";
505    check((ret = store(h, h1, NULL, OPERATION_SET, key4, val4, &i)) == ENGINE_SUCCESS,
506          "Failed set Japanese key");
507    h1->release(h, NULL, i);
508    // Indian char key, and no idea what it is
509    static const char key5[] = "हरियानवी";
510    static const char val5[] = "some Indian value";
511    check((ret = store(h, h1, NULL, OPERATION_SET, key5, val5, &i)) == ENGINE_SUCCESS,
512          "Failed set Indian key");
513    h1->release(h, NULL, i);
514    // Portuguese translation "couch potato"
515    static const char key6[] = "sedentário";
516    static const char val6[] = "some Portuguese value";
517    check((ret = store(h, h1, NULL, OPERATION_SET, key6, val6, &i)) == ENGINE_SUCCESS,
518          "Failed set Portuguese key");
519    h1->release(h, NULL, i);
520    // Arabic translation "couch potato"
521    static const char key7[] = "الحافلةالبطاطة";
522    static const char val7[] = "some Arabic value";
523    check((ret = store(h, h1, NULL, OPERATION_SET, key7, val7, &i)) == ENGINE_SUCCESS,
524          "Failed set Arabic key");
525    h1->release(h, NULL, i);
526
527    testHarness.reload_engine(&h, &h1,
528                              testHarness.engine_path,
529                              testHarness.get_current_testcase()->cfg,
530                              true, false);
531    wait_for_warmup_complete(h, h1);
532    check_key_value(h, h1, key0, val0, strlen(val0));
533    check_key_value(h, h1, key1, val1, strlen(val1));
534    check_key_value(h, h1, key2, val2, strlen(val2));
535    check_key_value(h, h1, key3, val3, strlen(val3));
536    check_key_value(h, h1, key4, val4, strlen(val4));
537    check_key_value(h, h1, key5, val5, strlen(val5));
538    check_key_value(h, h1, key6, val6, strlen(val6));
539    check_key_value(h, h1, key7, val7, strlen(val7));
540    return SUCCESS;
541}
542
543static enum test_result test_binKeys(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
544    item *i = NULL;
545    ENGINE_ERROR_CODE ret;
546
547    // binary key with char values beyond 0x7F
548    static const char key0[] = "\xe0\xed\xf1\x6f\x7f\xf8\xfa";
549    static const char val0[] = "some value val8";
550    check((ret = store(h, h1, NULL, OPERATION_SET, key0, val0, &i)) == ENGINE_SUCCESS,
551          "Failed set binary key0");
552    check_key_value(h, h1, key0, val0, strlen(val0));
553    h1->release(h, NULL, i);
554    // binary keys with char values beyond 0x7F
555    static const char key1[] = "\xf1\xfd\xfe\xff\xf0\xf8\xef";
556    static const char val1[] = "some value val9";
557    check((ret = store(h, h1, NULL, OPERATION_SET, key1, val1, &i)) == ENGINE_SUCCESS,
558          "Failed set binary key1");
559    check_key_value(h, h1, key1, val1, strlen(val1));
560    h1->release(h, NULL, i);
561    // binary keys with special utf-8 BOM (Byte Order Mark) values 0xBB 0xBF 0xEF
562    static const char key2[] = "\xff\xfe\xbb\xbf\xef";
563    static const char val2[] = "some utf-8 bom value";
564    check((ret = store(h, h1, NULL, OPERATION_SET, key2, val2, &i)) == ENGINE_SUCCESS,
565          "Failed set binary utf-8 bom key");
566    check_key_value(h, h1, key2, val2, strlen(val2));
567    h1->release(h, NULL, i);
568    // binary keys with special utf-16BE BOM values "U+FEFF"
569    static const char key3[] = "U+\xfe\xff\xefU+\xff\xfe";
570    static const char val3[] = "some utf-16 bom value";
571    check((ret = store(h, h1, NULL, OPERATION_SET, key3, val3, &i)) == ENGINE_SUCCESS,
572          "Failed set binary utf-16 bom key");
573    check_key_value(h, h1, key3, val3, strlen(val3));
574    h1->release(h, NULL, i);
575
576    testHarness.reload_engine(&h, &h1,
577                              testHarness.engine_path,
578                              testHarness.get_current_testcase()->cfg,
579                              true, false);
580    wait_for_warmup_complete(h, h1);
581    check_key_value(h, h1, key0, val0, strlen(val0));
582    check_key_value(h, h1, key1, val1, strlen(val1));
583    check_key_value(h, h1, key2, val2, strlen(val2));
584    check_key_value(h, h1, key3, val3, strlen(val3));
585    return SUCCESS;
586}
587
588static enum test_result test_restart_bin_val(ENGINE_HANDLE *h,
589                                             ENGINE_HANDLE_V1 *h1) {
590
591
592
593    char binaryData[] = "abcdefg\0gfedcba";
594    cb_assert(sizeof(binaryData) != strlen(binaryData));
595
596    item *i = NULL;
597    checkeq(ENGINE_SUCCESS,
598            storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
599                         binaryData, sizeof(binaryData), 82758, &i, 0, 0),
600            "Failed set.");
601    h1->release(h, NULL, i);
602
603    testHarness.reload_engine(&h, &h1,
604                              testHarness.engine_path,
605                              testHarness.get_current_testcase()->cfg,
606                              true, false);
607    wait_for_warmup_complete(h, h1);
608
609    check_key_value(h, h1, "key", binaryData, sizeof(binaryData));
610    return SUCCESS;
611}
612
613static enum test_result test_wrong_vb_get(ENGINE_HANDLE *h,
614                                          ENGINE_HANDLE_V1 *h1) {
615    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
616    checkeq(ENGINE_NOT_MY_VBUCKET, verify_key(h, h1, "key", 1),
617            "Expected wrong bucket.");
618    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
619    return SUCCESS;
620}
621
622static enum test_result test_vb_get_pending(ENGINE_HANDLE *h,
623                                            ENGINE_HANDLE_V1 *h1) {
624    check(set_vbucket_state(h, h1, 1, vbucket_state_pending),
625          "Failed to set vbucket state.");
626    const void *cookie = testHarness.create_cookie();
627    testHarness.set_ewouldblock_handling(cookie, false);
628
629    item *i = NULL;
630    checkeq(ENGINE_EWOULDBLOCK,
631            h1->get(h, cookie, &i, "key", strlen("key"), 1),
632            "Expected woodblock.");
633    h1->release(h, NULL, i);
634
635    testHarness.destroy_cookie(cookie);
636    return SUCCESS;
637}
638
639static enum test_result test_vb_get_replica(ENGINE_HANDLE *h,
640                                            ENGINE_HANDLE_V1 *h1) {
641    check(set_vbucket_state(h, h1, 1, vbucket_state_replica),
642          "Failed to set vbucket state.");
643    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
644    checkeq(ENGINE_NOT_MY_VBUCKET,
645            verify_key(h, h1, "key", 1),
646            "Expected not my bucket.");
647    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
648    return SUCCESS;
649}
650
651static enum test_result test_wrong_vb_incr(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
652    uint64_t result;
653    item *i = NULL;
654    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
655    checkeq(ENGINE_NOT_MY_VBUCKET,
656            h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
657                           &i, PROTOCOL_BINARY_RAW_BYTES, &result, 1),
658            "Expected not my vbucket.");
659    h1->release(h, NULL, i);
660    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
661    return SUCCESS;
662}
663
664static enum test_result test_vb_incr_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
665    const void *cookie = testHarness.create_cookie();
666    testHarness.set_ewouldblock_handling(cookie, false);
667    uint64_t result;
668    item *i = NULL;
669    check(set_vbucket_state(h, h1, 1, vbucket_state_pending),
670          "Failed to set vbucket state.");
671    checkeq(ENGINE_EWOULDBLOCK,
672            h1->arithmetic(h, cookie, "key", 3, true, false, 1, 1, 0,
673                           &i, PROTOCOL_BINARY_RAW_BYTES, &result, 1),
674            "Expected wouldblock.");
675    h1->release(h, NULL, i);
676    testHarness.destroy_cookie(cookie);
677    return SUCCESS;
678}
679
680static enum test_result test_vb_incr_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
681    uint64_t result;
682    item *i = NULL;
683    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set vbucket state.");
684    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
685    checkeq(ENGINE_NOT_MY_VBUCKET,
686            h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
687                           &i, PROTOCOL_BINARY_RAW_BYTES, &result, 1),
688            "Expected not my bucket.");
689    h1->release(h, NULL, i);
690    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
691    return SUCCESS;
692}
693
694static enum test_result test_wrong_vb_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
695    return test_wrong_vb_mutation(h, h1, OPERATION_SET);
696}
697
698static enum test_result test_wrong_vb_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
699    return test_wrong_vb_mutation(h, h1, OPERATION_CAS);
700}
701
702static enum test_result test_wrong_vb_add(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
703    return test_wrong_vb_mutation(h, h1, OPERATION_ADD);
704}
705
706static enum test_result test_wrong_vb_replace(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
707    return test_wrong_vb_mutation(h, h1, OPERATION_REPLACE);
708}
709
710static enum test_result test_wrong_vb_append(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
711    return test_wrong_vb_mutation(h, h1, OPERATION_APPEND);
712}
713
714static enum test_result test_wrong_vb_prepend(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
715    return test_wrong_vb_mutation(h, h1, OPERATION_PREPEND);
716}
717
718static enum test_result test_wrong_vb_del(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
719    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
720    checkeq(ENGINE_NOT_MY_VBUCKET, del(h, h1, "key", 0, 1),
721            "Expected wrong bucket.");
722    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
723    return SUCCESS;
724}
725
726/* Returns a string in the format "%Y-%m-%d %H:%M:%S" of the specified
727 * time point.
728 */
729std::string make_time_string(std::chrono::system_clock::time_point time_point) {
730    time_t tt = std::chrono::system_clock::to_time_t(time_point);
731#ifdef _MSC_VER
732    // Windows' gmtime() is already thread-safe.
733    struct tm* split = gmtime(&tt);
734#else
735    struct tm local_storage;
736    struct tm* split = gmtime_r(&tt, &local_storage);
737#endif
738    char timeStr[20];
739    strftime(timeStr, 20, "%Y-%m-%d %H:%M:%S", split);
740    return timeStr;
741}
742
743static enum test_result test_expiry_pager_settings(ENGINE_HANDLE *h,
744                                                   ENGINE_HANDLE_V1 *h1) {
745
746    cb_assert(!get_bool_stat(h, h1, "ep_exp_pager_enabled"));
747    checkeq(3600, get_int_stat(h, h1, "ep_exp_pager_stime"),
748            "Expiry pager sleep time not expected");
749    set_param(h, h1, protocol_binary_engine_param_flush,
750              "exp_pager_stime", "1");
751    checkeq(1, get_int_stat(h, h1, "ep_exp_pager_stime"),
752            "Expiry pager sleep time not updated");
753    cb_assert(!get_bool_stat(h, h1, "ep_exp_pager_enabled"));
754    sleep(1);
755    checkeq(0, get_int_stat(h, h1, "ep_num_expiry_pager_runs"),
756            "Expiry pager run count is not zero");
757
758    set_param(h, h1, protocol_binary_engine_param_flush,
759              "exp_pager_enabled", "true");
760    checkeq(1, get_int_stat(h, h1, "ep_exp_pager_stime"),
761            "Expiry pager sleep time not updated");
762    wait_for_stat_to_be_gte(h, h1, "ep_num_expiry_pager_runs", 1);
763
764    // Reload engine
765    testHarness.reload_engine(&h, &h1,
766                              testHarness.engine_path,
767                              testHarness.get_current_testcase()->cfg,
768                              true, false);
769    wait_for_warmup_complete(h, h1);
770    cb_assert(!get_bool_stat(h, h1, "ep_exp_pager_enabled"));
771
772    // Enable expiry pager again
773    set_param(h, h1, protocol_binary_engine_param_flush,
774              "exp_pager_enabled", "true");
775
776    checkeq(get_int_stat(h, h1, "ep_exp_pager_initial_run_time"), -1,
777            "Task time should be disable upon warmup");
778
779    std::string err_msg;
780    // Update exp_pager_initial_run_time and ensure the update is successful
781    set_param(h, h1, protocol_binary_engine_param_flush,
782              "exp_pager_initial_run_time", "3");
783    std::string expected_time = "03:00";
784    std::string str = get_str_stat(h, h1, "ep_expiry_pager_task_time");
785    err_msg.assign("Updated time incorrect, expect: " +
786                   expected_time + ", actual: " + str.substr(11, 5));
787    checkeq(0, str.substr(11, 5).compare(expected_time), err_msg.c_str());
788
789    // Update exp_pager_stime by 30 minutes and ensure that the update is successful
790    const std::chrono::minutes update_by{30};
791    std::string targetTaskTime1{make_time_string(std::chrono::system_clock::now() +
792                                                 update_by)};
793
794    set_param(h, h1, protocol_binary_engine_param_flush, "exp_pager_stime",
795              std::to_string(update_by.count() * 60).c_str());
796    str = get_str_stat(h, h1, "ep_expiry_pager_task_time");
797
798    std::string targetTaskTime2{make_time_string(std::chrono::system_clock::now() +
799                                                 update_by)};
800
801    // ep_expiry_pager_task_time should fall within the range of
802    // targetTaskTime1 and targetTaskTime2
803    err_msg.assign("Unexpected task time range, expect: " +
804                   targetTaskTime1 + " <= " + str + " <= " + targetTaskTime2);
805    check(targetTaskTime1 <= str, err_msg.c_str());
806    check(str <= targetTaskTime2, err_msg.c_str());
807
808    return SUCCESS;
809}
810
811static enum test_result test_expiry(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
812    const char *key = "test_expiry";
813    const char *data = "some test data here.";
814
815    item *it = NULL;
816
817    ENGINE_ERROR_CODE rv;
818    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 2,
819                      PROTOCOL_BINARY_RAW_BYTES);
820    checkeq(ENGINE_SUCCESS, rv, "Allocation failed.");
821
822    item_info info;
823    info.nvalue = 1;
824    if (!h1->get_item_info(h, NULL, it, &info)) {
825        abort();
826    }
827    memcpy(info.value[0].iov_base, data, strlen(data));
828
829    uint64_t cas = 0;
830    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
831    checkeq(ENGINE_SUCCESS, rv, "Set failed.");
832    check_key_value(h, h1, key, data, strlen(data));
833    h1->release(h, NULL, it);
834
835    testHarness.time_travel(5);
836    checkeq(ENGINE_KEY_ENOENT,
837            h1->get(h, NULL, &it, key, strlen(key), 0),
838            "Item didn't expire");
839
840    int expired_access = get_int_stat(h, h1, "ep_expired_access");
841    int expired_pager = get_int_stat(h, h1, "ep_expired_pager");
842    int active_expired = get_int_stat(h, h1, "vb_active_expired");
843    checkeq(0, expired_pager, "Expected zero expired item by pager");
844    checkeq(1, expired_access, "Expected an expired item on access");
845    checkeq(1, active_expired, "Expected an expired active item");
846    checkeq(ENGINE_SUCCESS, store(h, h1, NULL, OPERATION_SET, key, data, &it),
847            "Failed set.");
848    h1->release(h, NULL, it);
849
850    std::stringstream ss;
851    ss << "curr_items stat should be still 1 after ";
852    ss << "overwriting the key that was expired, but not purged yet";
853    checkeq(1, get_int_stat(h, h1, "curr_items"), ss.str().c_str());
854
855    return SUCCESS;
856}
857
858static enum test_result test_expiry_loader(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
859    const char *key = "test_expiry_loader";
860    const char *data = "some test data here.";
861
862    item *it = NULL;
863
864    ENGINE_ERROR_CODE rv;
865    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 2,
866                      PROTOCOL_BINARY_RAW_BYTES);
867    checkeq(ENGINE_SUCCESS, rv, "Allocation failed.");
868
869    item_info info;
870    info.nvalue = 1;
871    if (!h1->get_item_info(h, NULL, it, &info)) {
872        abort();
873    }
874    memcpy(info.value[0].iov_base, data, strlen(data));
875
876    uint64_t cas = 0;
877    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
878    checkeq(ENGINE_SUCCESS, rv, "Set failed.");
879    check_key_value(h, h1, key, data, strlen(data));
880    h1->release(h, NULL, it);
881
882    testHarness.time_travel(3);
883
884    checkeq(ENGINE_KEY_ENOENT,
885            h1->get(h, NULL, &it, key, strlen(key), 0),
886            "Item didn't expire");
887
888    // Restart the engine to ensure the above expired item is not loaded
889    testHarness.reload_engine(&h, &h1,
890                              testHarness.engine_path,
891                              testHarness.get_current_testcase()->cfg,
892                              true, false);
893    wait_for_warmup_complete(h, h1);
894    cb_assert(0 == get_int_stat(h, h1, "ep_warmup_value_count", "warmup"));
895
896    return SUCCESS;
897}
898
899static enum test_result test_expiration_on_compaction(ENGINE_HANDLE *h,
900                                                      ENGINE_HANDLE_V1 *h1) {
901    if (get_bool_stat(h, h1, "ep_exp_pager_enabled")) {
902        set_param(h, h1, protocol_binary_engine_param_flush,
903                  "exp_pager_enabled", "false");
904    }
905
906    for (int i = 0; i < 50; i++) {
907        item *itm = NULL;
908        std::stringstream ss;
909        ss << "key" << i;
910        checkeq(ENGINE_SUCCESS,
911                store(h, h1, NULL, OPERATION_SET, ss.str().c_str(),
912                      "somevalue", &itm, 0, 0, 10,
913                      PROTOCOL_BINARY_RAW_BYTES),
914                "Set failed.");
915        h1->release(h, NULL, itm);
916    }
917
918    wait_for_flusher_to_settle(h, h1);
919    checkeq(50, get_int_stat(h, h1, "curr_items"),
920            "Unexpected number of items on database");
921
922    testHarness.time_travel(15);
923
924    // Compaction on VBucket
925    compact_db(h, h1, 0, 0, 0, 0);
926    wait_for_stat_to_be(h, h1, "ep_pending_compactions", 0);
927
928    checkeq(50, get_int_stat(h, h1, "ep_expired_compactor"),
929            "Unexpected expirations by compactor");
930
931    return SUCCESS;
932}
933
934static enum test_result test_expiration_on_warmup(ENGINE_HANDLE *h,
935                                                  ENGINE_HANDLE_V1 *h1) {
936
937    set_param(h, h1, protocol_binary_engine_param_flush,
938              "exp_pager_enabled", "false");
939    int pager_runs = get_int_stat(h, h1, "ep_num_expiry_pager_runs");
940
941    const char *key = "KEY";
942    const char *data = "VALUE";
943
944    item *it = NULL;
945
946    ENGINE_ERROR_CODE rv;
947    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 10,
948                      PROTOCOL_BINARY_RAW_BYTES);
949    checkeq(ENGINE_SUCCESS, rv, "Allocation failed.");
950
951    item_info info;
952    info.nvalue = 1;
953    if (!h1->get_item_info(h, NULL, it, &info)) {
954        abort();
955    }
956    memcpy(info.value[0].iov_base, data, strlen(data));
957
958    uint64_t cas = 0;
959    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
960    checkeq(ENGINE_SUCCESS, rv, "Set failed.");
961    check_key_value(h, h1, key, data, strlen(data));
962    h1->release(h, NULL, it);
963    wait_for_flusher_to_settle(h, h1);
964
965    checkeq(1, get_int_stat(h, h1, "curr_items"), "Failed store item");
966    testHarness.time_travel(15);
967
968    checkeq(pager_runs, get_int_stat(h, h1, "ep_num_expiry_pager_runs"),
969            "Expiry pager shouldn't have run during this time");
970
971    // Restart the engine to ensure the above item is expired
972    testHarness.reload_engine(&h, &h1,
973                              testHarness.engine_path,
974                              testHarness.get_current_testcase()->cfg,
975                              true, false);
976    wait_for_warmup_complete(h, h1);
977    check(get_bool_stat(h, h1, "ep_exp_pager_enabled"),
978          "Expiry pager should be enabled on warmup");
979
980    // Wait for the expiry pager to run and expire our item.
981    wait_for_stat_to_be_gte(h, h1, "ep_expired_pager", 1, nullptr, /*secs*/10);
982
983    // Note: previously we checked that curr_items was zero here (immediately
984    // after waiting for ep_expired_pager == 1), however we cannot assume that
985    // - items are actually expired asynchronously.
986    // See EPStore::deleteExpiredItem - for non-temporary, expired items we
987    // call unlocked_softDelete (soft-marking the item as deleted in the
988    // hashtable), and then call queueDirty to queue a deletion, and then
989    // increment the expired stat. Only when that delete is actually persisted
990    // and the deleted callback is invoked -
991    // PeristenceCallback::callback(int&) - is curr_items finally decremented.
992    // Therefore we need to wait for the flusher to settle (i.e. delete
993    // callback to be called) for the curr_items stat to be accurate.
994    wait_for_flusher_to_settle(h, h1);
995
996    checkeq(0, get_int_stat(h, h1, "curr_items"),
997            "The item should have been expired.");
998
999    return SUCCESS;
1000
1001}
1002
1003static enum test_result test_bug3454(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1004    const char *key = "test_expiry_duplicate_warmup";
1005    const char *data = "some test data here.";
1006
1007    item *it = NULL;
1008
1009    ENGINE_ERROR_CODE rv;
1010    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 5,
1011                      PROTOCOL_BINARY_RAW_BYTES);
1012    checkeq(ENGINE_SUCCESS, rv, "Allocation failed.");
1013
1014    item_info info;
1015    info.nvalue = 1;
1016    if (!h1->get_item_info(h, NULL, it, &info)) {
1017        abort();
1018    }
1019    memcpy(info.value[0].iov_base, data, strlen(data));
1020
1021    uint64_t cas = 0;
1022    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
1023    checkeq(ENGINE_SUCCESS, rv, "Set failed.");
1024    check_key_value(h, h1, key, data, strlen(data));
1025    h1->release(h, NULL, it);
1026    wait_for_flusher_to_settle(h, h1);
1027
1028    // Advance the ep_engine time by 10 sec for the above item to be expired.
1029    testHarness.time_travel(10);
1030    checkeq(ENGINE_KEY_ENOENT,
1031            h1->get(h, NULL, &it, key, strlen(key), 0),
1032            "Item didn't expire");
1033
1034    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 0,
1035                      PROTOCOL_BINARY_RAW_BYTES);
1036    checkeq(ENGINE_SUCCESS, rv, "Allocation failed.");
1037
1038    info.nvalue = 1;
1039    if (!h1->get_item_info(h, NULL, it, &info)) {
1040        abort();
1041    }
1042    memcpy(info.value[0].iov_base, data, strlen(data));
1043
1044    cas = 0;
1045    // Add a new item with the same key.
1046    rv = h1->store(h, NULL, it, &cas, OPERATION_ADD, 0);
1047    checkeq(ENGINE_SUCCESS, rv, "Add failed.");
1048    check_key_value(h, h1, key, data, strlen(data));
1049    h1->release(h, NULL, it);
1050    wait_for_flusher_to_settle(h, h1);
1051
1052    checkeq(ENGINE_SUCCESS,
1053            h1->get(h, NULL, &it, key, strlen(key), 0),
1054            "Item shouldn't expire");
1055    h1->release(h, NULL, it);
1056
1057    // Restart the engine to ensure the above unexpired new item is loaded
1058    testHarness.reload_engine(&h, &h1,
1059                              testHarness.engine_path,
1060                              testHarness.get_current_testcase()->cfg,
1061                              true, false);
1062    wait_for_warmup_complete(h, h1);
1063    cb_assert(1 == get_int_stat(h, h1, "ep_warmup_value_count", "warmup"));
1064    cb_assert(0 == get_int_stat(h, h1, "ep_warmup_dups", "warmup"));
1065
1066    return SUCCESS;
1067}
1068
1069static enum test_result test_bug3522(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1070    const char *key = "test_expiry_no_items_warmup";
1071    const char *data = "some test data here.";
1072
1073    item *it = NULL;
1074
1075    ENGINE_ERROR_CODE rv;
1076    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 0,
1077                      PROTOCOL_BINARY_RAW_BYTES);
1078    checkeq(ENGINE_SUCCESS, rv, "Allocation failed.");
1079
1080    item_info info;
1081    info.nvalue = 1;
1082    if (!h1->get_item_info(h, NULL, it, &info)) {
1083        abort();
1084    }
1085    memcpy(info.value[0].iov_base, data, strlen(data));
1086
1087    uint64_t cas = 0;
1088    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
1089    checkeq(ENGINE_SUCCESS, rv, "Set failed.");
1090    check_key_value(h, h1, key, data, strlen(data));
1091    h1->release(h, NULL, it);
1092    wait_for_flusher_to_settle(h, h1);
1093
1094    // Add a new item with the same key and 2 sec of expiration.
1095    const char *new_data = "new data here.";
1096    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(new_data), 0, 2,
1097                      PROTOCOL_BINARY_RAW_BYTES);
1098    checkeq(ENGINE_SUCCESS, rv, "Allocation failed.");
1099
1100    info.nvalue = 1;
1101    if (!h1->get_item_info(h, NULL, it, &info)) {
1102        abort();
1103    }
1104    memcpy(info.value[0].iov_base, new_data, strlen(new_data));
1105
1106    int pager_runs = get_int_stat(h, h1, "ep_num_expiry_pager_runs");
1107    cas = 0;
1108    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
1109    checkeq(ENGINE_SUCCESS, rv, "Set failed.");
1110    check_key_value(h, h1, key, new_data, strlen(new_data));
1111    h1->release(h, NULL, it);
1112    testHarness.time_travel(3);
1113    wait_for_stat_change(h, h1, "ep_num_expiry_pager_runs", pager_runs);
1114    wait_for_flusher_to_settle(h, h1);
1115
1116    // Restart the engine.
1117    testHarness.reload_engine(&h, &h1,
1118                              testHarness.engine_path,
1119                              testHarness.get_current_testcase()->cfg,
1120                              true, false);
1121    wait_for_warmup_complete(h, h1);
1122    // TODO: modify this for a better test case
1123    cb_assert(0 == get_int_stat(h, h1, "ep_warmup_dups", "warmup"));
1124
1125    return SUCCESS;
1126}
1127
1128static enum test_result test_get_replica_active_state(ENGINE_HANDLE *h,
1129                                                      ENGINE_HANDLE_V1 *h1) {
1130    protocol_binary_request_header *pkt;
1131    pkt = prepare_get_replica(h, h1, vbucket_state_active);
1132    checkeq(ENGINE_SUCCESS,
1133            h1->unknown_command(h, NULL, pkt, add_response),
1134            "Get Replica Failed");
1135    checkeq(PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET, last_status.load(),
1136            "Expected PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET response.");
1137
1138    cb_free(pkt);
1139    return SUCCESS;
1140}
1141
1142static enum test_result test_get_replica_pending_state(ENGINE_HANDLE *h,
1143                                                       ENGINE_HANDLE_V1 *h1) {
1144    protocol_binary_request_header *pkt;
1145
1146    const void *cookie = testHarness.create_cookie();
1147    testHarness.set_ewouldblock_handling(cookie, false);
1148    pkt = prepare_get_replica(h, h1, vbucket_state_pending);
1149    checkeq(ENGINE_EWOULDBLOCK,
1150            h1->unknown_command(h, cookie, pkt, add_response),
1151            "Should have returned error for pending state");
1152    testHarness.destroy_cookie(cookie);
1153    cb_free(pkt);
1154    return SUCCESS;
1155}
1156
1157static enum test_result test_get_replica_dead_state(ENGINE_HANDLE *h,
1158                                                    ENGINE_HANDLE_V1 *h1) {
1159    protocol_binary_request_header *pkt;
1160    pkt = prepare_get_replica(h, h1, vbucket_state_dead);
1161    checkeq(ENGINE_SUCCESS,
1162            h1->unknown_command(h, NULL, pkt, add_response),
1163            "Get Replica Failed");
1164    checkeq(PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET, last_status.load(),
1165            "Expected PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET response.");
1166
1167    cb_free(pkt);
1168    return SUCCESS;
1169}
1170
1171static enum test_result test_get_replica(ENGINE_HANDLE *h,
1172                                         ENGINE_HANDLE_V1 *h1) {
1173    protocol_binary_request_header *pkt;
1174    pkt = prepare_get_replica(h, h1, vbucket_state_replica);
1175    checkeq(ENGINE_SUCCESS,
1176            h1->unknown_command(h, NULL, pkt, add_response),
1177            "Get Replica Failed");
1178    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1179            "Expected PROTOCOL_BINARY_RESPONSE_SUCCESS response.");
1180    checkeq(std::string("replicadata"), last_body,
1181            "Should have returned identical value");
1182
1183    cb_free(pkt);
1184    return SUCCESS;
1185}
1186
1187static enum test_result test_get_replica_non_resident(ENGINE_HANDLE *h,
1188                                                      ENGINE_HANDLE_V1 *h1) {
1189
1190    item *i = NULL;
1191    checkeq(ENGINE_SUCCESS,
1192            store(h, h1, NULL, OPERATION_SET, "key", "value", &i, 0, 0),
1193            "Store Failed");
1194    h1->release(h, NULL, i);
1195    wait_for_flusher_to_settle(h, h1);
1196    wait_for_stat_to_be(h, h1, "ep_total_persisted", 1);
1197
1198    evict_key(h, h1, "key", 0, "Ejected.");
1199    check(set_vbucket_state(h, h1, 0, vbucket_state_replica),
1200          "Failed to set vbucket to replica");
1201
1202    get_replica(h, h1, "key", 0);
1203    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1204            "Expected success");
1205
1206    return SUCCESS;
1207}
1208
1209static enum test_result test_get_replica_invalid_key(ENGINE_HANDLE *h,
1210                                                     ENGINE_HANDLE_V1 *h1) {
1211    protocol_binary_request_header *pkt;
1212    bool makeinvalidkey = true;
1213    pkt = prepare_get_replica(h, h1, vbucket_state_replica, makeinvalidkey);
1214    checkeq(ENGINE_SUCCESS,
1215            h1->unknown_command(h, NULL, pkt, add_response),
1216            "Get Replica Failed");
1217    checkeq(PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET, last_status.load(),
1218            "Expected PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET response.");
1219    cb_free(pkt);
1220    return SUCCESS;
1221}
1222
1223static enum test_result test_vb_del_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1224    const void *cookie = testHarness.create_cookie();
1225    testHarness.set_ewouldblock_handling(cookie, false);
1226    check(set_vbucket_state(h, h1, 1, vbucket_state_pending),
1227          "Failed to set vbucket state.");
1228    checkeq(ENGINE_EWOULDBLOCK, del(h, h1, "key", 0, 1, cookie),
1229            "Expected woodblock.");
1230    testHarness.destroy_cookie(cookie);
1231    return SUCCESS;
1232}
1233
1234static enum test_result test_vb_del_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1235    check(set_vbucket_state(h, h1, 1, vbucket_state_replica),
1236          "Failed to set vbucket state.");
1237    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1238    checkeq(ENGINE_NOT_MY_VBUCKET, del(h, h1, "key", 0, 1),
1239            "Expected not my vbucket.");
1240    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1241    return SUCCESS;
1242}
1243
1244static enum test_result test_vbucket_get_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1245    return verify_vbucket_missing(h, h1, 1) ? SUCCESS : FAIL;
1246}
1247
1248static enum test_result test_vbucket_get(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1249    return verify_vbucket_state(h, h1, 0, vbucket_state_active) ? SUCCESS : FAIL;
1250}
1251
1252static enum test_result test_vbucket_create(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1253    if (!verify_vbucket_missing(h, h1, 1)) {
1254        fprintf(stderr, "vbucket wasn't missing.\n");
1255        return FAIL;
1256    }
1257
1258    if (!set_vbucket_state(h, h1, 1, vbucket_state_active)) {
1259        fprintf(stderr, "set state failed.\n");
1260        return FAIL;
1261    }
1262
1263    return verify_vbucket_state(h, h1, 1, vbucket_state_active) ? SUCCESS : FAIL;
1264}
1265
1266static enum test_result test_takeover_stats_race_with_vb_create(ENGINE_HANDLE *h,
1267                                                                ENGINE_HANDLE_V1 *h1) {
1268    check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1269          "Failed to set vbucket state information");
1270
1271    int del_items = get_int_stat(h, h1, "on_disk_deletes", "dcp-vbtakeover 1");
1272
1273    checkeq(0, del_items, "Invalid number of on-disk deletes");
1274
1275    del_items = get_int_stat(h, h1, "on_disk_deletes", "tap-vbtakeover 1");
1276
1277    checkeq(0, del_items, "Invalid number of on-disk deletes");
1278
1279    return SUCCESS;
1280}
1281
1282static enum test_result test_vbucket_compact(ENGINE_HANDLE *h,
1283                                             ENGINE_HANDLE_V1 *h1) {
1284    const char *key = "Carss";
1285    const char *value = "pollute";
1286    if (!verify_vbucket_missing(h, h1, 0)) {
1287        fprintf(stderr, "vbucket wasn't missing.\n");
1288        return FAIL;
1289    }
1290
1291    if (!set_vbucket_state(h, h1, 0, vbucket_state_active)) {
1292        fprintf(stderr, "set state failed.\n");
1293        return FAIL;
1294    }
1295
1296    check(verify_vbucket_state(h, h1, 0, vbucket_state_active),
1297            "VBucket state not active");
1298
1299    // Set two keys - one to be expired and other to remain...
1300    item *itm = NULL;
1301    checkeq(ENGINE_SUCCESS,
1302            store(h, h1, NULL, OPERATION_SET, key, value, &itm),
1303            "Failed set.");
1304    h1->release(h, NULL, itm);
1305
1306    check_key_value(h, h1, key, value, strlen(value));
1307
1308    // Set a non-expiring key...
1309    checkeq(ENGINE_SUCCESS,
1310            store(h, h1, NULL, OPERATION_SET, "trees", "cleanse", &itm),
1311            "Failed set.");
1312    h1->release(h, NULL, itm);
1313
1314    check_key_value(h, h1, "trees", "cleanse", strlen("cleanse"));
1315
1316    touch(h, h1, key, 0, 11);
1317    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1318            "touch Carss");
1319
1320    testHarness.time_travel(12);
1321    wait_for_flusher_to_settle(h, h1);
1322
1323    // Store a dummy item since we do not purge the item with highest seqno
1324    checkeq(ENGINE_SUCCESS,
1325            store(h, h1, NULL, OPERATION_SET, "dummykey", "dummyvalue", &itm,
1326                  0, 0, 0),
1327            "Error setting.");
1328    h1->release(h, NULL, itm);
1329
1330    wait_for_flusher_to_settle(h, h1);
1331
1332    checkeq(0, get_int_stat(h, h1, "vb_0:purge_seqno", "vbucket-seqno"),
1333            "purge_seqno not found to be zero before compaction");
1334
1335    // Compaction on VBucket
1336    compact_db(h, h1, 0, 2, 3, 1);
1337
1338    wait_for_stat_to_be(h, h1, "ep_pending_compactions", 0);
1339
1340    // the key tree and its value should be intact...
1341    checkeq(ENGINE_SUCCESS, verify_key(h, h1, "trees"),
1342            "key trees should be found.");
1343    // the key Carrs should have disappeared...
1344    ENGINE_ERROR_CODE val = verify_key(h, h1, "Carss");
1345    checkeq(ENGINE_KEY_ENOENT, val, "Key Carss has not expired.");
1346
1347    checkeq(4, get_int_stat(h, h1, "vb_0:purge_seqno", "vbucket-seqno"),
1348            "purge_seqno didn't match expected value");
1349
1350    return SUCCESS;
1351}
1352
1353static enum test_result test_compaction_config(ENGINE_HANDLE *h,
1354                                               ENGINE_HANDLE_V1 *h1) {
1355
1356    checkeq(10000,
1357            get_int_stat(h, h1, "ep_compaction_write_queue_cap"),
1358            "Expected compaction queue cap to be 10000");
1359    set_param(h, h1, protocol_binary_engine_param_flush,
1360              "compaction_write_queue_cap", "100000");
1361    checkeq(100000, get_int_stat(h, h1, "ep_compaction_write_queue_cap"),
1362            "Expected compaction queue cap to be 100000");
1363    return SUCCESS;
1364}
1365
1366struct comp_thread_ctx {
1367    ENGINE_HANDLE *h;
1368    ENGINE_HANDLE_V1 *h1;
1369    uint16_t vbid;
1370};
1371
1372extern "C" {
1373    static void compaction_thread(void *arg) {
1374        struct comp_thread_ctx *ctx = static_cast<comp_thread_ctx *>(arg);
1375        compact_db(ctx->h, ctx->h1, ctx->vbid, 0, 0, 0);
1376    }
1377}
1378
1379static enum test_result test_multiple_vb_compactions(ENGINE_HANDLE *h,
1380                                                     ENGINE_HANDLE_V1 *h1) {
1381    for (uint16_t i = 0; i < 4; ++i) {
1382        if (!set_vbucket_state(h, h1, i, vbucket_state_active)) {
1383            fprintf(stderr, "set state failed for vbucket %d.\n", i);
1384            return FAIL;
1385        }
1386        check(verify_vbucket_state(h, h1, i, vbucket_state_active),
1387              "VBucket state not active");
1388    }
1389
1390    std::vector<std::string> keys;
1391    for (int j = 0; j < 20000; ++j) {
1392        std::stringstream ss;
1393        ss << "key" << j;
1394        std::string key(ss.str());
1395        keys.push_back(key);
1396    }
1397
1398    int count = 0;
1399    std::vector<std::string>::iterator it;
1400    for (it = keys.begin(); it != keys.end(); ++it) {
1401        uint16_t vbid = count % 4;
1402        item *i;
1403        checkeq(ENGINE_SUCCESS,
1404                store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str(), &i, 0, vbid),
1405                "Failed to store a value");
1406        h1->release(h, NULL, i);
1407        ++count;
1408    }
1409
1410    // Compact multiple vbuckets.
1411    const int n_threads = 4;
1412    cb_thread_t threads[n_threads];
1413    struct comp_thread_ctx ctx[n_threads];
1414
1415    for (int i = 0; i < n_threads; i++) {
1416        ctx[i].h = h;
1417        ctx[i].h1 = h1;
1418        ctx[i].vbid = static_cast<uint16_t>(i);
1419        int r = cb_create_thread(&threads[i], compaction_thread, &ctx[i], 0);
1420        cb_assert(r == 0);
1421    }
1422
1423    for (int i = 0; i < n_threads; i++) {
1424        int r = cb_join_thread(threads[i]);
1425        cb_assert(r == 0);
1426    }
1427
1428    wait_for_stat_to_be(h, h1, "ep_pending_compactions", 0);
1429
1430    return SUCCESS;
1431}
1432
1433static enum test_result
1434test_multi_vb_compactions_with_workload(ENGINE_HANDLE *h,
1435                                        ENGINE_HANDLE_V1 *h1) {
1436    for (uint16_t i = 0; i < 4; ++i) {
1437        if (!set_vbucket_state(h, h1, i, vbucket_state_active)) {
1438            fprintf(stderr, "set state failed for vbucket %d.\n", i);
1439            return FAIL;
1440        }
1441        check(verify_vbucket_state(h, h1, i, vbucket_state_active),
1442              "VBucket state not active");
1443    }
1444
1445    std::vector<std::string> keys;
1446    for (int j = 0; j < 10000; ++j) {
1447        std::stringstream ss;
1448        ss << "key" << j;
1449        std::string key(ss.str());
1450        keys.push_back(key);
1451    }
1452
1453    int count = 0;
1454    std::vector<std::string>::iterator it;
1455    for (it = keys.begin(); it != keys.end(); ++it) {
1456        uint16_t vbid = count % 4;
1457        item *i;
1458        checkeq(ENGINE_SUCCESS,
1459                store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str(),
1460                      &i, 0, vbid),
1461                "Failed to store a value");
1462        h1->release(h, NULL, i);
1463        ++count;
1464    }
1465    wait_for_flusher_to_settle(h, h1);
1466
1467    for (int i = 0; i < 2; ++i) {
1468        count = 0;
1469        for (it = keys.begin(); it != keys.end(); ++it) {
1470            uint16_t vbid = count % 4;
1471            item *i = NULL;
1472            checkeq(ENGINE_SUCCESS,
1473                    h1->get(h, NULL, &i, it->c_str(), strlen(it->c_str()), vbid),
1474                    "Unable to get stored item");
1475            h1->release(h, NULL, i);
1476            ++count;
1477        }
1478    }
1479    wait_for_stat_to_be(h, h1, "ep_workload_pattern", std::string{"read_heavy"});
1480
1481    // Compact multiple vbuckets.
1482    const int n_threads = 4;
1483    cb_thread_t threads[n_threads];
1484    struct comp_thread_ctx ctx[n_threads];
1485
1486    for (int i = 0; i < n_threads; i++) {
1487        ctx[i].h = h;
1488        ctx[i].h1 = h1;
1489        ctx[i].vbid = static_cast<uint16_t>(i);
1490        int r = cb_create_thread(&threads[i], compaction_thread, &ctx[i], 0);
1491        cb_assert(r == 0);
1492    }
1493
1494    for (int i = 0; i < n_threads; i++) {
1495        int r = cb_join_thread(threads[i]);
1496        cb_assert(r == 0);
1497    }
1498
1499    wait_for_stat_to_be(h, h1, "ep_pending_compactions", 0);
1500
1501    return SUCCESS;
1502}
1503
1504static enum test_result vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1505                                             const char* value = NULL) {
1506    check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1507          "Failed to set vbucket state.");
1508
1509    vbucketDelete(h, h1, 2, value);
1510    checkeq(PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
1511            last_status.load(),
1512            "Expected failure deleting non-existent bucket.");
1513
1514    check(set_vbucket_state(h, h1, 1, vbucket_state_dead),
1515          "Failed set set vbucket 1 state.");
1516
1517    vbucketDelete(h, h1, 1, value);
1518    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1519            "Expected failure deleting non-existent bucket.");
1520
1521    check(verify_vbucket_missing(h, h1, 1),
1522          "vbucket 0 was not missing after deleting it.");
1523
1524    return SUCCESS;
1525}
1526
1527static enum test_result test_vbucket_destroy_stats(ENGINE_HANDLE *h,
1528                                                   ENGINE_HANDLE_V1 *h1) {
1529
1530    int cacheSize = get_int_stat(h, h1, "ep_total_cache_size");
1531    int overhead = get_int_stat(h, h1, "ep_overhead");
1532    int nonResident = get_int_stat(h, h1, "ep_num_non_resident");
1533
1534    check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1535          "Failed to set vbucket state.");
1536
1537    std::vector<std::string> keys;
1538    for (int j = 0; j < 2000; ++j) {
1539        std::stringstream ss;
1540        ss << "key" << j;
1541        std::string key(ss.str());
1542        keys.push_back(key);
1543    }
1544
1545    int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
1546    std::vector<std::string>::iterator it;
1547    for (it = keys.begin(); it != keys.end(); ++it) {
1548        item *i;
1549        checkeq(ENGINE_SUCCESS,
1550                store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str(),
1551                      &i, 0, 1),
1552                "Failed to store a value");
1553        h1->release(h, NULL, i);
1554    }
1555    wait_for_flusher_to_settle(h, h1);
1556    testHarness.time_travel(65);
1557    wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
1558
1559    check(set_vbucket_state(h, h1, 1, vbucket_state_dead),
1560          "Failed set set vbucket 1 state.");
1561
1562    int vbucketDel = get_int_stat(h, h1, "ep_vbucket_del");
1563    vbucketDelete(h, h1, 1);
1564    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS,
1565            last_status.load(),
1566            "Expected failure deleting non-existent bucket.");
1567
1568    check(verify_vbucket_missing(h, h1, 1),
1569          "vbucket 1 was not missing after deleting it.");
1570
1571    wait_for_stat_change(h, h1, "ep_vbucket_del", vbucketDel);
1572
1573    wait_for_stat_to_be(h, h1, "ep_total_cache_size", cacheSize);
1574    wait_for_stat_to_be(h, h1, "ep_overhead", overhead);
1575    wait_for_stat_to_be(h, h1, "ep_num_non_resident", nonResident);
1576
1577    return SUCCESS;
1578}
1579
1580static enum test_result vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1581                                                const char* value = NULL) {
1582    check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1583          "Failed to set vbucket state.");
1584
1585    // Store a value so the restart will try to resurrect it.
1586    item *i = NULL;
1587    checkeq(ENGINE_SUCCESS,
1588            store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i, 0, 1),
1589            "Failed to set a value");
1590    check_key_value(h, h1, "key", "somevalue", 9, 1);
1591    h1->release(h, NULL, i);
1592
1593    // Reload to get a flush forced.
1594    testHarness.reload_engine(&h, &h1,
1595                              testHarness.engine_path,
1596                              testHarness.get_current_testcase()->cfg,
1597                              true, false);
1598    wait_for_warmup_complete(h, h1);
1599
1600    check(verify_vbucket_state(h, h1, 1, vbucket_state_active),
1601          "Bucket state was what it was initially, after restart.");
1602    check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1603          "Failed to set vbucket state.");
1604    check_key_value(h, h1, "key", "somevalue", 9, 1);
1605
1606    check(set_vbucket_state(h, h1, 1, vbucket_state_dead),
1607          "Failed set set vbucket 1 state.");
1608
1609    vbucketDelete(h, h1, 1, value);
1610    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1611            "Expected failure deleting non-existent bucket.");
1612
1613    check(verify_vbucket_missing(h, h1, 1),
1614          "vbucket 1 was not missing after deleting it.");
1615
1616    testHarness.reload_engine(&h, &h1,
1617                              testHarness.engine_path,
1618                              testHarness.get_current_testcase()->cfg,
1619                              true, false);
1620    wait_for_warmup_complete(h, h1);
1621
1622    if (verify_vbucket_state(h, h1, 1, vbucket_state_pending, true)) {
1623        std::cerr << "Bucket came up in pending state after delete." << std::endl;
1624        abort();
1625    }
1626
1627    check(verify_vbucket_missing(h, h1, 1),
1628          "vbucket 1 was not missing after restart.");
1629
1630    return SUCCESS;
1631}
1632
1633static enum test_result test_async_vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1634    return vbucket_destroy(h, h1);
1635}
1636
1637static enum test_result test_sync_vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1638    return vbucket_destroy(h, h1, "async=0");
1639}
1640
1641static enum test_result test_async_vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1642    return vbucket_destroy_restart(h, h1);
1643}
1644
1645static enum test_result test_sync_vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1646    return vbucket_destroy_restart(h, h1, "async=0");
1647}
1648
1649static enum test_result test_vb_set_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1650    return test_pending_vb_mutation(h, h1, OPERATION_SET);
1651}
1652
1653static enum test_result test_vb_add_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1654    return test_pending_vb_mutation(h, h1, OPERATION_ADD);
1655}
1656
1657static enum test_result test_vb_cas_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1658    return test_pending_vb_mutation(h, h1, OPERATION_CAS);
1659}
1660
1661static enum test_result test_vb_append_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1662    return test_pending_vb_mutation(h, h1, OPERATION_APPEND);
1663}
1664
1665static enum test_result test_vb_prepend_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1666    return test_pending_vb_mutation(h, h1, OPERATION_PREPEND);
1667}
1668
1669static enum test_result test_vb_set_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1670    return test_replica_vb_mutation(h, h1, OPERATION_SET);
1671}
1672
1673static enum test_result test_vb_replace_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1674    return test_replica_vb_mutation(h, h1, OPERATION_REPLACE);
1675}
1676
1677static enum test_result test_vb_replace_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1678    return test_pending_vb_mutation(h, h1, OPERATION_REPLACE);
1679}
1680
1681static enum test_result test_vb_add_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1682    return test_replica_vb_mutation(h, h1, OPERATION_ADD);
1683}
1684
1685static enum test_result test_vb_cas_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1686    return test_replica_vb_mutation(h, h1, OPERATION_CAS);
1687}
1688
1689static enum test_result test_vb_append_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1690    return test_replica_vb_mutation(h, h1, OPERATION_APPEND);
1691}
1692
1693static enum test_result test_vb_prepend_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1694    return test_replica_vb_mutation(h, h1, OPERATION_PREPEND);
1695}
1696
1697static enum test_result test_stats_seqno(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1698    check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1699          "Failed to set vbucket state.");
1700
1701    int num_keys = 100;
1702    for (int ii = 0; ii < num_keys; ++ii) {
1703        std::stringstream ss;
1704        ss << "key" << ii;
1705        checkeq(ENGINE_SUCCESS,
1706                store(h, h1, NULL, OPERATION_SET, ss.str().c_str(),
1707                      "value", NULL, 0, 0),
1708                "Failed to store an item.");
1709    }
1710    wait_for_flusher_to_settle(h, h1);
1711
1712    checkeq(100, get_int_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno"),
1713            "Invalid seqno");
1714    checkeq(100, get_int_stat(h, h1, "vb_0:last_persisted_seqno", "vbucket-seqno"),
1715            "Unexpected last_persisted_seqno");
1716    checkeq(0, get_int_stat(h, h1, "vb_1:high_seqno", "vbucket-seqno"),
1717            "Invalid seqno");
1718    checkeq(0, get_int_stat(h, h1, "vb_1:high_seqno", "vbucket-seqno 1"),
1719            "Invalid seqno");
1720    checkeq(0, get_int_stat(h, h1, "vb_1:last_persisted_seqno", "vbucket-seqno 1"),
1721            "Invalid last_persisted_seqno");
1722
1723    uint64_t vb_uuid = get_ull_stat(h, h1, "vb_1:0:id", "failovers");
1724
1725    auto seqno_stats = get_all_stats(h, h1, "vbucket-seqno 1");
1726    checkeq(vb_uuid, uint64_t(std::stoull(seqno_stats.at("vb_1:uuid"))),
1727            "Invalid uuid");
1728
1729    checkeq(size_t(7), seqno_stats.size(), "Expected seven stats");
1730
1731    // Check invalid vbucket
1732    checkeq(ENGINE_NOT_MY_VBUCKET,
1733            h1->get_stats(h, NULL, "vbucket-seqno 2", 15, add_stats),
1734            "Expected not my vbucket");
1735
1736    // Check bad vbucket parameter (not numeric)
1737    checkeq(ENGINE_EINVAL,
1738            h1->get_stats(h, NULL, "vbucket-seqno tt2", 17, add_stats),
1739            "Expected invalid");
1740
1741    // Check extra spaces at the end
1742    checkeq(ENGINE_EINVAL,
1743            h1->get_stats(h, NULL, "vbucket-seqno    ", 17, add_stats),
1744            "Expected invalid");
1745
1746    return SUCCESS;
1747}
1748
1749static enum test_result test_stats_diskinfo(ENGINE_HANDLE *h,
1750                                            ENGINE_HANDLE_V1 *h1) {
1751    check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1752          "Failed to set vbucket state.");
1753
1754    int num_keys = 100;
1755    for (int ii = 0; ii < num_keys; ++ii) {
1756        std::stringstream ss;
1757        ss << "key" << ii;
1758        checkeq(ENGINE_SUCCESS,
1759                store(h, h1, NULL, OPERATION_SET, ss.str().c_str(),
1760                      "value", NULL, 0, 1),
1761                "Failed to store an item.");
1762    }
1763    wait_for_flusher_to_settle(h, h1);
1764
1765    size_t file_size = get_int_stat(h, h1, "ep_db_file_size", "diskinfo");
1766    size_t data_size = get_int_stat(h, h1, "ep_db_data_size", "diskinfo");
1767    check(file_size > 0, "DB file size should be greater than 0");
1768    check(data_size > 0, "DB data size should be greater than 0");
1769    check(file_size >= data_size, "DB file size should be >= DB data size");
1770    check(get_int_stat(h, h1, "vb_1:data_size", "diskinfo detail") > 0,
1771          "VB 1 data size should be greater than 0");
1772
1773    checkeq(ENGINE_EINVAL,
1774            h1->get_stats(h, NULL, "diskinfo ", 9, add_stats),
1775            "Expected invalid");
1776
1777    checkeq(ENGINE_EINVAL,
1778            h1->get_stats(h, NULL, "diskinfo detai", 14, add_stats),
1779            "Expected invalid");
1780
1781    checkeq(ENGINE_EINVAL,
1782            h1->get_stats(h, NULL, "diskinfo detaillll", 18, add_stats),
1783            "Expected invalid");
1784
1785    return SUCCESS;
1786}
1787
1788static enum test_result test_uuid_stats(ENGINE_HANDLE *h,
1789                                        ENGINE_HANDLE_V1 *h1)
1790{
1791    vals.clear();
1792    checkeq(ENGINE_SUCCESS,
1793            h1->get_stats(h, NULL, "uuid", 4, add_stats),
1794            "Failed to get stats.");
1795    check(vals["uuid"] == "foobar", "Incorrect uuid");
1796    return SUCCESS;
1797}
1798
1799static enum test_result test_item_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1800    item *i = NULL;
1801    checkeq(ENGINE_SUCCESS,
1802            store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i, 0, 0),
1803            "Failed set.");
1804    h1->release(h, NULL, i);
1805    wait_for_flusher_to_settle(h, h1);
1806    checkeq(ENGINE_SUCCESS,
1807            store(h, h1, NULL, OPERATION_SET, "key", "somevalueX", &i, 0, 0),
1808            "Failed set.");
1809    h1->release(h, NULL, i);
1810    wait_for_flusher_to_settle(h, h1);
1811    checkeq(ENGINE_SUCCESS,
1812            store(h, h1, NULL, OPERATION_SET, "key1", "somevalueY", &i, 0, 0),
1813            "Failed set.");
1814    h1->release(h, NULL, i);
1815    wait_for_flusher_to_settle(h, h1);
1816
1817    check_key_value(h, h1, "key", "somevalueX", 10);
1818    check_key_value(h, h1, "key1", "somevalueY", 10);
1819
1820    checkeq(ENGINE_SUCCESS, del(h, h1, "key1", 0, 0),
1821            "Failed remove with value.");
1822    wait_for_flusher_to_settle(h, h1);
1823
1824    checkeq(ENGINE_SUCCESS,
1825            store(h, h1, NULL, OPERATION_SET, "key1", "someothervalue", &i, 0, 0),
1826            "Failed set.");
1827    h1->release(h, NULL, i);
1828    wait_for_flusher_to_settle(h, h1);
1829
1830    check_key_value(h, h1, "key1", "someothervalue", 14);
1831
1832    checkeq(3, get_int_stat(h, h1, "vb_active_ops_create"),
1833            "Expected 3 creations");
1834    checkeq(1, get_int_stat(h, h1, "vb_active_ops_update"),
1835            "Expected 1 updation");
1836    checkeq(1, get_int_stat(h, h1, "vb_active_ops_delete"),
1837            "Expected 1 deletion");
1838
1839    return SUCCESS;
1840}
1841
1842static enum test_result test_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1843    vals.clear();
1844    checkeq(ENGINE_SUCCESS,
1845            h1->get_stats(h, NULL, NULL, 0, add_stats),
1846            "Failed to get stats.");
1847    check(vals.size() > 10, "Kind of expected more stats than that.");
1848    check(vals.find("ep_version") != vals.end(), "Found no ep_version.");
1849
1850    return SUCCESS;
1851}
1852
1853static enum test_result test_mem_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1854    char value[2048];
1855    memset(value, 'b', sizeof(value));
1856    strcpy(value + sizeof(value) - 4, "\r\n");
1857    int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
1858    wait_for_persisted_value(h, h1, "key", value);
1859    testHarness.time_travel(65);
1860    wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
1861    int mem_used = get_int_stat(h, h1, "mem_used");
1862    int cache_size = get_int_stat(h, h1, "ep_total_cache_size");
1863    int overhead = get_int_stat(h, h1, "ep_overhead");
1864    int value_size = get_int_stat(h, h1, "ep_value_size");
1865    check((mem_used - overhead) > cache_size,
1866          "ep_kv_size should be greater than the hashtable cache size due to the checkpoint overhead");
1867    evict_key(h, h1, "key", 0, "Ejected.");
1868
1869    check(get_int_stat(h, h1, "ep_total_cache_size") <= cache_size,
1870          "Evict a value shouldn't increase the total cache size");
1871    check(get_int_stat(h, h1, "mem_used") < mem_used,
1872          "Expected mem_used to decrease when an item is evicted");
1873
1874    check_key_value(h, h1, "key", value, strlen(value), 0); // Load an item from disk again.
1875
1876    check(get_int_stat(h, h1, "mem_used") >= mem_used,
1877          "Expected mem_used to remain the same after an item is loaded from disk");
1878    check(get_int_stat(h, h1, "ep_value_size") == value_size,
1879          "Expected ep_value_size to remain the same after item is loaded from disk");
1880
1881    return SUCCESS;
1882}
1883
1884static enum test_result test_io_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1885    h1->reset_stats(h, NULL);
1886
1887    check(get_int_stat(h, h1, "rw_0:io_num_read", "kvstore") == 0,
1888          "Expected reset stats to set io_num_read to zero");
1889    check(get_int_stat(h, h1, "rw_0:io_num_write", "kvstore") == 0,
1890          "Expected reset stats to set io_num_write to zero");
1891    check(get_int_stat(h, h1, "rw_0:io_read_bytes", "kvstore") == 0,
1892          "Expected reset stats to set io_read_bytes to zero");
1893    check(get_int_stat(h, h1, "rw_0:io_write_bytes", "kvstore") == 0,
1894          "Expected reset stats to set io_write_bytes to zero");
1895    wait_for_persisted_value(h, h1, "a", "b\r\n");
1896    check(get_int_stat(h, h1, "rw_0:io_num_read", "kvstore") == 0 &&
1897          get_int_stat(h, h1, "rw_0:io_read_bytes", "kvstore") == 0,
1898          "Expected storing one value to not change the read counter");
1899
1900    check(get_int_stat(h, h1, "rw_0:io_num_write", "kvstore") == 1 &&
1901          get_int_stat(h, h1, "rw_0:io_write_bytes", "kvstore") == 22,
1902          "Expected storing the key to update the write counter");
1903    evict_key(h, h1, "a", 0, "Ejected.");
1904
1905    check_key_value(h, h1, "a", "b\r\n", 3, 0);
1906
1907    check(get_int_stat(h, h1, "ro_0:io_num_read", "kvstore") == 1 &&
1908          get_int_stat(h, h1, "ro_0:io_read_bytes", "kvstore") == 4,
1909          "Expected reading the value back in to update the read counter");
1910    check(get_int_stat(h, h1, "rw_0:io_num_write", "kvstore") == 1 &&
1911          get_int_stat(h, h1, "rw_0:io_write_bytes", "kvstore") == 22,
1912          "Expected reading the value back in to not update the write counter");
1913
1914    return SUCCESS;
1915}
1916
1917static enum test_result test_vb_file_stats(ENGINE_HANDLE *h,
1918                                        ENGINE_HANDLE_V1 *h1) {
1919    wait_for_flusher_to_settle(h, h1);
1920    wait_for_stat_change(h, h1, "ep_db_data_size", 0);
1921
1922    int old_data_size = get_int_stat(h, h1, "ep_db_data_size");
1923    int old_file_size = get_int_stat(h, h1, "ep_db_file_size");
1924    check(old_file_size != 0, "Expected a non-zero value for ep_db_file_size");
1925
1926    // Write a value and test ...
1927    wait_for_persisted_value(h, h1, "a", "b\r\n");
1928    check(get_int_stat(h, h1, "ep_db_data_size") > old_data_size,
1929          "Expected the DB data size to increase");
1930    check(get_int_stat(h, h1, "ep_db_file_size") > old_file_size,
1931          "Expected the DB file size to increase");
1932
1933    check(get_int_stat(h, h1, "vb_0:db_data_size", "vbucket-details 0") > 0,
1934          "Expected the vbucket DB data size to non-zero");
1935    check(get_int_stat(h, h1, "vb_0:db_file_size", "vbucket-details 0") > 0,
1936          "Expected the vbucket DB file size to non-zero");
1937    return SUCCESS;
1938}
1939
1940static enum test_result test_vb_file_stats_after_warmup(ENGINE_HANDLE *h,
1941                                                        ENGINE_HANDLE_V1 *h1) {
1942
1943    item *it = NULL;
1944    for (int i = 0; i < 100; ++i) {
1945        std::stringstream key;
1946        key << "key-" << i;
1947        checkeq(ENGINE_SUCCESS,
1948                store(h, h1, NULL, OPERATION_SET, key.str().c_str(), "somevalue", &it),
1949                "Error setting.");
1950        h1->release(h, NULL, it);
1951    }
1952    wait_for_flusher_to_settle(h, h1);
1953
1954    int fileSize = get_int_stat(h, h1, "vb_0:db_file_size", "vbucket-details 0");
1955    int spaceUsed = get_int_stat(h, h1, "vb_0:db_data_size", "vbucket-details 0");
1956
1957    // Restart the engine.
1958    testHarness.reload_engine(&h, &h1,
1959                              testHarness.engine_path,
1960                              testHarness.get_current_testcase()->cfg,
1961                              true, false);
1962    wait_for_warmup_complete(h, h1);
1963
1964    int newFileSize = get_int_stat(h, h1, "vb_0:db_file_size", "vbucket-details 0");
1965    int newSpaceUsed = get_int_stat(h, h1, "vb_0:db_data_size", "vbucket-details 0");
1966
1967    check((float)newFileSize >= 0.9 * fileSize, "Unexpected fileSize for vbucket");
1968    check((float)newSpaceUsed >= 0.9 * spaceUsed, "Unexpected spaceUsed for vbucket");
1969
1970    return SUCCESS;
1971}
1972
1973static enum test_result test_bg_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1974    h1->reset_stats(h, NULL);
1975    wait_for_persisted_value(h, h1, "a", "b\r\n");
1976    evict_key(h, h1, "a", 0, "Ejected.");
1977    testHarness.time_travel(43);
1978    check_key_value(h, h1, "a", "b\r\n", 3, 0);
1979
1980    auto stats = get_all_stats(h, h1);
1981    checkeq(1, std::stoi(stats.at("ep_bg_num_samples")),
1982               "Expected one sample");
1983
1984    const char* bg_keys[] = { "ep_bg_min_wait",
1985                              "ep_bg_max_wait",
1986                              "ep_bg_wait_avg",
1987                              "ep_bg_min_load",
1988                              "ep_bg_max_load",
1989                              "ep_bg_load_avg"};
1990    for (const auto* key : bg_keys) {
1991        check(stats.find(key) != stats.end(),
1992              (std::string("Found no ") + key).c_str());
1993    }
1994
1995    evict_key(h, h1, "a", 0, "Ejected.");
1996    check_key_value(h, h1, "a", "b\r\n", 3, 0);
1997    check(get_int_stat(h, h1, "ep_bg_num_samples") == 2,
1998          "Expected one sample");
1999
2000    h1->reset_stats(h, NULL);
2001    checkeq(0, get_int_stat(h, h1, "ep_bg_fetched"),
2002            "ep_bg_fetched is not reset to 0");
2003    return SUCCESS;
2004}
2005
2006static enum test_result test_bg_meta_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2007    item *itm = NULL;
2008    h1->reset_stats(h, NULL);
2009
2010    wait_for_persisted_value(h, h1, "k1", "v1");
2011    wait_for_persisted_value(h, h1, "k2", "v2");
2012
2013    evict_key(h, h1, "k1", 0, "Ejected.");
2014    checkeq(ENGINE_SUCCESS,
2015            del(h, h1, "k2", 0, 0), "Failed remove with value.");
2016    wait_for_stat_to_be(h, h1, "curr_items", 1);
2017
2018    checkeq(0, get_int_stat(h, h1, "ep_bg_fetched"), "Expected bg_fetched to be 0");
2019    checkeq(0, get_int_stat(h, h1, "ep_bg_meta_fetched"), "Expected bg_meta_fetched to be 0");
2020
2021    check(get_meta(h, h1, "k2"), "Get meta failed");
2022    checkeq(0, get_int_stat(h, h1, "ep_bg_fetched"), "Expected bg_fetched to be 0");
2023    checkeq(1, get_int_stat(h, h1, "ep_bg_meta_fetched"), "Expected bg_meta_fetched to be 1");
2024
2025    checkeq(ENGINE_SUCCESS, h1->get(h, NULL, &itm, "k1", 2, 0), "Missing key");
2026    checkeq(1, get_int_stat(h, h1, "ep_bg_fetched"), "Expected bg_fetched to be 1");
2027    checkeq(1, get_int_stat(h, h1, "ep_bg_meta_fetched"), "Expected bg_meta_fetched to be 1");
2028    h1->release(h, NULL, itm);
2029
2030    // store new key with some random metadata
2031    const size_t keylen = strlen("k3");
2032    ItemMetaData itemMeta;
2033    itemMeta.revSeqno = 10;
2034    itemMeta.cas = 0xdeadbeef;
2035    itemMeta.exptime = 0;
2036    itemMeta.flags = 0xdeadbeef;
2037
2038    add_with_meta(h, h1, "k3", keylen, NULL, 0, 0, &itemMeta);
2039    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Set meta failed");
2040
2041    check(get_meta(h, h1, "k2"), "Get meta failed");
2042    checkeq(1, get_int_stat(h, h1, "ep_bg_fetched"), "Expected bg_fetched to be 1");
2043    checkeq(1, get_int_stat(h, h1, "ep_bg_meta_fetched"),
2044            "Expected bg_meta_fetched to remain at 1");
2045
2046    return SUCCESS;
2047}
2048
2049static enum test_result test_key_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2050    item *i = NULL;
2051
2052    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed set vbucket 1 state.");
2053
2054    // set (k1,v1) in vbucket 0
2055    checkeq(ENGINE_SUCCESS,
2056            store(h, h1, NULL, OPERATION_SET,"k1", "v1", &i, 0, 0),
2057            "Failed to store an item.");
2058    h1->release(h, NULL, i);
2059    // set (k2,v2) in vbucket 1
2060    checkeq(ENGINE_SUCCESS,
2061            store(h, h1, NULL, OPERATION_SET,"k2", "v2", &i, 0, 1),
2062            "Failed to store an item.");
2063    h1->release(h, NULL, i);
2064
2065    const void *cookie = testHarness.create_cookie();
2066
2067    // stat for key "k1" and vbucket "0"
2068    const char *statkey1 = "key k1 0";
2069    checkeq(ENGINE_SUCCESS,
2070            h1->get_stats(h, cookie, statkey1, strlen(statkey1), add_stats),
2071            "Failed to get stats.");
2072    check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2073    check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2074    check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2075    check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2076    check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2077
2078    // stat for key "k2" and vbucket "1"
2079    const char *statkey2 = "key k2 1";
2080    checkeq(ENGINE_SUCCESS,
2081            h1->get_stats(h, cookie, statkey2, strlen(statkey2), add_stats),
2082            "Failed to get stats.");
2083    check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2084    check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2085    check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2086    check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2087    check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2088
2089    testHarness.destroy_cookie(cookie);
2090    return SUCCESS;
2091}
2092
2093static enum test_result test_vkey_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2094    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed set vbucket 1 state.");
2095    check(set_vbucket_state(h, h1, 2, vbucket_state_active), "Failed set vbucket 2 state.");
2096    check(set_vbucket_state(h, h1, 3, vbucket_state_active), "Failed set vbucket 3 state.");
2097    check(set_vbucket_state(h, h1, 4, vbucket_state_active), "Failed set vbucket 4 state.");
2098
2099    wait_for_persisted_value(h, h1, "k1", "v1");
2100    wait_for_persisted_value(h, h1, "k2", "v2", 1);
2101    wait_for_persisted_value(h, h1, "k3", "v3", 2);
2102    wait_for_persisted_value(h, h1, "k4", "v4", 3);
2103    wait_for_persisted_value(h, h1, "k5", "v5", 4);
2104
2105    check(set_vbucket_state(h, h1, 2, vbucket_state_replica), "Failed to set VB2 state.");
2106    check(set_vbucket_state(h, h1, 3, vbucket_state_pending), "Failed to set VB3 state.");
2107    check(set_vbucket_state(h, h1, 4, vbucket_state_dead), "Failed to set VB4 state.");
2108
2109    const void *cookie = testHarness.create_cookie();
2110
2111    // stat for key "k1" and vbucket "0"
2112    const char *statkey1 = "vkey k1 0";
2113    checkeq(ENGINE_SUCCESS,
2114            h1->get_stats(h, cookie, statkey1, strlen(statkey1), add_stats),
2115            "Failed to get stats.");
2116    check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2117    check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2118    check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2119    check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2120    check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2121    check(vals.find("key_valid") != vals.end(), "Found no key_valid");
2122
2123    // stat for key "k2" and vbucket "1"
2124    const char *statkey2 = "vkey k2 1";
2125    checkeq(ENGINE_SUCCESS,
2126            h1->get_stats(h, cookie, statkey2, strlen(statkey2), add_stats),
2127            "Failed to get stats.");
2128    check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2129    check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2130    check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2131    check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2132    check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2133    check(vals.find("key_valid") != vals.end(), "Found no key_valid");
2134
2135    // stat for key "k3" and vbucket "2"
2136    const char *statkey3 = "vkey k3 2";
2137    checkeq(ENGINE_SUCCESS,
2138            h1->get_stats(h, cookie, statkey3, strlen(statkey3), add_stats),
2139            "Failed to get stats.");
2140    check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2141    check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2142    check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2143    check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2144    check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2145    check(vals.find("key_valid") != vals.end(), "Found no key_valid");
2146
2147    // stat for key "k4" and vbucket "3"
2148    const char *statkey4 = "vkey k4 3";
2149    checkeq(ENGINE_SUCCESS,
2150            h1->get_stats(h, cookie, statkey4, strlen(statkey4), add_stats),
2151            "Failed to get stats.");
2152    check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2153    check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2154    check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2155    check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2156    check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2157    check(vals.find("key_valid") != vals.end(), "Found no key_valid");
2158
2159    // stat for key "k5" and vbucket "4"
2160    const char *statkey5 = "vkey k5 4";
2161    checkeq(ENGINE_SUCCESS,
2162            h1->get_stats(h, cookie, statkey5, strlen(statkey5), add_stats),
2163            "Failed to get stats.");
2164    check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2165    check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2166    check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2167    check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2168    check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2169    check(vals.find("key_valid") != vals.end(), "Found no key_valid");
2170
2171    testHarness.destroy_cookie(cookie);
2172    return SUCCESS;
2173}
2174
2175static enum test_result test_warmup_conf(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2176    checkeq(100, get_int_stat(h, h1, "ep_warmup_min_items_threshold"),
2177            "Incorrect initial warmup min items threshold.");
2178    checkeq(100, get_int_stat(h, h1, "ep_warmup_min_memory_threshold"),
2179            "Incorrect initial warmup min memory threshold.");
2180
2181    check(!set_param(h, h1, protocol_binary_engine_param_flush,
2182                     "warmup_min_items_threshold", "a"),
2183          "Set warmup_min_items_threshold should have failed");
2184    check(!set_param(h, h1, protocol_binary_engine_param_flush,
2185                     "warmup_min_items_threshold", "a"),
2186          "Set warmup_min_memory_threshold should have failed");
2187
2188    check(set_param(h, h1, protocol_binary_engine_param_flush,
2189                    "warmup_min_items_threshold", "80"),
2190          "Set warmup_min_items_threshold should have worked");
2191    check(set_param(h, h1, protocol_binary_engine_param_flush,
2192                    "warmup_min_memory_threshold", "80"),
2193          "Set warmup_min_memory_threshold should have worked");
2194
2195    checkeq(80, get_int_stat(h, h1, "ep_warmup_min_items_threshold"),
2196            "Incorrect smaller warmup min items threshold.");
2197    checkeq(80, get_int_stat(h, h1, "ep_warmup_min_memory_threshold"),
2198            "Incorrect smaller warmup min memory threshold.");
2199
2200    item *it = NULL;
2201    for (int i = 0; i < 100; ++i) {
2202        std::stringstream key;
2203        key << "key-" << i;
2204        checkeq(ENGINE_SUCCESS,
2205                store(h, h1, NULL, OPERATION_SET, key.str().c_str(), "somevalue", &it),
2206                "Error setting.");
2207        h1->release(h, NULL, it);
2208    }
2209
2210    // Restart the server.
2211    std::string config(testHarness.get_current_testcase()->cfg);
2212    config = config + "warmup_min_memory_threshold=0";
2213    testHarness.reload_engine(&h, &h1,
2214                              testHarness.engine_path,
2215                              config.c_str(),
2216                              true, false);
2217    wait_for_warmup_complete(h, h1);
2218
2219    const std::string eviction_policy = get_str_stat(h, h1, "ep_item_eviction_policy");
2220    if (eviction_policy == "value_only") {
2221        checkeq(100, get_int_stat(h, h1, "ep_warmup_key_count", "warmup"),
2222                "Expected 100 keys loaded after warmup");
2223    } else { // Full eviction mode
2224        checkeq(0, get_int_stat(h, h1, "ep_warmup_key_count", "warmup"),
2225                "Expected 0 keys loaded after warmup");
2226    }
2227
2228    checkeq(0, get_int_stat(h, h1, "ep_warmup_value_count", "warmup"),
2229            "Expected 0 values loaded after warmup");
2230
2231    return SUCCESS;
2232}
2233
2234static enum test_result test_bloomfilter_conf(ENGINE_HANDLE *h,
2235                                              ENGINE_HANDLE_V1 *h1) {
2236
2237    if (get_bool_stat(h, h1, "ep_bfilter_enabled") == false) {
2238        check(set_param(h, h1, protocol_binary_engine_param_flush,
2239                        "bfilter_enabled", "true"),
2240              "Set bloomfilter_enabled should have worked");
2241    }
2242    check(get_bool_stat(h, h1, "ep_bfilter_enabled"),
2243          "Bloom filter wasn't enabled");
2244
2245    check(get_float_stat(h, h1, "ep_bfilter_residency_threshold") == (float)0.1,
2246          "Incorrect initial bfilter_residency_threshold.");
2247
2248    check(set_param(h, h1, protocol_binary_engine_param_flush,
2249          "bfilter_enabled", "false"),
2250          "Set bloomfilter_enabled should have worked.");
2251    check(set_param(h, h1, protocol_binary_engine_param_flush,
2252          "bfilter_residency_threshold", "0.15"),
2253          "Set bfilter_residency_threshold should have worked.");
2254
2255    check(get_bool_stat(h, h1, "ep_bfilter_enabled") == false,
2256          "Bloom filter should have been disabled.");
2257    check(get_float_stat(h, h1, "ep_bfilter_residency_threshold") == (float)0.15,
2258          "Incorrect bfilter_residency_threshold.");
2259
2260    return SUCCESS;
2261}
2262
2263static enum test_result test_bloomfilters(ENGINE_HANDLE *h,
2264                                          ENGINE_HANDLE_V1 *h1) {
2265
2266    if (get_bool_stat(h, h1, "ep_bfilter_enabled") == false) {
2267        check(set_param(h, h1, protocol_binary_engine_param_flush,
2268                    "bfilter_enabled", "true"),
2269                "Set bloomfilter_enabled should have worked");
2270    }
2271    check(get_bool_stat(h, h1, "ep_bfilter_enabled"),
2272            "Bloom filter wasn't enabled");
2273
2274    // Key is only present if bgOperations is non-zero.
2275    int num_read_attempts = get_int_stat_or_default(h, h1, 0,
2276                                                    "ep_bg_num_samples");
2277
2278    // Ensure vbucket's bloom filter is enabled
2279    checkeq(std::string("ENABLED"),
2280            get_str_stat(h, h1, "vb_0:bloom_filter", "vbucket-details 0"),
2281            "Vbucket 0's bloom filter wasn't enabled upon setup!");
2282
2283    int i;
2284    item *it = NULL;
2285
2286    // Insert 10 items.
2287    for (i = 0; i < 10; ++i) {
2288        std::stringstream key;
2289        key << "key-" << i;
2290        checkeq(ENGINE_SUCCESS,
2291                store(h, h1, NULL, OPERATION_SET, key.str().c_str(),
2292                      "somevalue", &it),
2293                "Error setting.");
2294        h1->release(h, NULL, it);
2295    }
2296    wait_for_flusher_to_settle(h, h1);
2297
2298    // Evict all 10 items.
2299    for (i = 0; i < 10; ++i) {
2300        std::stringstream key;
2301        key << "key-" << i;
2302        evict_key(h, h1, key.str().c_str(), 0, "Ejected.");
2303    }
2304    wait_for_flusher_to_settle(h, h1);
2305
2306    // Ensure 10 items are non-resident.
2307    cb_assert(10 == get_int_stat(h, h1, "ep_num_non_resident"));
2308
2309    // Issue delete on first 5 items.
2310    for (i = 0; i < 5; ++i) {
2311        std::stringstream key;
2312        key << "key-" << i;
2313        checkeq(ENGINE_SUCCESS,
2314                del(h, h1, key.str().c_str(), 0, 0),
2315                "Failed remove with value.");
2316    }
2317    wait_for_flusher_to_settle(h, h1);
2318
2319    // Ensure that there are 5 non-resident items
2320    cb_assert(5 == get_int_stat(h, h1, "ep_num_non_resident"));
2321    cb_assert(5 == get_int_stat(h, h1, "curr_items"));
2322
2323    checkeq(ENGINE_SUCCESS,
2324            h1->get_stats(h, NULL, NULL, 0, add_stats),
2325            "Failed to get stats.");
2326    std::string eviction_policy = vals.find("ep_item_eviction_policy")->second;
2327
2328    useconds_t sleepTime = 128;
2329
2330    if (eviction_policy == "value_only") {  // VALUE-ONLY EVICTION MODE
2331
2332        checkeq(5,
2333                get_int_stat(h, h1, "vb_0:bloom_filter_key_count",
2334                             "vbucket-details 0"),
2335                "Unexpected no. of keys in bloom filter");
2336
2337        checkeq(num_read_attempts,
2338                get_int_stat_or_default(h, h1, 0, "ep_bg_num_samples"),
2339                "Expected bgFetch attempts to remain unchanged");
2340
2341        for (i = 0; i < 5; ++i) {
2342            std::stringstream key;
2343            key << "key-" << i;
2344            check(get_meta(h, h1, key.str().c_str()), "Get meta failed");
2345        }
2346
2347        // GetMeta would cause bgFetches as bloomfilter contains
2348        // the deleted items.
2349        checkeq(num_read_attempts + 5,
2350                get_int_stat(h, h1, "ep_bg_num_samples"),
2351                "Expected bgFetch attempts to increase by five");
2352
2353        // Run compaction, with drop_deletes
2354        compact_db(h, h1, 0, 15, 15, 1);
2355        while (get_int_stat(h, h1, "ep_pending_compactions") != 0) {
2356            decayingSleep(&sleepTime);
2357        }
2358
2359        for (i = 0; i < 5; ++i) {
2360            std::stringstream key;
2361            key << "key-" << i;
2362            check(get_meta(h, h1, key.str().c_str()), "Get meta failed");
2363        }
2364        checkeq(num_read_attempts + 5,
2365                get_int_stat(h, h1, "ep_bg_num_samples"),
2366                "Expected bgFetch attempts to stay as before");
2367
2368    } else {                                // FULL EVICTION MODE
2369
2370        checkeq(10,
2371                get_int_stat(h, h1, "vb_0:bloom_filter_key_count",
2372                             "vbucket-details 0"),
2373                "Unexpected no. of keys in bloom filter");
2374
2375
2376        // Because of issuing deletes on non-resident items
2377        checkeq(num_read_attempts + 5,
2378                get_int_stat(h, h1, "ep_bg_num_samples"),
2379                "Expected bgFetch attempts to increase by five, after deletes");
2380
2381        // Run compaction, with drop_deletes, to exclude deleted items
2382        // from bloomfilter.
2383        compact_db(h, h1, 0, 15, 15, 1);
2384        while (get_int_stat(h, h1, "ep_pending_compactions") != 0) {
2385            decayingSleep(&sleepTime);
2386        }
2387
2388        for (i = 0; i < 5; i++) {
2389            std::stringstream key;
2390            key << "key-" << i;
2391            checkeq(ENGINE_KEY_ENOENT,
2392                    h1->get(h, NULL, &it, key.str().c_str(), key.str().length(), 0),
2393                    "Unable to get stored item");
2394        }
2395        // + 6 because last delete is not purged by the compactor
2396        checkeq(num_read_attempts + 6,
2397                get_int_stat(h, h1, "ep_bg_num_samples"),
2398                "Expected bgFetch attempts to stay as before");
2399    }
2400
2401    return SUCCESS;
2402}
2403
2404static enum test_result test_bloomfilters_with_store_apis(ENGINE_HANDLE *h,
2405                                                          ENGINE_HANDLE_V1 *h1) {
2406    if (get_bool_stat(h, h1, "ep_bfilter_enabled") == false) {
2407        check(set_param(h, h1, protocol_binary_engine_param_flush,
2408                    "bfilter_enabled", "true"),
2409                "Set bloomfilter_enabled should have worked");
2410    }
2411    check(get_bool_stat(h, h1, "ep_bfilter_enabled"),
2412            "Bloom filter wasn't enabled");
2413
2414    int num_read_attempts = get_int_stat_or_default(h, h1, 0,
2415                                                    "ep_bg_num_samples");
2416
2417    // Ensure vbucket's bloom filter is enabled
2418    checkeq(std::string("ENABLED"),
2419            get_str_stat(h, h1, "vb_0:bloom_filter", "vbucket-details 0"),
2420            "Vbucket 0's bloom filter wasn't enabled upon setup!");
2421
2422    for (int i = 0; i < 1000; i++) {
2423        std::stringstream key;
2424        key << "key-" << i;
2425        check(!get_meta(h, h1, key.str().c_str()),
2426                "Get meta should fail.");
2427    }
2428
2429    checkeq(num_read_attempts,
2430            get_int_stat_or_default(h, h1, 0, "ep_bg_num_samples"),
2431            "Expected no bgFetch attempts");
2432
2433    checkeq(ENGINE_SUCCESS,
2434            h1->get_stats(h, NULL, NULL, 0, add_stats),
2435            "Failed to get stats.");
2436    std::string eviction_policy = vals.find("ep_item_eviction_policy")->second;
2437
2438    if (eviction_policy == "full_eviction") {  // FULL EVICTION MODE
2439        // Set with Meta
2440        int j;
2441        for (j = 0; j < 10; j++) {
2442            uint64_t cas_for_set = last_cas;
2443            // init some random metadata
2444            ItemMetaData itm_meta;
2445            itm_meta.revSeqno = 10;
2446            itm_meta.cas = 0xdeadbeef;
2447            itm_meta.exptime = time(NULL) + 300;
2448            itm_meta.flags = 0xdeadbeef;
2449
2450            std::stringstream key;
2451            key << "swm-" << j;
2452            set_with_meta(h, h1, key.str().c_str(), key.str().length(),
2453                          "somevalue", 9, 0, &itm_meta, cas_for_set);
2454        }
2455
2456        checkeq(num_read_attempts,
2457                get_int_stat_or_default(h, h1, 0, "ep_bg_num_samples"),
2458                "Expected no bgFetch attempts");
2459
2460        item *itm = NULL;
2461        // Add
2462        for (j = 0; j < 10; j++) {
2463            std::stringstream key;
2464            key << "add-" << j;
2465
2466            checkeq(ENGINE_SUCCESS,
2467                    store(h, h1, NULL, OPERATION_ADD, key.str().c_str(),
2468                          "newvalue", &itm),
2469                    "Failed to add value again.");
2470            h1->release(h, NULL, itm);
2471        }
2472
2473        checkeq(num_read_attempts,
2474                get_int_stat_or_default(h, h1, 0, "ep_bg_num_samples"),
2475                "Expected no bgFetch attempts");
2476
2477        // Delete
2478        for (j = 0; j < 10; j++) {
2479            std::stringstream key;
2480            key << "del-" << j;
2481            checkeq(ENGINE_KEY_ENOENT,
2482                    del(h, h1, key.str().c_str(), 0, 0),
2483                    "Failed remove with value.");
2484        }
2485
2486        checkeq(num_read_attempts,
2487                get_int_stat_or_default(h, h1, 0, "ep_bg_num_samples"),
2488                "Expected no bgFetch attempts");
2489
2490    }
2491
2492    return SUCCESS;
2493}
2494
2495static enum test_result test_bloomfilter_delete_plus_set_scenario(
2496                                       ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2497    if (get_bool_stat(h, h1, "ep_bfilter_enabled") == false) {
2498        check(set_param(h, h1, protocol_binary_engine_param_flush,
2499                    "bfilter_enabled", "true"),
2500                "Set bloomfilter_enabled should have worked");
2501    }
2502    check(get_bool_stat(h, h1, "ep_bfilter_enabled"),
2503            "Bloom filter wasn't enabled");
2504
2505    // Ensure vbucket's bloom filter is enabled
2506    checkeq(std::string("ENABLED"),
2507            get_str_stat(h, h1, "vb_0:bloom_filter", "vbucket-details 0"),
2508            "Vbucket 0's bloom filter wasn't enabled upon setup!");
2509
2510    item *itm = NULL;
2511    checkeq(ENGINE_SUCCESS,
2512            store(h, h1, NULL, OPERATION_SET,"k1", "v1", &itm),
2513            "Failed to fail to store an item.");
2514    h1->release(h, NULL, itm);
2515
2516    wait_for_flusher_to_settle(h, h1);
2517    int num_writes = get_int_stat(h, h1, "rw_0:io_num_write", "kvstore");
2518    int num_persisted = get_int_stat(h, h1, "ep_total_persisted");
2519    cb_assert(num_writes == 1 && num_persisted == 1);
2520
2521    checkeq(ENGINE_SUCCESS,
2522            del(h, h1, "k1", 0, 0), "Failed remove with value.");
2523    stop_persistence(h, h1);
2524    checkeq(ENGINE_SUCCESS,
2525            store(h, h1, NULL, OPERATION_SET,"k1", "v2", &itm, 0, 0),
2526            "Failed to fail to store an item.");
2527    h1->release(h, NULL, itm);
2528    int key_count = get_int_stat(h, h1, "vb_0:bloom_filter_key_count",
2529                                 "vbucket-details 0");
2530
2531    if (key_count == 0) {
2532        check(get_int_stat(h, h1, "rw_0:io_num_write", "kvstore") <= 2,
2533                "Unexpected number of writes");
2534        start_persistence(h, h1);
2535        wait_for_flusher_to_settle(h, h1);
2536        checkeq(0, get_int_stat(h, h1, "vb_0:bloom_filter_key_count",
2537                                "vbucket-details 0"),
2538                "Unexpected number of keys in bloomfilter");
2539    } else {
2540        cb_assert(key_count == 1);
2541        checkeq(2, get_int_stat(h, h1, "rw_0:io_num_write", "kvstore"),
2542                "Unexpected number of writes");
2543        start_persistence(h, h1);
2544        wait_for_flusher_to_settle(h, h1);
2545        checkeq(1, get_int_stat(h, h1, "vb_0:bloom_filter_key_count",
2546                                "vbucket-details 0"),
2547                "Unexpected number of keys in bloomfilter");
2548    }
2549
2550    return SUCCESS;
2551}
2552
2553static enum test_result test_datatype(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2554    const void *cookie = testHarness.create_cookie();
2555    testHarness.set_datatype_support(cookie, true);
2556
2557    item *itm = NULL;
2558    const std::string key("{\"foo\":\"bar\"}");
2559    const protocol_binary_datatypes datatype = PROTOCOL_BINARY_DATATYPE_JSON;
2560    uint64_t cas = 0;
2561    std::string value("x");
2562    checkeq(ENGINE_SUCCESS,
2563            storeCasOut(h, h1, NULL, 0, key, value, datatype, itm, cas),
2564            "Expected set to succeed");
2565
2566    checkeq(ENGINE_SUCCESS,
2567            h1->get(h, cookie, &itm, key.c_str(), key.size(), 0),
2568            "Unable to get stored item");
2569
2570    item_info info;
2571    info.nvalue = 1;
2572    h1->get_item_info(h, cookie, itm, &info);
2573    h1->release(h, cookie, itm);
2574    checkeq(static_cast<uint8_t>(PROTOCOL_BINARY_DATATYPE_JSON),
2575            info.datatype, "Invalid datatype");
2576
2577    const char* key1 = "foo";
2578    const char* val1 = "{\"foo1\":\"bar1\"}";
2579    ItemMetaData itm_meta;
2580    itm_meta.revSeqno = 10;
2581    itm_meta.cas = info.cas;
2582    itm_meta.exptime = info.exptime;
2583    itm_meta.flags = info.flags;
2584    set_with_meta(h, h1, key1, strlen(key1), val1, strlen(val1), 0, &itm_meta,
2585                  last_cas, 0, info.datatype, cookie);
2586
2587    checkeq(ENGINE_SUCCESS,
2588            h1->get(h, cookie, &itm, key1, strlen(key1), 0),
2589            "Unable to get stored item");
2590
2591    h1->get_item_info(h, cookie, itm, &info);
2592    h1->release(h, cookie, itm);
2593    checkeq(static_cast<uint8_t>(PROTOCOL_BINARY_DATATYPE_JSON),
2594            info.datatype, "Invalid datatype, when setWithMeta");
2595
2596    testHarness.destroy_cookie(cookie);
2597    return SUCCESS;
2598}
2599
2600static enum test_result test_datatype_with_unknown_command(ENGINE_HANDLE *h,
2601                                                           ENGINE_HANDLE_V1 *h1) {
2602    const void *cookie = testHarness.create_cookie();
2603    testHarness.set_datatype_support(cookie, true);
2604    item *itm = NULL;
2605    const char* key = "foo";
2606    const char* val = "{\"foo\":\"bar\"}";
2607    uint8_t datatype = PROTOCOL_BINARY_DATATYPE_JSON;
2608
2609    ItemMetaData itm_meta;
2610    itm_meta.revSeqno = 10;
2611    itm_meta.cas = 0x1;
2612    itm_meta.exptime = 0;
2613    itm_meta.flags = 0;
2614
2615    //SET_WITH_META
2616    set_with_meta(h, h1, key, strlen(key), val, strlen(val), 0, &itm_meta,
2617                  0, 0, datatype, cookie);
2618
2619    checkeq(ENGINE_SUCCESS,
2620            h1->get(h, cookie, &itm, key, strlen(key), 0),
2621            "Unable to get stored item");
2622
2623    item_info info;
2624    info.nvalue = 1;
2625    h1->get_item_info(h, cookie, itm, &info);
2626    h1->release(h, NULL, itm);
2627    checkeq(static_cast<uint8_t>(PROTOCOL_BINARY_DATATYPE_JSON),
2628            info.datatype, "Invalid datatype, when setWithMeta");
2629
2630    //SET_RETURN_META
2631    set_ret_meta(h, h1, "foo1", 4, val, strlen(val), 0, 0, 0, 0, datatype,
2632                 cookie);
2633    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
2634            "Expected set returing meta to succeed");
2635    checkeq(static_cast<uint8_t>(PROTOCOL_BINARY_DATATYPE_JSON),
2636            last_datatype.load(), "Invalid datatype, when set_return_meta");
2637
2638    testHarness.destroy_cookie(cookie);
2639    return SUCCESS;
2640}
2641
2642static enum test_result test_session_cas_validation(ENGINE_HANDLE *h,
2643                                                    ENGINE_HANDLE_V1 *h1) {
2644    //Testing PROTOCOL_BINARY_CMD_SET_VBUCKET..
2645    char ext[4];
2646    protocol_binary_request_header *pkt;
2647    vbucket_state_t state = vbucket_state_active;
2648    uint32_t val = static_cast<uint32_t>(state);
2649    val = htonl(val);
2650    memcpy(ext, (char*)&val, sizeof(val));
2651
2652    uint64_t cas = 0x0101010101010101;
2653    pkt = createPacket(PROTOCOL_BINARY_CMD_SET_VBUCKET, 0, cas, ext, 4);
2654    checkeq(ENGINE_SUCCESS,
2655            h1->unknown_command(h, NULL, pkt, add_response),
2656            "SET_VBUCKET command failed");
2657    cb_free(pkt);
2658    cb_assert(last_status == PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS);
2659
2660    cas = 0x0102030405060708;
2661    pkt = createPacket(PROTOCOL_BINARY_CMD_SET_VBUCKET, 0, cas, ext, 4);
2662    checkeq(ENGINE_SUCCESS,
2663            h1->unknown_command(h, NULL, pkt, add_response),
2664            "SET_VBUCKET command failed");
2665    cb_free(pkt);
2666    cb_assert(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS);
2667
2668    return SUCCESS;
2669}
2670
2671static enum test_result test_access_scanner_settings(ENGINE_HANDLE *h,
2672                                                     ENGINE_HANDLE_V1 *h1) {
2673
2674    checkeq(ENGINE_SUCCESS,
2675            h1->get_stats(h, NULL, NULL, 0, add_stats),
2676            "Failed to get stats.");
2677    std::string policy = vals.find("ep_item_eviction_policy")->second;
2678    std::string name = vals.find("ep_alog_path")->second;
2679    check(!name.empty(), "No access log path specified!");
2680
2681    std::string oldparam(".log");
2682    std::string newparam(policy + oldparam);
2683    std::string config = testHarness.get_current_testcase()->cfg;
2684    std::string::size_type found = config.find(oldparam);
2685    if (found != config.npos) {
2686        config.replace(found, oldparam.size(), newparam);
2687    }
2688
2689    testHarness.reload_engine(&h, &h1,
2690                              testHarness.engine_path,
2691                              config.c_str(),
2692                              true, false);
2693    wait_for_warmup_complete(h, h1);
2694
2695    std::string err_msg;
2696    // Check access scanner is enabled and alog_task_time is at default
2697    checkeq(true, get_bool_stat(h, h1, "ep_access_scanner_enabled"),
2698            "Expected access scanner to be enabled");
2699    cb_assert(get_int_stat(h, h1, "ep_alog_task_time") == 2);
2700
2701    // Ensure access_scanner_task_time is what its expected to be.
2702    // Need to wait until the AccessScanner task has been setup.
2703    wait_for_stat_change(h, h1, "ep_access_scanner_task_time",
2704                         std::string{"NOT_SCHEDULED"});
2705
2706    std::string str = get_str_stat(h, h1, "ep_access_scanner_task_time");
2707    std::string expected_time = "02:00";
2708    err_msg.assign("Initial time incorrect, expect: " +
2709                   expected_time + ", actual: " + str.substr(11, 5));
2710    checkeq(0, str.substr(11, 5).compare(expected_time), err_msg.c_str());
2711
2712    // Update alog_task_time and ensure the update is successful
2713    set_param(h, h1, protocol_binary_engine_param_flush, "alog_task_time", "5");
2714    expected_time = "05:00";
2715    str = get_str_stat(h, h1, "ep_access_scanner_task_time");
2716    err_msg.assign("Updated time incorrect, expect: " +
2717                   expected_time + ", actual: " + str.substr(11, 5));
2718    checkeq(0, str.substr(11, 5).compare(expected_time), err_msg.c_str());
2719
2720    // Update alog_sleep_time by 10 mins and ensure the update is successful.
2721    const std::chrono::minutes update_by{10};
2722    std::string targetTaskTime1{make_time_string(std::chrono::system_clock::now() +
2723                                                 update_by)};
2724
2725    set_param(h, h1, protocol_binary_engine_param_flush, "alog_sleep_time",
2726              std::to_string(update_by.count()).c_str());
2727    str = get_str_stat(h, h1, "ep_access_scanner_task_time");
2728
2729    // Recalculate now() + 10mins as upper bound on when the task should be
2730    // scheduled.
2731    std::string targetTaskTime2{make_time_string(std::chrono::system_clock::now() +
2732                                                 update_by)};
2733
2734    // ep_access_scanner_task_time should fall within the range of
2735    // targetTaskTime1 and targetTaskTime2
2736    err_msg.assign("Unexpected task time range, expect: " +
2737                   targetTaskTime1 + " <= " + str + " <= " + targetTaskTime2);
2738    check(targetTaskTime1 <= str, err_msg.c_str());
2739    check(str <= targetTaskTime2, err_msg.c_str());
2740
2741    return SUCCESS;
2742}
2743
2744static enum test_result test_access_scanner(ENGINE_HANDLE *h,
2745                                            ENGINE_HANDLE_V1 *h1) {
2746
2747    checkeq(ENGINE_SUCCESS,
2748            h1->get_stats(h, NULL, NULL, 0, add_stats),
2749            "Failed to get stats.");
2750    std::string policy = vals.find("ep_item_eviction_policy")->second;
2751    std::string name = vals.find("ep_alog_path")->second;
2752    check(!name.empty(), "No access log path specified!");
2753
2754    std::string oldparam(".log");
2755    std::string newparam(policy + oldparam);
2756    std::string config = testHarness.get_current_testcase()->cfg;
2757    std::string::size_type found = config.find(oldparam);
2758    if (found != config.npos) {
2759        config.replace(found, oldparam.size(), newparam);
2760    }
2761
2762    /* We do not want the access scanner task to be running while we initiate it
2763       explicitly below. Hence set the alog_task_time to about 1 ~ 2 hours
2764       from now */
2765    const time_t now = time(nullptr);
2766    struct tm tm_now;
2767    cb_gmtime_r(&now, &tm_now);
2768    const auto two_hours_hence = (tm_now.tm_hour + 2) % 24;
2769
2770    set_param(h, h1, protocol_binary_engine_param_flush, "alog_task_time",
2771              std::to_string(two_hours_hence).c_str());
2772    wait_for_stat_to_be(h, h1, "ep_alog_task_time", two_hours_hence);
2773
2774    testHarness.reload_engine(&h, &h1,
2775                              testHarness.engine_path,
2776                              config.c_str(),
2777                              true, false);
2778    wait_for_warmup_complete(h, h1);
2779
2780    checkeq(ENGINE_SUCCESS,
2781            h1->get_stats(h, NULL, NULL, 0, add_stats),
2782            "Failed to get stats.");
2783    name = vals.find("ep_alog_path")->second;
2784
2785    /* Check access scanner is enabled */
2786    checkeq(true, get_bool_stat(h, h1, "ep_access_scanner_enabled"),
2787            "Access scanner task not enabled by default. Check test config");
2788
2789    const int num_shards = get_int_stat(h, h1, "ep_workload:num_shards",
2790                                        "workload");
2791    name = name + ".0";
2792    std::string prev(name + ".old");
2793
2794    /* Get the resident ratio down to below 95% - point at which access.log
2795     * generation occurs.
2796     */
2797    int num_items = 0;
2798    while (true) {
2799        // Gathering stats on every store is expensive, just check every 100 iterations
2800        if ((num_items % 100) == 0) {
2801            if (get_int_stat(h, h1, "vb_active_perc_mem_resident") < 94) {
2802                break;
2803            }
2804        }
2805
2806        item *itm = NULL;
2807        std::string key("key" + std::to_string(num_items));
2808        ENGINE_ERROR_CODE ret = store(h, h1, NULL, OPERATION_SET,
2809                                      key.c_str(), "somevalue", &itm);
2810        switch (ret) {
2811            case ENGINE_SUCCESS:
2812                num_items++;
2813                h1->release(h, NULL, itm);
2814                break;
2815
2816            case ENGINE_ENOMEM:
2817            case ENGINE_TMPFAIL:
2818                // Returned when at high watermark; simply retry the op.
2819                h1->release(h, NULL, itm);
2820                break;
2821
2822            default:
2823                fprintf(stderr, "test_access_scanner: Unexpected result from store(): %d\n",
2824                        ret);
2825                abort();
2826        }
2827
2828    }
2829
2830    wait_for_flusher_to_settle(h, h1);
2831    verify_curr_items(h, h1, num_items, "Wrong number of items");
2832    int num_non_resident = get_int_stat(h, h1, "vb_active_num_non_resident");
2833    checkge(num_non_resident, num_items * 6 / 100,
2834            "Expected num_non_resident to be at least 6% of total items");
2835
2836    /* Run access scanner task once and expect it to generate access log */
2837    check(set_param(h, h1, protocol_binary_engine_param_flush,
2838                    "access_scanner_run", "true"),
2839          "Failed to trigger access scanner");
2840
2841    // Wait for the number of runs to equal the number of shards.
2842    wait_for_stat_to_be(h, h1, "ep_num_access_scanner_runs", num_shards);
2843
2844    /* This time since resident ratio is < 95% access log should be generated */
2845    checkeq(0, access(name.c_str(), F_OK), "access log file should exist");
2846
2847    /* Increase resident ratio by deleting items */
2848    vbucketDelete(h, h1, 0);
2849    check(set_vbucket_state(h, h1, 0, vbucket_state_active),
2850          "Failed to set VB0 state.");
2851
2852    /* Run access scanner task once */
2853    const int access_scanner_skips =
2854            get_int_stat(h, h1, "ep_num_access_scanner_skips");
2855    check(set_param(h, h1, protocol_binary_engine_param_flush,
2856                    "access_scanner_run", "true"),
2857          "Failed to trigger access scanner");
2858    wait_for_stat_to_be(h, h1, "ep_num_access_scanner_skips",
2859                        access_scanner_skips + num_shards);
2860
2861    /* Access log files should be removed because resident ratio > 95% */
2862    checkeq(-1, access(prev.c_str(), F_OK),
2863            ".old access log file should not exist");
2864    checkeq(-1, access(name.c_str(), F_OK), "access log file should not exist");
2865
2866    return SUCCESS;
2867}
2868
2869static enum test_result test_set_param_message(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2870    set_param(h, h1, protocol_binary_engine_param_flush, "alog_task_time", "50");
2871
2872    checkeq(PROTOCOL_BINARY_RESPONSE_EINVAL, last_status.load(),
2873        "Expected an invalid value error for an out of bounds alog_task_time");
2874    check(std::string("Validation Error").compare(last_body), "Expected a "
2875            "validation error in the response body");
2876    return SUCCESS;
2877}
2878
2879static enum test_result test_warmup_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2880    item *it = NULL;
2881    check(set_vbucket_state(h, h1, 0, vbucket_state_active), "Failed to set VB0 state.");
2882    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set VB1 state.");
2883
2884    for (int i = 0; i < 5000; ++i) {
2885        std::stringstream key;
2886        key << "key-" << i;
2887        checkeq(ENGINE_SUCCESS,
2888                store(h, h1, NULL, OPERATION_SET, key.str().c_str(),
2889                      "somevalue", &it),
2890                "Error setting.");
2891        h1->release(h, NULL, it);
2892    }
2893
2894    // Restart the server.
2895    testHarness.reload_engine(&h, &h1,
2896                              testHarness.engine_path,
2897                              testHarness.get_current_testcase()->cfg,
2898                              true, false);
2899
2900    wait_for_warmup_complete(h, h1);
2901
2902    const auto warmup_stats = get_all_stats(h, h1, "warmup");
2903
2904    // Check all expected warmup stats exists.
2905    const char* warmup_keys[] = { "ep_warmup_thread",
2906                                  "ep_warmup_value_count",
2907                                  "ep_warmup_key_count",
2908                                  "ep_warmup_dups",
2909                                  "ep_warmup_oom",
2910                                  "ep_warmup_time"};
2911    for (const auto* key : warmup_keys) {
2912        check(warmup_stats.find(key) != warmup_stats.end(),
2913              (std::string("Found no ") + key).c_str());
2914    }
2915
2916    std::string warmup_time = warmup_stats.at("ep_warmup_time");
2917    cb_assert(std::stoi(warmup_time) > 0);
2918
2919    const auto prev_vb_stats = get_all_stats(h, h1, "prev-vbucket");
2920
2921    check(prev_vb_stats.find("vb_0") != prev_vb_stats.end(),
2922          "Found no previous state for VB0");
2923    check(prev_vb_stats.find("vb_1") != prev_vb_stats.end(),
2924          "Found no previous state for VB1");
2925
2926    checkeq(std::string("active"), prev_vb_stats.at("vb_0"),
2927            "Unexpected stats for vb 0");
2928    checkeq(std::string("replica"), prev_vb_stats.at("vb_1"),
2929            "Unexpected stats for vb 1");
2930
2931    const auto vb_details_stats = get_all_stats(h, h1, "vbucket-details");
2932    checkeq(5000, std::stoi(vb_details_stats.at("vb_0:num_items")),
2933            "Unexpected item count for vb 0");
2934    checkeq(0, std::stoi(vb_details_stats.at("vb_1:num_items")),
2935            "Unexpected item count for vb 1");
2936
2937    return SUCCESS;
2938}
2939
2940static enum test_result test_warmup_with_threshold(ENGINE_HANDLE *h,
2941                                                   ENGINE_HANDLE_V1 *h1) {
2942    item *it = NULL;
2943    check(set_vbucket_state(h, h1, 0, vbucket_state_active), "Failed set vbucket 1 state.");
2944    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed set vbucket 2 state.");
2945    check(set_vbucket_state(h, h1, 2, vbucket_state_active), "Failed set vbucket 3 state.");
2946    check(set_vbucket_state(h, h1, 3, vbucket_state_active), "Failed set vbucket 4 state.");
2947
2948    for (int i = 0; i < 10000; ++i) {
2949        std::stringstream key;
2950        key << "key+" << i;
2951        checkeq(ENGINE_SUCCESS,
2952                store(h, h1, NULL, OPERATION_SET, key.str().c_str(), "somevalue", &it,
2953                     0, (i % 4)),
2954                "Error setting.");
2955        h1->release(h, NULL, it);
2956    }
2957
2958    // Restart the server.
2959    testHarness.reload_engine(&h, &h1,
2960                              testHarness.engine_path,
2961                              testHarness.get_current_testcase()->cfg,
2962                              true, false);
2963
2964    wait_for_warmup_complete(h, h1);
2965
2966    checkeq(1,
2967            get_int_stat(h, h1, "ep_warmup_min_item_threshold", "warmup"),
2968            "Unable to set warmup_min_item_threshold to 1%");
2969
2970    const std::string policy = get_str_stat(h, h1, "ep_item_eviction_policy");
2971
2972    if (policy == "full_eviction") {
2973        checkeq(get_int_stat(h, h1, "ep_warmup_key_count", "warmup"),
2974                get_int_stat(h, h1, "ep_warmup_value_count", "warmup"),
2975                "Warmed up key count didn't match warmed up value count");
2976    } else {
2977        checkeq(10000, get_int_stat(h, h1, "ep_warmup_key_count", "warmup"),
2978                "Warmup didn't warmup all keys");
2979    }
2980    check(get_int_stat(h, h1, "ep_warmup_value_count", "warmup") <= 110,
2981            "Warmed up value count found to be greater than 1%");
2982
2983    cb_assert(get_int_stat(h, h1, "ep_warmup_time", "warmup") > 0);
2984
2985    return SUCCESS;
2986}
2987
2988#if 0
2989// Comment out the entire test since the hack gave warnings on win32
2990static enum test_result test_warmup_accesslog(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2991#ifdef __APPLE__
2992    /* I'm getting a weird link error from clang.. disable the test until I
2993    ** understand why
2994    */
2995    return SKIPPED;
2996#else
2997    item *it = NULL;
2998
2999    int n_items_to_store1 = 10;
3000    for (int i = 0; i < n_items_to_store1; ++i) {
3001        std::stringstream key;
3002        key << "key-" << i;
3003        const char* keystr = key.str().c_str();
3004        checkeq(ENGINE_SUCCESS,
3005                store(h, h1, NULL, OPERATION_SET, keystr, "somevalue", &it, 0, 0),
3006                "Error setting.");
3007        h1->release(h, NULL, it);
3008    }
3009
3010    wait_for_flusher_to_settle(h, h1);
3011
3012    int n_items_to_access = 10;
3013    for (int i = 0; i < n_items_to_access; ++i) {
3014        std::stringstream key;
3015        key << "key-" << i;
3016        const char* keystr = key.str().c_str();
3017        checkeq(ENGINE_SUCCESS,
3018                h1->get(h, NULL, &it, keystr, strlen(keystr), 0),
3019                "Error getting.");
3020        h1->release(h, NULL, it);
3021    }
3022
3023    // sleep so that scanner task can have timew to generate access log
3024    sleep(61);
3025
3026    // store additional items
3027    int n_items_to_store2 = 10;
3028    for (int i = 0; i < n_items_to_store2; ++i) {
3029        std::stringstream key;
3030        key << "key2-" << i;
3031        const char* keystr = key.str().c_str();
3032        checkeq(ENGINE_SUCCESS,
3033                store(h, h1, NULL, OPERATION_SET, keystr, "somevalue", &it, 0, 0),
3034                "Error setting.");
3035        h1->release(h, NULL, it);
3036    }
3037
3038    // Restart the server.
3039    testHarness.reload_engine(&h, &h1,
3040                              testHarness.engine_path,
3041                              testHarness.get_current_testcase()->cfg,
3042                              true, false);
3043
3044    wait_for_warmup_complete(h, h1);
3045    // n_items_to_access items should be loaded from access log first
3046    // but we continue to load until we hit 75% item watermark
3047
3048    int warmedup = get_int_stat(h, h1, "ep_warmup_value_count", "warmup");
3049    //    std::cout << "ep_warmup_value_count = " << warmedup << std::endl;
3050    int expected = (n_items_to_store1 + n_items_to_store2) * 0.75 + 1;
3051
3052    check(warmedup == expected, "Expected 16 items to be resident");
3053    return SUCCESS;
3054#endif
3055}
3056#endif
3057
3058static enum test_result test_warmup_oom(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
3059
3060    write_items(h, h1, 20000, 0, "superlongnameofkey1234567890123456789012345678902");
3061
3062    wait_for_flusher_to_settle(h, h1);
3063
3064    std::string config(testHarness.get_current_testcase()->cfg);
3065    config = config + "max_size=2097152;item_eviction_policy=value_only";
3066
3067    testHarness.reload_engine(&h, &h1,
3068                              testHarness.engine_path,
3069                              config.c_str(),
3070                              true, false);
3071
3072    wait_for_warmup_complete(h, h1);
3073
3074    protocol_binary_request_header *pkt = createPacket(PROTOCOL_BINARY_CMD_ENABLE_TRAFFIC);
3075    checkeq(ENGINE_SUCCESS,
3076            h1->unknown_command(h, NULL, pkt, add_response),
3077            "Failed to send data traffic command to the services");
3078    checkeq(PROTOCOL_BINARY_RESPONSE_ENOMEM, last_status.load(),
3079            "Data traffic command should have failed with enomem");
3080    cb_free(pkt);
3081
3082    return SUCCESS;
3083}
3084
3085static enum test_result test_cbd_225(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
3086    item *i = NULL;
3087
3088    // get engine startup token
3089    time_t token1 = get_int_stat(h, h1, "ep_startup_time");
3090    check(token1 != 0, "Expected non-zero startup token");
3091
3092    // store some random data
3093    checkeq(ENGINE_SUCCESS,
3094            store(h, h1, NULL, OPERATION_SET,"k1", "v1", &i),
3095            "Failed to fail to store an item.");
3096    h1->release(h, NULL, i);
3097    checkeq(ENGINE_SUCCESS,
3098            store(h, h1, NULL, OPERATION_SET,"k2", "v2", &i),
3099            "Failed to fail to store an item.");
3100    h1->release(h, NULL, i);
3101    wait_for_flusher_to_settle(h, h1);
3102
3103    // check token again, which should be the same as before
3104    time_t token2 = get_int_stat(h, h1, "ep_startup_time");
3105    check(token2 == token1, "Expected the same startup token");
3106
3107    // reload the engine
3108    testHarness.time_travel(10);
3109    testHarness.reload_engine(&h, &h1,
3110                              testHarness.engine_path,
3111                              testHarness.get_current_testcase()->cfg,
3112                              true, false);
3113    wait_for_warmup_complete(h, h1);
3114
3115    // check token, this time we should get a different one
3116    time_t token3 = get_int_stat(h, h1, "ep_startup_time");
3117    check(token3 != token1, "Expected a different startup token");
3118
3119    return SUCCESS;
3120}
3121
3122static enum test_result test_workload_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
3123    const void* cookie = testHarness.create_cookie();
3124    checkeq(ENGINE_SUCCESS,
3125            h1->get_stats(h, cookie, "workload",
3126                          strlen("workload"), add_stats),
3127            "Falied to get workload stats");
3128    testHarness.destroy_cookie(cookie);
3129    int num_read_threads = get_int_stat(h, h1, "ep_workload:num_readers",
3130                                               "workload");
3131    int num_write_threads = get_int_stat(h, h1, "ep_workload:num_writers",
3132                                                "workload");
3133    int num_auxio_threads = get_int_stat(h, h1, "ep_workload:num_auxio",
3134                                                "workload");
3135    int num_nonio_threads = get_int_stat(h, h1, "ep_workload:num_nonio",
3136                                                "workload");
3137    int max_read_threads = get_int_stat(h, h1, "ep_workload:max_readers",
3138                                               "workload");
3139    int max_write_threads = get_int_stat(h, h1, "ep_workload:max_writers",
3140                                                "workload");
3141    int max_auxio_threads = get_int_stat(h, h1, "ep_workload:max_auxio",
3142                                                "workload");
3143    int max_nonio_threads = get_int_stat(h, h1, "ep_workload:max_nonio",
3144                                                "workload");
3145    int num_shards = get_int_stat(h, h1, "ep_workload:num_shards", "workload");
3146    checkeq(4, num_read_threads, "Incorrect number of readers");
3147    // MB-12279: limiting max writers to 4 for DGM bgfetch performance
3148    checkeq(4, num_write_threads, "Incorrect number of writers");
3149    checkeq(1, num_auxio_threads, "Incorrect number of auxio threads");
3150    check(num_nonio_threads > 1 && num_nonio_threads <= 8,
3151          "Incorrect number of nonio threads");
3152    checkeq(4, max_read_threads, "Incorrect limit of readers");
3153    // MB-12279: limiting max writers to 4 for DGM bgfetch performance
3154    checkeq(4, max_write_threads, "Incorrect limit of writers");
3155    checkeq(1, max_auxio_threads, "Incorrect limit of auxio threads");
3156    check(max_nonio_threads > 1 && max_nonio_threads <=8,
3157          "Incorrect limit of nonio threads");
3158    checkeq(5, num_shards, "Incorrect number of shards");
3159    return SUCCESS;
3160}
3161
3162static enum test_result test_max_workload_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
3163    const void* cookie = testHarness.create_cookie();
3164    checkeq(ENGINE_SUCCESS,
3165            h1->get_stats(h, cookie, "workload",
3166                          strlen("workload"), add_stats),
3167            "Failed to get workload stats");
3168    testHarness.destroy_cookie(cookie);
3169    int num_read_threads = get_int_stat(h, h1, "ep_workload:num_readers",
3170                                               "workload");
3171    int num_write_threads = get_int_stat(h, h1, "ep_workload:num_writers",
3172                                                "workload");
3173    int num_auxio_threads = get_int_stat(h, h1, "ep_workload:num_auxio",
3174                                                "workload");
3175    int num_nonio_threads = get_int_stat(h, h1, "ep_workload:num_nonio",
3176                                                "workload");
3177