xref: /3.0.3-GA/ep-engine/tests/ep_testsuite.cc (revision 88917e7b)
1/* -*- MODE: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/*
3 *     Copyright 2010 Couchbase, Inc
4 *
5 *   Licensed under the Apache License, Version 2.0 (the "License");
6 *   you may not use this file except in compliance with the License.
7 *   You may obtain a copy of the License at
8 *
9 *       http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *   Unless required by applicable law or agreed to in writing, software
12 *   distributed under the License is distributed on an "AS IS" BASIS,
13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *   See the License for the specific language governing permissions and
15 *   limitations under the License.
16 */
17
18// Usage: (to repeatedly run just a single test case)
19// make engine_tests IS_LOOP=-L EP_TEST_NUM=3
20
21#include "config.h"
22
23#include <stdio.h>
24#include <stdlib.h>
25#include <string.h>
26#include <sys/stat.h>
27#ifdef _MSC_VER
28#include <direct.h>
29#define mkdir(a, b) _mkdir(a)
30#else
31#include <sys/wait.h>
32#endif
33
34#include <cstdlib>
35#include <iostream>
36#include <map>
37#include <set>
38#include <sstream>
39#include <string>
40#include <vector>
41
42#include <platform/dirutils.h>
43
44#include "atomic.h"
45#include "ep-engine/command_ids.h"
46#include "ep_test_apis.h"
47#include "ep_testsuite.h"
48#include "locks.h"
49#include "mock/mccouch.h"
50#include "mock/mock_upr.h"
51#include "mutex.h"
52
53#include <snappy-c.h>
54
55#ifdef linux
56/* /usr/include/netinet/in.h defines macros from ntohs() to _bswap_nn to
57 * optimize the conversion functions, but the prototypes generate warnings
58 * from gcc. The conversion methods isn't the bottleneck for my app, so
59 * just remove the warnings by undef'ing the optimization ..
60 */
61#undef ntohs
62#undef ntohl
63#undef htons
64#undef htonl
65#endif
66
67// ptr_fun don't like the extern "C" thing for unlock cookie.. cast it
68// away ;)
69typedef void (*UNLOCK_COOKIE_T)(const void *cookie);
70
71extern "C" bool abort_msg(const char *expr, const char *msg, int line);
72
73
74template <typename T>
75static void checkeqfn(T exp, T got, const char *msg, const char *file, const int linenum) {
76    if (exp != got) {
77        std::stringstream ss;
78        ss << "Expected `" << exp << "', got `" << got << "' - " << msg;
79        abort_msg(ss.str().c_str(), file, linenum);
80    }
81}
82
83#define checkeq(a, b, c) checkeqfn(a, b, c, __FILE__, __LINE__)
84
85extern "C" {
86
87#define check(expr, msg) \
88    static_cast<void>((expr) ? 0 : abort_msg(#expr, msg, __LINE__))
89
90#define WHITESPACE_DB "whitespace sucks.db"
91#define MULTI_DISPATCHER_CONFIG \
92    "ht_size=129;ht_locks=3;chk_remover_stime=1;chk_period=60"
93
94struct test_harness testHarness;
95
96class ThreadData {
97public:
98    ThreadData(ENGINE_HANDLE *eh, ENGINE_HANDLE_V1 *ehv1,
99               int e=0) : h(eh), h1(ehv1), extra(e) {}
100    ENGINE_HANDLE    *h;
101    ENGINE_HANDLE_V1 *h1;
102    int               extra;
103};
104
105bool abort_msg(const char *expr, const char *msg, int line) {
106    fprintf(stderr, "%s:%d Test failed: `%s' (%s)\n",
107            __FILE__, line, msg, expr);
108    abort();
109    // UNREACHABLE
110    return false;
111}
112
113static const char *dbname_env;
114static enum test_result rmdb(void)
115{
116    const char *files[] = { WHITESPACE_DB,
117                            "/tmp/test",
118                            "/tmp/mutation.log",
119                            dbname_env,
120                            NULL };
121    int ii = 0;
122    while (files[ii] != NULL) {
123        CouchbaseDirectoryUtilities::rmrf(files[ii]);
124        if (access(files[ii], F_OK) != -1) {
125            std::cerr << "Failed to remove: " << files[ii] << " " << std::endl;
126            return FAIL;
127        }
128        ++ii;
129    }
130
131    return SUCCESS;
132}
133
134static enum test_result skipped_test_function(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
135    (void) h;
136    (void) h1;
137    return SKIPPED;
138}
139
140static bool teardown(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
141    (void)h; (void)h1;
142    vals.clear();
143    return true;
144}
145
146static const void* createTapConn(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
147                                 const char *name) {
148    const void *cookie = testHarness.create_cookie();
149    testHarness.lock_cookie(cookie);
150    TAP_ITERATOR iter = h1->get_tap_iterator(h, cookie, name,
151                                             strlen(name),
152                                             TAP_CONNECT_FLAG_DUMP, NULL,
153                                             0);
154    check(iter != NULL, "Failed to create a tap iterator");
155    return cookie;
156}
157
158static void check_key_value(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
159                            const char* key, const char* val, size_t vlen,
160                            uint16_t vbucket = 0) {
161    item_info info;
162    check(get_item_info(h, h1, &info, key, vbucket), "checking key and value");
163    check(info.nvalue == 1, "info.nvalue != 1");
164    check(vlen == info.value[0].iov_len, "Value length mismatch");
165    check(memcmp(info.value[0].iov_base, val, vlen) == 0, "Data mismatch");
166}
167
168static bool test_setup(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
169    wait_for_warmup_complete(h, h1);
170    wait_for_stat_change(h, h1, "ep_vb_snapshot_total", 0);
171
172    // warmup is complete, notify ep engine that it must now enable
173    // data traffic
174    protocol_binary_request_header *pkt = createPacket(PROTOCOL_BINARY_CMD_ENABLE_TRAFFIC);
175    check(h1->unknown_command(h, NULL, pkt, add_response) == ENGINE_SUCCESS,
176          "Failed to enable data traffic");
177    free(pkt);
178
179    return true;
180}
181
182static enum test_result test_getl(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
183    const char *key = "k1";
184    uint16_t vbucketId = 0;
185    uint32_t expiration = 25;
186
187    getl(h, h1, key, vbucketId, expiration);
188    check(last_status == PROTOCOL_BINARY_RESPONSE_KEY_ENOENT,
189          "expected the key to be missing...");
190    if (last_body != NULL && (strcmp(last_body, "NOT_FOUND") != 0)) {
191        fprintf(stderr, "Should have returned NOT_FOUND. Getl Failed");
192        abort();
193    }
194
195    item *i = NULL;
196    check(store(h, h1, NULL, OPERATION_SET, key, "{\"lock\":\"data\"}",
197                &i, 0, vbucketId, 3600, PROTOCOL_BINARY_DATATYPE_JSON)
198          == ENGINE_SUCCESS, "Failed to store an item.");
199    h1->release(h, NULL, i);
200
201    /* retry getl, should succeed */
202    getl(h, h1, key, vbucketId, expiration);
203    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
204          "Expected to be able to getl on first try");
205    check(strcmp("{\"lock\":\"data\"}", last_body) == 0, "Body was malformed.");
206    check(last_datatype == PROTOCOL_BINARY_DATATYPE_JSON,
207            "Expected datatype to be JSON");
208
209    /* wait 16 seconds */
210    testHarness.time_travel(16);
211
212    /* lock's taken so this should fail */
213    getl(h, h1, key, vbucketId, expiration);
214    check(last_status == PROTOCOL_BINARY_RESPONSE_ETMPFAIL,
215          "Expected to fail getl on second try");
216
217    if (last_body != NULL && (strcmp(last_body, "LOCK_ERROR") != 0)) {
218        fprintf(stderr, "Should have returned LOCK_ERROR. Getl Failed");
219        abort();
220    }
221
222    check(store(h, h1, NULL, OPERATION_SET, key, "lockdata2", &i, 0, vbucketId)
223          != ENGINE_SUCCESS, "Should have failed to store an item.");
224    h1->release(h, NULL, i);
225
226    /* wait another 10 seconds */
227    testHarness.time_travel(10);
228
229    /* retry set, should succeed */
230    check(store(h, h1, NULL, OPERATION_SET, key, "lockdata", &i, 0, vbucketId)
231          == ENGINE_SUCCESS, "Failed to store an item.");
232    h1->release(h, NULL, i);
233
234    /* point to wrong vbucket, to test NOT_MY_VB response */
235    getl(h, h1, key, 10, expiration);
236    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
237          "Should have received not my vbucket response");
238
239    /* acquire lock, should succeed */
240    getl(h, h1, key, vbucketId, expiration);
241    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
242          "Aquire lock should have succeeded");
243    check(last_datatype == PROTOCOL_BINARY_RAW_BYTES,
244            "Expected datatype to be RAW BYTES");
245
246    /* try an incr operation followed by a delete, both of which should fail */
247    uint64_t cas = 0;
248    uint64_t result = 0;
249
250    check(h1->arithmetic(h, NULL, key, 2, true, false, 1, 1, 0,
251                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
252                         0)  == ENGINE_TMPFAIL, "Incr failed");
253
254
255    check(del(h, h1, key, 0, 0) == ENGINE_TMPFAIL, "Delete failed");
256
257
258    /* bug MB 2699 append after getl should fail with ENGINE_TMPFAIL */
259
260    testHarness.time_travel(26);
261
262    char binaryData1[] = "abcdefg\0gfedcba";
263    char binaryData2[] = "abzdefg\0gfedcba";
264
265    check(storeCasVb11(h, h1, NULL, OPERATION_SET, key,
266                       binaryData1, sizeof(binaryData1) - 1, 82758, &i, 0, 0)
267          == ENGINE_SUCCESS,
268          "Failed set.");
269    h1->release(h, NULL, i);
270
271    /* acquire lock, should succeed */
272    getl(h, h1, key, vbucketId, expiration);
273    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
274          "Aquire lock should have succeeded");
275
276    /* append should fail */
277    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, key,
278                       binaryData2, sizeof(binaryData2) - 1, 82758, &i, 0, 0)
279          == ENGINE_TMPFAIL,
280          "Append should fail.");
281    h1->release(h, NULL, i);
282
283    /* bug MB 3252 & MB 3354.
284     * 1. Set a key with an expiry value.
285     * 2. Take a lock on the item before it expires
286     * 3. Wait for the item to expire
287     * 4. Perform a CAS operation, should fail
288     * 5. Perform a set operation, should succeed
289     */
290    const char *ekey = "test_expiry";
291    const char *edata = "some test data here.";
292
293    item *it = NULL;
294
295    check(h1->allocate(h, NULL, &it, ekey, strlen(ekey), strlen(edata), 0, 2,
296          PROTOCOL_BINARY_RAW_BYTES) == ENGINE_SUCCESS, "Allocation Failed");
297
298    item_info info;
299    info.nvalue = 1;
300    if (!h1->get_item_info(h, NULL, it, &info)) {
301        abort();
302    }
303    memcpy(info.value[0].iov_base, edata, strlen(edata));
304
305    check(h1->store(h, NULL, it, &cas, OPERATION_SET, 0) ==
306        ENGINE_SUCCESS, "Failed to Store item");
307    check_key_value(h, h1, ekey, edata, strlen(edata));
308    h1->release(h, NULL, it);
309
310    testHarness.time_travel(3);
311    cas = last_cas;
312
313    /* cas should fail */
314    check(storeCasVb11(h, h1, NULL, OPERATION_CAS, ekey,
315                       binaryData1, sizeof(binaryData1) - 1, 82758, &i, cas, 0)
316          != ENGINE_SUCCESS,
317          "CAS succeeded.");
318    h1->release(h, NULL, i);
319
320    /* but a simple store should succeed */
321    check(store(h, h1, NULL, OPERATION_SET, ekey, edata, &i, 0, vbucketId)
322          == ENGINE_SUCCESS, "Failed to store an item.");
323    h1->release(h, NULL, i);
324    return SUCCESS;
325}
326
327static enum test_result test_unl(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
328
329    const char *key = "k2";
330    uint16_t vbucketId = 0;
331
332    unl(h, h1, key, vbucketId);
333    check(last_status != PROTOCOL_BINARY_RESPONSE_SUCCESS,
334          "expected the key to be missing...");
335
336    item *i = NULL;
337    check(store(h, h1, NULL, OPERATION_SET, key, "lockdata", &i, 0, vbucketId)
338          == ENGINE_SUCCESS, "Failed to store an item.");
339    h1->release(h, NULL, i);
340
341    /* getl, should succeed */
342    getl(h, h1, key, vbucketId, 0);
343    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
344          "Expected to be able to getl on first try");
345
346    /* save the returned cas value for later */
347    uint64_t cas = last_cas;
348
349    /* lock's taken unlocking with a random cas value should fail */
350    unl(h, h1, key, vbucketId);
351    check(last_status == PROTOCOL_BINARY_RESPONSE_ETMPFAIL,
352          "Expected to fail getl on second try");
353
354    if (last_body != NULL && (strcmp(last_body, "UNLOCK_ERROR") != 0)) {
355        fprintf(stderr, "Should have returned UNLOCK_ERROR. Unl Failed");
356        abort();
357    }
358
359    unl(h, h1, key, vbucketId, cas);
360    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
361          "Expected to succed unl with correct cas");
362
363    /* acquire lock, should succeed */
364    getl(h, h1, key, vbucketId, 0);
365
366    /* wait 16 seconds */
367    testHarness.time_travel(16);
368
369    /* lock has expired, unl should fail */
370    unl(h, h1, key, vbucketId, last_cas);
371    check(last_status == PROTOCOL_BINARY_RESPONSE_ETMPFAIL,
372          "Expected to fail unl on lock timeout");
373
374    return SUCCESS;
375}
376
377static enum test_result test_wrong_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
378                                               ENGINE_STORE_OPERATION op) {
379    item *i = NULL;
380    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
381    uint64_t cas = 11;
382    if (op == OPERATION_ADD) {
383        // Add operation with cas != 0 doesn't make sense
384        cas = 0;
385    }
386    check(store(h, h1, NULL, op,
387                "key", "somevalue", &i, cas, 1) == ENGINE_NOT_MY_VBUCKET,
388        "Expected not_my_vbucket");
389    h1->release(h, NULL, i);
390    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
391    return SUCCESS;
392}
393
394static enum test_result test_pending_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
395                                                 ENGINE_STORE_OPERATION op) {
396    const void *cookie = testHarness.create_cookie();
397    testHarness.set_ewouldblock_handling(cookie, false);
398    item *i = NULL;
399    check(set_vbucket_state(h, h1, 1, vbucket_state_pending), "Failed to set vbucket state.");
400    check(verify_vbucket_state(h, h1, 1, vbucket_state_pending), "Bucket state was not set to pending.");
401    uint64_t cas = 11;
402    if (op == OPERATION_ADD) {
403        // Add operation with cas != 0 doesn't make sense..
404        cas = 0;
405    }
406    check(store(h, h1, cookie, op,
407                "key", "somevalue", &i, cas, 1) == ENGINE_EWOULDBLOCK,
408        "Expected woodblock");
409    h1->release(h, NULL, i);
410    testHarness.destroy_cookie(cookie);
411    return SUCCESS;
412}
413
414static enum test_result test_replica_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
415                                                 ENGINE_STORE_OPERATION op) {
416    item *i = NULL;
417    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set vbucket state.");
418    check(verify_vbucket_state(h, h1, 1, vbucket_state_replica), "Bucket state was not set to replica.");
419    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
420
421    uint64_t cas = 11;
422    if (op == OPERATION_ADD) {
423        // performing add with a CAS != 0 doesn't make sense...
424        cas = 0;
425    }
426    check(store(h, h1, NULL, op,
427                "key", "somevalue", &i, cas, 1) == ENGINE_NOT_MY_VBUCKET,
428        "Expected not my vbucket");
429    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
430    h1->release(h, NULL, i);
431    return SUCCESS;
432}
433
434//
435// ----------------------------------------------------------------------
436// The actual tests are below.
437// ----------------------------------------------------------------------
438//
439
440static enum test_result test_get_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
441    check(verify_key(h, h1, "k") == ENGINE_KEY_ENOENT, "Expected miss.");
442    return SUCCESS;
443}
444
445static enum test_result test_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
446    item *i = NULL;
447    check(ENGINE_SUCCESS ==
448          store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
449          "Error setting.");
450    h1->release(h, NULL, i);
451    return SUCCESS;
452}
453
454struct handle_pair {
455    ENGINE_HANDLE *h;
456    ENGINE_HANDLE_V1 *h1;
457};
458
459extern "C" {
460    static void conc_del_set_thread(void *arg) {
461        struct handle_pair *hp = static_cast<handle_pair *>(arg);
462        item *it = NULL;
463
464        for (int i = 0; i < 5000; ++i) {
465            store(hp->h, hp->h1, NULL, OPERATION_ADD,
466                  "key", "somevalue", &it);
467            hp->h1->release(hp->h, NULL, it);
468            usleep(10);
469            checkeq(ENGINE_SUCCESS,
470                    store(hp->h, hp->h1, NULL, OPERATION_SET,
471                          "key", "somevalue", &it),
472                    "Error setting.");
473            hp->h1->release(hp->h, NULL, it);
474            usleep(10);
475            // Ignoring the result here -- we're racing.
476            del(hp->h, hp->h1, "key", 0, 0);
477            usleep(10);
478        }
479    }
480}
481
482static enum test_result test_conc_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
483
484    const int n_threads = 8;
485    cb_thread_t threads[n_threads];
486    struct handle_pair hp = {h, h1};
487
488    wait_for_persisted_value(h, h1, "key", "value1");
489
490    for (int i = 0; i < n_threads; i++) {
491        int r = cb_create_thread(&threads[i], conc_del_set_thread, &hp, 0);
492        cb_assert(r == 0);
493    }
494
495    for (int i = 0; i < n_threads; i++) {
496        int r = cb_join_thread(threads[i]);
497        cb_assert(r == 0);
498    }
499
500    wait_for_flusher_to_settle(h, h1);
501
502    testHarness.reload_engine(&h, &h1,
503                              testHarness.engine_path,
504                              testHarness.get_current_testcase()->cfg,
505                              true, false);
506    wait_for_warmup_complete(h, h1);
507
508    cb_assert(0 == get_int_stat(h, h1, "ep_warmed_dups"));
509
510    return SUCCESS;
511}
512
513static enum test_result test_set_get_hit(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
514    item *i = NULL;
515    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
516          "store failure");
517    check_key_value(h, h1, "key", "somevalue", 9);
518    h1->release(h, NULL, i);
519    return SUCCESS;
520}
521
522static enum test_result test_set_get_hit_bin(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
523    char binaryData[] = "abcdefg\0gfedcba";
524    cb_assert(sizeof(binaryData) != strlen(binaryData));
525
526    item *i = NULL;
527    check(ENGINE_SUCCESS ==
528          storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
529                       binaryData, sizeof(binaryData), 82758, &i, 0, 0),
530          "Failed to set.");
531    h1->release(h, NULL, i);
532    check_key_value(h, h1, "key", binaryData, sizeof(binaryData));
533    return SUCCESS;
534}
535
536static enum test_result test_set_with_cas_non_existent(ENGINE_HANDLE *h,
537                                                       ENGINE_HANDLE_V1 *h1) {
538    const char *key = "test_expiry_flush";
539    item *i = NULL;
540
541    check(h1->allocate(h, NULL, &i, key, strlen(key), 10, 0, 0,
542          PROTOCOL_BINARY_RAW_BYTES) == ENGINE_SUCCESS, "Allocation failed.");
543
544    Item *it = reinterpret_cast<Item*>(i);
545    it->setCas(1234);
546
547    uint64_t cas = 0;
548    check(h1->store(h, NULL, i, &cas, OPERATION_SET, 0) == ENGINE_KEY_ENOENT,
549          "Expected not found");
550    h1->release(h, NULL, i);
551
552    return SUCCESS;
553}
554
555static enum test_result test_set_change_flags(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
556    item *i = NULL;
557    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
558          "Failed to set.");
559    h1->release(h, NULL, i);
560
561    item_info info;
562    uint32_t flags = 828258;
563    check(get_item_info(h, h1, &info, "key"), "Failed to get value.");
564    cb_assert(info.flags != flags);
565
566    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
567                       "newvalue", strlen("newvalue"), flags, &i, 0, 0) == ENGINE_SUCCESS,
568          "Failed to set again.");
569    h1->release(h, NULL, i);
570
571    check(get_item_info(h, h1, &info, "key"), "Failed to get value.");
572
573    return info.flags == flags ? SUCCESS : FAIL;
574}
575
576static enum test_result test_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
577    item *i = NULL;
578    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
579          "Failed to do initial set.");
580    h1->release(h, NULL, i);
581    check(store(h, h1, NULL, OPERATION_CAS, "key", "failcas", &i) != ENGINE_SUCCESS,
582          "Failed to fail initial CAS.");
583    h1->release(h, NULL, i);
584    check_key_value(h, h1, "key", "somevalue", 9);
585
586    check(h1->get(h, NULL, &i, "key", 3, 0) == ENGINE_SUCCESS,
587          "Failed to get value.");
588
589    item_info info;
590    info.nvalue = 1;
591    check(h1->get_item_info(h, NULL, i, &info), "Failed to get item info.");
592    h1->release(h, NULL, i);
593
594    check(store(h, h1, NULL, OPERATION_CAS, "key", "winCas", &i,
595                info.cas) == ENGINE_SUCCESS,
596          "Failed to store CAS");
597    h1->release(h, NULL, i);
598    check_key_value(h, h1, "key", "winCas", 6);
599
600    uint64_t cval = 99999;
601    check(store(h, h1, NULL, OPERATION_CAS, "non-existing", "winCas", &i,
602                cval) == ENGINE_KEY_ENOENT,
603          "CAS for non-existing key returned the wrong error code");
604    h1->release(h, NULL, i);
605    return SUCCESS;
606}
607
608static enum test_result test_add(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
609    item *i = NULL;
610    check(store(h, h1, NULL, OPERATION_ADD,"key", "somevalue", &i) == ENGINE_SUCCESS,
611          "Failed to add value.");
612    h1->release(h, NULL, i);
613    check(store(h, h1, NULL, OPERATION_ADD,"key", "somevalue", &i) == ENGINE_NOT_STORED,
614          "Failed to fail to re-add value.");
615    h1->release(h, NULL, i);
616
617    // This aborts on failure.
618    check_key_value(h, h1, "key", "somevalue", 9);
619
620    // Expiration above was an hour, so let's go to The Future
621    testHarness.time_travel(3800);
622
623    check(store(h, h1, NULL, OPERATION_ADD,"key", "newvalue", &i) == ENGINE_SUCCESS,
624          "Failed to add value again.");
625    h1->release(h, NULL, i);
626    check_key_value(h, h1, "key", "newvalue", 8);
627    return SUCCESS;
628}
629
630static enum test_result test_add_add_with_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
631    item *i = NULL;
632    check(store(h, h1, NULL, OPERATION_ADD, "key",
633                "somevalue", &i) == ENGINE_SUCCESS,
634          "Failed set.");
635    check_key_value(h, h1, "key", "somevalue", 9);
636    item_info info;
637    info.nvalue = 1;
638    info.nvalue = 1;
639    check(h1->get_item_info(h, NULL, i, &info) == true,
640          "Should be able to get info");
641
642    item *i2 = NULL;
643    ENGINE_ERROR_CODE ret;
644    check((ret = store(h, h1, NULL, OPERATION_ADD, "key",
645                       "somevalue", &i2, info.cas)) == ENGINE_KEY_EEXISTS,
646          "Should not be able to add the key two times");
647
648    h1->release(h, NULL, i);
649    h1->release(h, NULL, i2);
650    return SUCCESS;
651}
652
653static enum test_result test_replace(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
654    item *i = NULL;
655    check(store(h, h1, NULL, OPERATION_REPLACE,"key", "somevalue", &i) != ENGINE_SUCCESS,
656          "Failed to fail to replace non-existing value.");
657    h1->release(h, NULL, i);
658    check(store(h, h1, NULL, OPERATION_SET,"key", "somevalue", &i) == ENGINE_SUCCESS,
659          "Failed to set value.");
660    h1->release(h, NULL, i);
661    check(store(h, h1, NULL, OPERATION_REPLACE,"key", "somevalue", &i) == ENGINE_SUCCESS,
662          "Failed to replace existing value.");
663    h1->release(h, NULL, i);
664    check_key_value(h, h1, "key", "somevalue", 9);
665    return SUCCESS;
666}
667
668static enum test_result test_replace_with_eviction(ENGINE_HANDLE *h,
669                                                   ENGINE_HANDLE_V1 *h1) {
670    item *i = NULL;
671    check(store(h, h1, NULL, OPERATION_SET,"key", "somevalue", &i) == ENGINE_SUCCESS,
672          "Failed to set value.");
673    h1->release(h, NULL, i);
674    wait_for_flusher_to_settle(h, h1);
675    evict_key(h, h1, "key");
676    int numBgFetched = get_int_stat(h, h1, "ep_bg_fetched");
677
678    check(store(h, h1, NULL, OPERATION_REPLACE,"key", "somevalue1", &i) == ENGINE_SUCCESS,
679          "Failed to replace existing value.");
680
681    check(h1->get_stats(h, NULL, NULL, 0, add_stats) == ENGINE_SUCCESS,
682          "Failed to get stats.");
683    std::string eviction_policy = vals.find("ep_item_eviction_policy")->second;
684    if (eviction_policy == "full_eviction") {
685        numBgFetched++;
686    }
687
688    check(get_int_stat(h, h1, "ep_bg_fetched") == numBgFetched,
689          "Bg fetched value didn't match");
690
691    h1->release(h, NULL, i);
692    check_key_value(h, h1, "key", "somevalue1", 10);
693    return SUCCESS;
694}
695
696static enum test_result test_incr_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
697    uint64_t cas = 0, result = 0;
698    h1->arithmetic(h, NULL, "key", 3, true, false, 1, 0, 0,
699                   &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
700                   0);
701    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected to not find key");
702    return SUCCESS;
703}
704
705static enum test_result test_incr_default(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
706    uint64_t cas = 0, result = 0;
707    check(h1->arithmetic(h, NULL, "key", 3, true, true, 1, 1, 0,
708                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
709                         0) == ENGINE_SUCCESS,
710          "Failed first arith");
711    check(result == 1, "Failed result verification.");
712
713    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
714                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
715                         0) == ENGINE_SUCCESS,
716          "Failed second arith.");
717    check(result == 2, "Failed second result verification.");
718
719    check(h1->arithmetic(h, NULL, "key", 3, true, true, 1, 1, 0,
720                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
721                         0) == ENGINE_SUCCESS,
722          "Failed third arith.");
723    check(result == 3, "Failed third result verification.");
724
725    check_key_value(h, h1, "key", "3", 1);
726    return SUCCESS;
727}
728
729static enum test_result test_append(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
730    item *i = NULL;
731
732    // MB-11332: append on non-existing key should return NOT_STORED
733    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key",
734                       "foo\r\n", 5, 82758, &i, 0, 0)
735          == ENGINE_NOT_STORED,
736          "MB-11332: Failed append.");
737
738    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
739                       "\r\n", 2, 82758, &i, 0, 0)
740          == ENGINE_SUCCESS,
741          "Failed set.");
742    h1->release(h, NULL, i);
743
744    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key",
745                       "foo\r\n", 5, 82758, &i, 0, 0)
746          == ENGINE_SUCCESS,
747          "Failed append.");
748    h1->release(h, NULL, i);
749
750    check_key_value(h, h1, "key", "\r\nfoo\r\n", 7);
751
752    char binaryData1[] = "abcdefg\0gfedcba\r\n";
753    char binaryData2[] = "abzdefg\0gfedcba\r\n";
754    size_t dataSize = 20*1024*1024;
755    char *bigBinaryData3 = new char[dataSize];
756    memset(bigBinaryData3, '\0', dataSize);
757
758    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
759                       binaryData1, sizeof(binaryData1) - 1, 82758, &i, 0, 0)
760          == ENGINE_SUCCESS,
761          "Failed set.");
762    h1->release(h, NULL, i);
763
764    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key",
765                       bigBinaryData3, dataSize, 82758, &i, 0, 0)
766          == ENGINE_E2BIG,
767          "Expected append failure.");
768    h1->release(h, NULL, i);
769    delete[] bigBinaryData3;
770
771    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key",
772                       binaryData2, sizeof(binaryData2) - 1, 82758, &i, 0, 0)
773          == ENGINE_SUCCESS,
774          "Failed append.");
775    h1->release(h, NULL, i);
776
777    std::string expected;
778    expected.append(binaryData1, sizeof(binaryData1) - 1);
779    expected.append(binaryData2, sizeof(binaryData2) - 1);
780
781    check_key_value(h, h1, "key", expected.data(), expected.length());
782    return SUCCESS;
783}
784
785static enum test_result test_prepend(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
786    item *i = NULL;
787
788    // MB-11332: prepend on non-existing key should return NOT_STORED
789    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key",
790                       "foo\r\n", 5, 82758, &i, 0, 0)
791          == ENGINE_NOT_STORED,
792          "MB-11332: Failed prepend.");
793
794
795    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
796                       "\r\n", 2, 82758, &i, 0, 0)
797          == ENGINE_SUCCESS,
798          "Failed set.");
799    h1->release(h, NULL, i);
800
801    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key",
802                       "foo\r\n", 5, 82758, &i, 0, 0)
803          == ENGINE_SUCCESS,
804          "Failed append.");
805    h1->release(h, NULL, i);
806
807    check_key_value(h, h1, "key", "foo\r\n\r\n", 7);
808
809    char binaryData1[] = "abcdefg\0gfedcba\r\n";
810    char binaryData2[] = "abzdefg\0gfedcba\r\n";
811    size_t dataSize = 20*1024*1024;
812    char *bigBinaryData3 = new char[dataSize];
813    memset(bigBinaryData3, '\0', dataSize);
814
815    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
816                       binaryData1, sizeof(binaryData1) - 1, 82758, &i, 0, 0)
817          == ENGINE_SUCCESS,
818          "Failed set.");
819    h1->release(h, NULL, i);
820
821    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key",
822                       bigBinaryData3, dataSize, 82758, &i, 0, 0)
823          == ENGINE_E2BIG,
824          "Expected prepend failure.");
825    h1->release(h, NULL, i);
826    delete[] bigBinaryData3;
827
828    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key",
829                       binaryData2, sizeof(binaryData2) - 1, 82758, &i, 0, 0)
830          == ENGINE_SUCCESS,
831          "Failed append.");
832    h1->release(h, NULL, i);
833
834    std::string expected;
835    expected.append(binaryData2, sizeof(binaryData2) - 1);
836    expected.append(binaryData1, sizeof(binaryData1) - 1);
837
838    check_key_value(h, h1, "key", expected.data(), expected.length());
839    return SUCCESS;
840}
841
842static enum test_result test_append_compressed(ENGINE_HANDLE *h,
843                                               ENGINE_HANDLE_V1 *h1) {
844
845    item *i = NULL;
846
847    size_t len = snappy_max_compressed_length(2);
848    char *newBuf = (char *) malloc (len);
849    snappy_compress("\r\n", 2, newBuf, &len);
850    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key1",
851                       (const char *)newBuf, len, 82758, &i, 0, 0, 3600,
852                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
853          == ENGINE_SUCCESS, "Failed set.");
854    h1->release(h, NULL, i);
855    free (newBuf);
856
857    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key1",
858                       "foo\r\n", 5, 82758, &i, 0, 0)
859          == ENGINE_SUCCESS,
860          "Failed append uncompressed to compressed.");
861    h1->release(h, NULL, i);
862
863    size_t len1 = snappy_max_compressed_length(7);
864    char *newBuf1 = (char *) malloc (len1);
865    snappy_compress("\r\nfoo\r\n", 7, newBuf1, &len1);
866
867    item_info info;
868    check(get_item_info(h, h1, &info, "key1", 0), "checking key and value");
869    check(info.nvalue == 1, "info.nvalue != 1");
870    check(len1 == info.value[0].iov_len, "Value length mismatch");
871    check(memcmp(info.value[0].iov_base, newBuf1, len1) == 0, "Data mismatch");
872    check(info.datatype == 0x02, "Datatype mismatch");
873    free (newBuf1);
874
875    len = snappy_max_compressed_length(3);
876    newBuf = (char *) malloc (len);
877    snappy_compress("bar", 3, newBuf, &len);
878    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key1",
879                       (const char*)newBuf, len, 82758, &i, 0, 0, 3600,
880                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
881            == ENGINE_SUCCESS,
882            "Failed append compressed to compressed.");
883    h1->release(h, NULL, i);
884    free (newBuf);
885
886    len1 = snappy_max_compressed_length(10);
887    newBuf1 = (char *) malloc (len1);
888    snappy_compress("\r\nfoo\r\nbar", 10, newBuf1, &len1);
889
890    check(get_item_info(h, h1, &info, "key1", 0), "checking key and value");
891    check(info.nvalue == 1, "info.nvalue != 1");
892    check(len1 == info.value[0].iov_len, "Value length mismatch");
893    check(memcmp(info.value[0].iov_base, newBuf1, len1) == 0, "Data mismatch");
894    check(info.datatype == 0x02, "Datatype mismatch");
895    free (newBuf1);
896
897    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key2",
898                       "foo", 3, 82758, &i, 0, 0, 3600,
899                       PROTOCOL_BINARY_RAW_BYTES)
900          == ENGINE_SUCCESS, "Failed set.");
901    h1->release(h, NULL, i);
902
903    len = snappy_max_compressed_length(3);
904    newBuf = (char *) malloc (len);
905    snappy_compress("bar", 3, newBuf, &len);
906    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key2",
907                       newBuf, len, 82758, &i, 0, 0, 3600,
908                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
909            == ENGINE_SUCCESS,
910            "Failed append compressed to uncompressed.");
911    h1->release(h, NULL, i);
912    free (newBuf);
913
914    check(get_item_info(h, h1, &info, "key2", 0), "checking key and value");
915    check(info.nvalue == 1, "info.nvalue != 1");
916    check(info.value[0].iov_len == 6, "Value length mismatch");
917    check(memcmp(info.value[0].iov_base, "foobar", 6) == 0, "Data mismatch");
918    check(info.datatype == 0x00, "Datatype mismatch");
919
920    return SUCCESS;
921}
922
923static enum test_result test_prepend_compressed(ENGINE_HANDLE *h,
924                                               ENGINE_HANDLE_V1 *h1) {
925
926    item *i = NULL;
927
928    size_t len = snappy_max_compressed_length(2);
929    char *newBuf = (char *) malloc (len);
930    snappy_compress("\r\n", 2, newBuf, &len);
931    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key1",
932                       (const char *)newBuf, len, 82758, &i, 0, 0, 3600,
933                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
934          == ENGINE_SUCCESS, "Failed set.");
935    h1->release(h, NULL, i);
936    free (newBuf);
937
938    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key1",
939                       "foo\r\n", 5, 82758, &i, 0, 0)
940          == ENGINE_SUCCESS,
941          "Failed prepend uncompressed to compressed.");
942    h1->release(h, NULL, i);
943
944    size_t len1 = snappy_max_compressed_length(7);
945    char *newBuf1 = (char *) malloc (len1);
946    snappy_compress("foo\r\n\r\n", 7, newBuf1, &len1);
947
948    item_info info;
949    check(get_item_info(h, h1, &info, "key1", 0), "checking key and value");
950    check(info.nvalue == 1, "info.nvalue != 1");
951    check(len1 == info.value[0].iov_len, "Value length mismatch");
952    check(memcmp(info.value[0].iov_base, newBuf1, len1) == 0, "Data mismatch");
953    check(info.datatype == 0x02, "Datatype mismatch");
954    free (newBuf1);
955
956    len = snappy_max_compressed_length(3);
957    newBuf = (char *) malloc (len);
958    snappy_compress("bar", 3, newBuf, &len);
959    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key1",
960                       (const char*)newBuf, len, 82758, &i, 0, 0, 3600,
961                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
962            == ENGINE_SUCCESS,
963            "Failed prepend compressed to compressed.");
964    h1->release(h, NULL, i);
965    free (newBuf);
966
967    len1 = snappy_max_compressed_length(10);
968    newBuf1 = (char *) malloc (len1);
969    snappy_compress("barfoo\r\n\r\n", 10, newBuf1, &len1);
970
971    check(get_item_info(h, h1, &info, "key1", 0), "checking key and value");
972    check(info.nvalue == 1, "info.nvalue != 1");
973    check(len1 == info.value[0].iov_len, "Value length mismatch");
974    check(memcmp(info.value[0].iov_base, newBuf1, len1) == 0, "Data mismatch");
975    check(info.datatype == 0x02, "Datatype mismatch");
976    free (newBuf1);
977
978    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key2",
979                       "foo", 3, 82758, &i, 0, 0, 3600,
980                       PROTOCOL_BINARY_RAW_BYTES)
981          == ENGINE_SUCCESS, "Failed set.");
982    h1->release(h, NULL, i);
983
984    len = snappy_max_compressed_length(3);
985    newBuf = (char *) malloc (len);
986    snappy_compress("bar", 3, newBuf, &len);
987    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key2",
988                       newBuf, len, 82758, &i, 0, 0, 3600,
989                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
990            == ENGINE_SUCCESS,
991            "Failed prepend compressed to uncompressed.");
992    h1->release(h, NULL, i);
993    free (newBuf);
994
995    check(get_item_info(h, h1, &info, "key2", 0), "checking key and value");
996    check(info.nvalue == 1, "info.nvalue != 1");
997    check(info.value[0].iov_len == 6, "Value length mismatch");
998    check(memcmp(info.value[0].iov_base, "barfoo", 6) == 0, "Data mismatch");
999    check(info.datatype == 0x00, "Datatype mismatch");
1000
1001    return SUCCESS;
1002}
1003
1004static enum test_result test_incr(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1005    uint64_t cas = 0, result = 0;
1006    item *i = NULL;
1007    check(store(h, h1, NULL, OPERATION_ADD,"key", "1", &i) == ENGINE_SUCCESS,
1008          "Failed to add value.");
1009    h1->release(h, NULL, i);
1010
1011    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
1012                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1013                         0) == ENGINE_SUCCESS,
1014          "Failed to incr value.");
1015
1016    check_key_value(h, h1, "key", "2", 1);
1017    return SUCCESS;
1018}
1019
1020static enum test_result test_bug2799(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1021    uint64_t cas = 0, result = 0;
1022    item *i = NULL;
1023    check(store(h, h1, NULL, OPERATION_ADD, "key", "1", &i) == ENGINE_SUCCESS,
1024          "Failed to add value.");
1025    h1->release(h, NULL, i);
1026
1027    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
1028                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1029                         0) == ENGINE_SUCCESS,
1030          "Failed to incr value.");
1031
1032    check_key_value(h, h1, "key", "2", 1);
1033
1034    testHarness.time_travel(3617);
1035
1036    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1037    return SUCCESS;
1038}
1039
1040static enum test_result test_flush(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1041    item *i = NULL;
1042    // First try to delete something we know to not be there.
1043    check(del(h, h1, "key", 0, 0) == ENGINE_KEY_ENOENT, "Failed to fail initial delete.");
1044    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1045          "Failed set.");
1046    h1->release(h, NULL, i);
1047    check_key_value(h, h1, "key", "somevalue", 9);
1048    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS,
1049          "Failed to flush");
1050    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1051
1052    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1053          "Failed post-flush set.");
1054    h1->release(h, NULL, i);
1055    check_key_value(h, h1, "key", "somevalue", 9);
1056
1057    return SUCCESS;
1058}
1059
1060static enum test_result test_flush_disabled(ENGINE_HANDLE *h,
1061                                            ENGINE_HANDLE_V1 *h1) {
1062    item *i = NULL;
1063    // start an engine with disabled flush, the flush() should be noop and
1064    // we expect to see the key after flush()
1065
1066    // store a key and check its existence
1067    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1068          "Failed set.");
1069    h1->release(h, NULL, i);
1070    check_key_value(h, h1, "key", "somevalue", 9);
1071    // expect error msg engine does not support operation
1072    check(h1->flush(h, NULL, 0) == ENGINE_ENOTSUP, "Flush should be disabled");
1073    //check the key
1074    check(ENGINE_SUCCESS == verify_key(h, h1, "key"), "Expected key");
1075
1076    // restart engine with flush enabled and redo the test, we expect flush to succeed
1077    std::string param = "flushall_enabled=false";
1078    std::string config = testHarness.get_current_testcase()->cfg;
1079    size_t found = config.find(param);
1080    if(found != config.npos) {
1081        config.replace(found, param.size(), "flushall_enabled=true");
1082    }
1083    testHarness.reload_engine(&h, &h1,
1084                              testHarness.engine_path,
1085                              config.c_str(),
1086                              true, false);
1087    wait_for_warmup_complete(h, h1);
1088
1089    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS, "Flush should be enabled");
1090
1091    //expect missing key
1092    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1093
1094    return SUCCESS;
1095}
1096
1097static enum test_result test_flush_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1098    item *i = NULL;
1099    int mem_used = get_int_stat(h, h1, "mem_used");
1100    int overhead = get_int_stat(h, h1, "ep_overhead");
1101    int cacheSize = get_int_stat(h, h1, "ep_total_cache_size");
1102    int nonResident = get_int_stat(h, h1, "ep_num_non_resident");
1103
1104    int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
1105    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1106          "Failed set.");
1107    h1->release(h, NULL, i);
1108    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i) == ENGINE_SUCCESS,
1109          "Failed set.");
1110    h1->release(h, NULL, i);
1111    testHarness.time_travel(65);
1112    wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
1113
1114    check(ENGINE_SUCCESS == verify_key(h, h1, "key"), "Expected key");
1115    check(ENGINE_SUCCESS == verify_key(h, h1, "key2"), "Expected key2");
1116
1117    check_key_value(h, h1, "key", "somevalue", 9);
1118    check_key_value(h, h1, "key2", "somevalue", 9);
1119
1120    int mem_used2 = get_int_stat(h, h1, "mem_used");
1121    int overhead2 = get_int_stat(h, h1, "ep_overhead");
1122    int cacheSize2 = get_int_stat(h, h1, "ep_total_cache_size");
1123
1124    cb_assert(mem_used2 > mem_used);
1125    // "mem_used2 - overhead2" (i.e., ep_kv_size) should be greater than the hashtable cache size
1126    // due to the checkpoint overhead
1127    cb_assert(mem_used2 - overhead2 > cacheSize2);
1128
1129    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS, "Failed to flush");
1130    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1131    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key2"), "Expected missing key");
1132
1133    wait_for_flusher_to_settle(h, h1);
1134
1135    mem_used2 = get_int_stat(h, h1, "mem_used");
1136    overhead2 = get_int_stat(h, h1, "ep_overhead");
1137    cacheSize2 = get_int_stat(h, h1, "ep_total_cache_size");
1138    int nonResident2 = get_int_stat(h, h1, "ep_num_non_resident");
1139
1140    cb_assert(mem_used2 == mem_used);
1141    cb_assert(overhead2 == overhead);
1142    cb_assert(nonResident2 == nonResident);
1143    cb_assert(cacheSize2 == cacheSize);
1144
1145    return SUCCESS;
1146}
1147
1148static enum test_result test_flush_multiv(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1149    item *i = NULL;
1150    check(set_vbucket_state(h, h1, 2, vbucket_state_active), "Failed to set vbucket state.");
1151    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1152          "Failed set.");
1153    h1->release(h, NULL, i);
1154    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i,
1155                0, 2) == ENGINE_SUCCESS,
1156          "Failed set in vb2.");
1157    h1->release(h, NULL, i);
1158
1159    check(ENGINE_SUCCESS == verify_key(h, h1, "key"), "Expected key");
1160    check(ENGINE_SUCCESS == verify_key(h, h1, "key2", 2), "Expected key2");
1161
1162    check_key_value(h, h1, "key", "somevalue", 9);
1163    check_key_value(h, h1, "key2", "somevalue", 9, 2);
1164
1165    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS, "Failed to flush");
1166
1167    vals.clear();
1168    check(h1->get_stats(h, NULL, NULL, 0, add_stats) == ENGINE_SUCCESS,
1169          "Failed to get stats.");
1170    check(vals.find("ep_flush_all") != vals.end(), "Failed to get the status of flush_all");
1171
1172    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1173    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key2", 2), "Expected missing key");
1174
1175    return SUCCESS;
1176}
1177
1178static int checkCurrItemsAfterShutdown(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1179                                       int numItems2Load, bool shutdownForce) {
1180    std::vector<std::string> keys;
1181    for (int index = 0; index < numItems2Load; ++index) {
1182        std::stringstream s;
1183        s << "keys_2_load-" << index;
1184        std::string key(s.str());
1185        keys.push_back(key);
1186    }
1187
1188    check(get_int_stat(h, h1, "ep_total_persisted") == 0,
1189          "Expected ep_total_persisted equals 0");
1190    check(get_int_stat(h, h1, "curr_items") == 0,
1191          "Expected curr_items equals 0");
1192
1193    // stop flusher before loading new items
1194    protocol_binary_request_header *pkt = createPacket(PROTOCOL_BINARY_CMD_STOP_PERSISTENCE);
1195    check(h1->unknown_command(h, NULL, pkt, add_response) == ENGINE_SUCCESS,
1196          "CMD_STOP_PERSISTENCE failed!");
1197    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
1198          "Failed to stop persistence!");
1199    free(pkt);
1200
1201    std::vector<std::string>::iterator itr;
1202    for (itr = keys.begin(); itr != keys.end(); ++itr) {
1203        item *i;
1204        check(store(h, h1, NULL, OPERATION_SET, itr->c_str(), "oracle", &i, 0, 0)
1205              == ENGINE_SUCCESS, "Failed to store a value");
1206        h1->release(h, NULL, i);
1207    }
1208
1209    check(get_int_stat(h, h1, "ep_total_persisted") == 0,
1210          "Incorrect ep_total_persisted, expected 0");
1211    std::stringstream ss;
1212    ss << "Incorrect curr_items, expected " << numItems2Load;
1213    std::string errmsg(ss.str());
1214    check(get_int_stat(h, h1, "curr_items") == numItems2Load,
1215          errmsg.c_str());
1216
1217    // resume flusher before shutdown + warmup
1218    pkt = createPacket(PROTOCOL_BINARY_CMD_START_PERSISTENCE);
1219    check(h1->unknown_command(h, NULL, pkt, add_response) == ENGINE_SUCCESS,
1220          "CMD_START_PERSISTENCE failed!");
1221    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
1222          "Failed to start persistence!");
1223    free(pkt);
1224
1225    // shutdown engine force and restart
1226    testHarness.reload_engine(&h, &h1,
1227                              testHarness.engine_path,
1228                              testHarness.get_current_testcase()->cfg,
1229                              true, shutdownForce);
1230    wait_for_warmup_complete(h, h1);
1231    return get_int_stat(h, h1, "curr_items");
1232}
1233
1234static enum test_result test_flush_shutdown_force(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1235    int numItems2load = 3000;
1236    bool shutdownForce = true;
1237    int currItems = checkCurrItemsAfterShutdown(h, h1, numItems2load, shutdownForce);
1238    check (currItems <= numItems2load,
1239           "Number of curr items should be <= 3000, unless previous "
1240           "shutdown force had to wait for the flusher");
1241    return SUCCESS;
1242}
1243
1244static enum test_result test_flush_shutdown_noforce(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1245    int numItems2load = 3000;
1246    bool shutdownForce = false;
1247    int currItems = checkCurrItemsAfterShutdown(h, h1, numItems2load, shutdownForce);
1248    check (currItems == numItems2load,
1249           "Number of curr items should be equal to 3000, unless previous "
1250           "shutdown did not wait for the flusher");
1251    return SUCCESS;
1252}
1253
1254static enum test_result test_flush_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1255    item *i = NULL;
1256    // First try to delete something we know to not be there.
1257    check(del(h, h1, "key", 0, 0) == ENGINE_KEY_ENOENT, "Failed to fail initial delete.");
1258    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1259          "Failed set.");
1260    h1->release(h, NULL, i);
1261    check_key_value(h, h1, "key", "somevalue", 9);
1262
1263    // Restart once to ensure written to disk.
1264    testHarness.reload_engine(&h, &h1,
1265                              testHarness.engine_path,
1266                              testHarness.get_current_testcase()->cfg,
1267                              true, false);
1268    wait_for_warmup_complete(h, h1);
1269
1270    // Read value from disk.
1271    check_key_value(h, h1, "key", "somevalue", 9);
1272
1273    // Flush
1274    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS,
1275          "Failed to flush");
1276
1277    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i) == ENGINE_SUCCESS,
1278          "Failed post-flush set.");
1279    h1->release(h, NULL, i);
1280    check_key_value(h, h1, "key2", "somevalue", 9);
1281
1282    // Restart again, ensure written to disk.
1283    testHarness.reload_engine(&h, &h1,
1284                              testHarness.engine_path,
1285                              testHarness.get_current_testcase()->cfg,
1286                              true, false);
1287    wait_for_warmup_complete(h, h1);
1288
1289    check(store(h, h1, NULL, OPERATION_SET, "key3", "somevalue", &i) == ENGINE_SUCCESS,
1290          "Failed post-flush, post-restart set.");
1291    h1->release(h, NULL, i);
1292    check_key_value(h, h1, "key3", "somevalue", 9);
1293
1294    // Read value again, should not be there.
1295    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1296    return SUCCESS;
1297}
1298
1299static enum test_result test_flush_multiv_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1300    item *i = NULL;
1301    check(set_vbucket_state(h, h1, 2, vbucket_state_active), "Failed to set vbucket state.");
1302    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1303          "Failed set.");
1304    h1->release(h, NULL, i);
1305    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i,
1306                0, 2) == ENGINE_SUCCESS,
1307          "Failed set in vb2.");
1308    h1->release(h, NULL, i);
1309
1310    // Restart once to ensure written to disk.
1311    testHarness.reload_engine(&h, &h1,
1312                              testHarness.engine_path,
1313                              testHarness.get_current_testcase()->cfg,
1314                              true, false);
1315    wait_for_warmup_complete(h, h1);
1316
1317    // Read value from disk.
1318    check_key_value(h, h1, "key", "somevalue", 9);
1319
1320    // Flush
1321    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS,
1322          "Failed to flush");
1323
1324    // Restart again, ensure written to disk.
1325    testHarness.reload_engine(&h, &h1,
1326                              testHarness.engine_path,
1327                              testHarness.get_current_testcase()->cfg,
1328                              true, false);
1329    wait_for_warmup_complete(h, h1);
1330
1331    // Read value again, should not be there.
1332    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1333    check(verify_vbucket_missing(h, h1, 2), "Bucket 2 came back.");
1334    return SUCCESS;
1335}
1336
1337static enum test_result test_delete(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1338    item *i = NULL;
1339    // First try to delete something we know to not be there.
1340    check(del(h, h1, "key", 0, 0) == ENGINE_KEY_ENOENT, "Failed to fail initial delete.");
1341    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1342          "Failed set.");
1343    Item *it = reinterpret_cast<Item*>(i);
1344    uint64_t orig_cas = it->getCas();
1345    h1->release(h, NULL, i);
1346    check_key_value(h, h1, "key", "somevalue", 9);
1347
1348    uint64_t cas = 0;
1349    check(h1->remove(h, NULL, "key", 3, &cas, 0) == ENGINE_SUCCESS,
1350          "Failed remove with value.");
1351    check(orig_cas + 1 == cas, "Cas mismatch on delete");
1352    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1353
1354    // Can I time travel to an expired object and delete it?
1355    checkeq(ENGINE_SUCCESS, store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1356            "Failed set.");
1357    h1->release(h, NULL, i);
1358    testHarness.time_travel(3617);
1359    checkeq(ENGINE_KEY_ENOENT, del(h, h1, "key", 0, 0),
1360            "Did not get ENOENT removing an expired object.");
1361    checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"), "Expected missing key");
1362
1363    return SUCCESS;
1364}
1365
1366static enum test_result test_set_delete(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1367    item *i = NULL;
1368    checkeq(ENGINE_SUCCESS, store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1369            "Failed set.");
1370    h1->release(h, NULL, i);
1371    check_key_value(h, h1, "key", "somevalue", 9);
1372    checkeq(ENGINE_SUCCESS, del(h, h1, "key", 0, 0),
1373            "Failed remove with value.");
1374    checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"), "Expected missing key");
1375    wait_for_flusher_to_settle(h, h1);
1376    wait_for_stat_to_be(h, h1, "curr_items", 0);
1377    return SUCCESS;
1378}
1379
1380static enum test_result test_set_delete_invalid_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1381    item *i = NULL;
1382    check(store(h, h1, NULL, OPERATION_SET, "key",
1383                "somevalue", &i) == ENGINE_SUCCESS,
1384          "Failed set.");
1385    check_key_value(h, h1, "key", "somevalue", 9);
1386    item_info info;
1387    info.nvalue = 1;
1388    info.nvalue = 1;
1389    check(h1->get_item_info(h, NULL, i, &info) == true,
1390          "Should be able to get info");
1391    h1->release(h, NULL, i);
1392
1393    check(del(h, h1, "key", info.cas + 1, 0) == ENGINE_KEY_EEXISTS,
1394          "Didn't expect to be able to remove the item with wrong cas");
1395    return SUCCESS;
1396}
1397
1398static enum test_result test_bug2509(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1399    for (int j = 0; j < 10000; ++j) {
1400        item *itm = NULL;
1401        checkeq(ENGINE_SUCCESS,
1402                store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &itm),
1403                "Failed set.");
1404        h1->release(h, NULL, itm);
1405        usleep(10);
1406        checkeq(ENGINE_SUCCESS, del(h, h1, "key", 0, 0), "Failed remove with value.");
1407        usleep(10);
1408    }
1409
1410    // Restart again, to verify we don't have any duplicates.
1411    testHarness.reload_engine(&h, &h1,
1412                              testHarness.engine_path,
1413                              testHarness.get_current_testcase()->cfg,
1414                              true, false);
1415    wait_for_warmup_complete(h, h1);
1416
1417    return get_int_stat(h, h1, "ep_warmup_dups") == 0 ? SUCCESS : FAIL;
1418}
1419
1420static enum test_result test_bug7023(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1421    std::vector<std::string> keys;
1422    // Make a vbucket mess.
1423    for (int j = 0; j < 10000; ++j) {
1424        std::stringstream ss;
1425        ss << "key" << j;
1426        std::string key(ss.str());
1427        keys.push_back(key);
1428    }
1429
1430    std::vector<std::string>::iterator it;
1431    for (int j = 0; j < 5; ++j) {
1432        check(set_vbucket_state(h, h1, 0, vbucket_state_dead), "Failed set set vbucket 0 dead.");
1433        vbucketDelete(h, h1, 0);
1434        check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
1435              "Expected vbucket deletion to work.");
1436        check(set_vbucket_state(h, h1, 0, vbucket_state_active), "Failed set set vbucket 0 active.");
1437        for (it = keys.begin(); it != keys.end(); ++it) {
1438            item *i;
1439            check(store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str(), &i)
1440                  == ENGINE_SUCCESS, "Failed to store a value");
1441        }
1442    }
1443    wait_for_flusher_to_settle(h, h1);
1444
1445    // Restart again, to verify no data loss.
1446    testHarness.reload_engine(&h, &h1,
1447                              testHarness.engine_path,
1448                              testHarness.get_current_testcase()->cfg,
1449                              true, false);
1450    wait_for_warmup_complete(h, h1);
1451    return get_int_stat(h, h1, "ep_warmup_value_count", "warmup") == 10000 ? SUCCESS : FAIL;
1452}
1453
1454static enum test_result test_delete_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1455    wait_for_persisted_value(h, h1, "key", "value1");
1456
1457    check(del(h, h1, "key", 0, 0) == ENGINE_SUCCESS, "Failed remove with value.");
1458
1459    wait_for_persisted_value(h, h1, "key", "value2");
1460
1461    testHarness.reload_engine(&h, &h1,
1462                              testHarness.engine_path,
1463                              testHarness.get_current_testcase()->cfg,
1464                              true, false);
1465    wait_for_warmup_complete(h, h1);
1466
1467    check_key_value(h, h1, "key", "value2", 6);
1468    check(del(h, h1, "key", 0, 0) == ENGINE_SUCCESS, "Failed remove with value.");
1469    wait_for_flusher_to_settle(h, h1);
1470
1471    testHarness.reload_engine(&h, &h1,
1472                              testHarness.engine_path,
1473                              testHarness.get_current_testcase()->cfg,
1474                              true, false);
1475    wait_for_warmup_complete(h, h1);
1476
1477    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1478
1479    return SUCCESS;
1480}
1481
1482static enum test_result test_get_delete_missing_file(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1483    const char *key = "key";
1484    wait_for_persisted_value(h, h1, key, "value2delete");
1485
1486    // whack the db file and directory where the key is stored
1487    rmdb();
1488
1489    item *i = NULL;
1490    int errorCode = h1->get(h, NULL, &i, key, strlen(key), 0);
1491    h1->release(h, NULL, i);
1492
1493    // ep engine must be unaware of well-being of the db file as long as
1494    // the item is still in the memory
1495    check(errorCode == ENGINE_SUCCESS, "Expected success for get");
1496
1497    i = NULL;
1498    evict_key(h, h1, key);
1499    errorCode = h1->get(h, NULL, &i, key, strlen(key), 0);
1500    h1->release(h, NULL, i);
1501
1502    // ep engine must be now aware of the ill-fated db file where
1503    // the item is supposedly stored
1504    check(errorCode == ENGINE_TMPFAIL, "Expected tmp fail for get");
1505
1506    return SUCCESS;
1507}
1508
1509
1510static enum test_result test_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1511    item *i = NULL;
1512    static const char val[] = "somevalue";
1513    ENGINE_ERROR_CODE ret;
1514    check((ret = store(h, h1, NULL, OPERATION_SET, "key", val, &i)) == ENGINE_SUCCESS,
1515          "Failed set.");
1516    h1->release(h, NULL, i);
1517
1518    testHarness.reload_engine(&h, &h1,
1519                              testHarness.engine_path,
1520                              testHarness.get_current_testcase()->cfg,
1521                              true, false);
1522    wait_for_warmup_complete(h, h1);
1523    check_key_value(h, h1, "key", val, strlen(val));
1524    return SUCCESS;
1525}
1526
1527static enum test_result test_restart_session_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1528    createTapConn(h, h1, "tap_client_thread");
1529
1530    testHarness.reload_engine(&h, &h1,
1531                              testHarness.engine_path,
1532                              testHarness.get_current_testcase()->cfg,
1533                              true, false);
1534    wait_for_warmup_complete(h, h1);
1535    createTapConn(h, h1, "tap_client_thread");
1536
1537    check(h1->get_stats(h, NULL, "tap", 3, add_stats) == ENGINE_SUCCESS,
1538          "Failed to get stats.");
1539    std::string val = vals["eq_tapq:tap_client_thread:backfill_completed"];
1540    check(strcmp(val.c_str(), "true") == 0, "Don't expect the backfill upon restart");
1541    return SUCCESS;
1542}
1543
1544static enum test_result test_specialKeys(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1545    item *i = NULL;
1546    ENGINE_ERROR_CODE ret;
1547
1548    // Simplified Chinese "Couchbase"
1549    static const char key0[] = "沙发数据库";
1550    static const char val0[] = "some Chinese value";
1551    check((ret = store(h, h1, NULL, OPERATION_SET, key0, val0, &i)) == ENGINE_SUCCESS,
1552          "Failed set Chinese key");
1553    check_key_value(h, h1, key0, val0, strlen(val0));
1554    h1->release(h, NULL, i);
1555    // Traditional Chinese "Couchbase"
1556    static const char key1[] = "沙發數據庫";
1557    static const char val1[] = "some Traditional Chinese value";
1558    check((ret = store(h, h1, NULL, OPERATION_SET, key1, val1, &i)) == ENGINE_SUCCESS,
1559          "Failed set Traditional Chinese key");
1560    h1->release(h, NULL, i);
1561    // Korean "couch potato"
1562    static const char key2[] = "쇼파감자";
1563    static const char val2[] = "some Korean value";
1564    check((ret = store(h, h1, NULL, OPERATION_SET, key2, val2, &i)) == ENGINE_SUCCESS,
1565          "Failed set Korean key");
1566    h1->release(h, NULL, i);
1567    // Russian "couch potato"
1568    static const char key3[] = "лодырь, лентяй";
1569    static const char val3[] = "some Russian value";
1570    check((ret = store(h, h1, NULL, OPERATION_SET, key3, val3, &i)) == ENGINE_SUCCESS,
1571          "Failed set Russian key");
1572    h1->release(h, NULL, i);
1573    // Japanese "couch potato"
1574    static const char key4[] = "カウチポテト";
1575    static const char val4[] = "some Japanese value";
1576    check((ret = store(h, h1, NULL, OPERATION_SET, key4, val4, &i)) == ENGINE_SUCCESS,
1577          "Failed set Japanese key");
1578    h1->release(h, NULL, i);
1579    // Indian char key, and no idea what it is
1580    static const char key5[] = "हरियानवी";
1581    static const char val5[] = "some Indian value";
1582    check((ret = store(h, h1, NULL, OPERATION_SET, key5, val5, &i)) == ENGINE_SUCCESS,
1583          "Failed set Indian key");
1584    h1->release(h, NULL, i);
1585    // Portuguese translation "couch potato"
1586    static const char key6[] = "sedentário";
1587    static const char val6[] = "some Portuguese value";
1588    check((ret = store(h, h1, NULL, OPERATION_SET, key6, val6, &i)) == ENGINE_SUCCESS,
1589          "Failed set Portuguese key");
1590    h1->release(h, NULL, i);
1591    // Arabic translation "couch potato"
1592    static const char key7[] = "الحافلةالبطاطة";
1593    static const char val7[] = "some Arabic value";
1594    check((ret = store(h, h1, NULL, OPERATION_SET, key7, val7, &i)) == ENGINE_SUCCESS,
1595          "Failed set Arabic key");
1596    h1->release(h, NULL, i);
1597
1598    testHarness.reload_engine(&h, &h1,
1599                              testHarness.engine_path,
1600                              testHarness.get_current_testcase()->cfg,
1601                              true, false);
1602    wait_for_warmup_complete(h, h1);
1603    check_key_value(h, h1, key0, val0, strlen(val0));
1604    check_key_value(h, h1, key1, val1, strlen(val1));
1605    check_key_value(h, h1, key2, val2, strlen(val2));
1606    check_key_value(h, h1, key3, val3, strlen(val3));
1607    check_key_value(h, h1, key4, val4, strlen(val4));
1608    check_key_value(h, h1, key5, val5, strlen(val5));
1609    check_key_value(h, h1, key6, val6, strlen(val6));
1610    check_key_value(h, h1, key7, val7, strlen(val7));
1611    return SUCCESS;
1612}
1613
1614static enum test_result test_binKeys(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1615    item *i = NULL;
1616    ENGINE_ERROR_CODE ret;
1617
1618    // binary key with char values beyond 0x7F
1619    static const char key0[] = "\xe0\xed\xf1\x6f\x7f\xf8\xfa";
1620    static const char val0[] = "some value val8";
1621    check((ret = store(h, h1, NULL, OPERATION_SET, key0, val0, &i)) == ENGINE_SUCCESS,
1622          "Failed set binary key0");
1623    check_key_value(h, h1, key0, val0, strlen(val0));
1624    h1->release(h, NULL, i);
1625    // binary keys with char values beyond 0x7F
1626    static const char key1[] = "\xf1\xfd\xfe\xff\xf0\xf8\xef";
1627    static const char val1[] = "some value val9";
1628    check((ret = store(h, h1, NULL, OPERATION_SET, key1, val1, &i)) == ENGINE_SUCCESS,
1629          "Failed set binary key1");
1630    check_key_value(h, h1, key1, val1, strlen(val1));
1631    h1->release(h, NULL, i);
1632    // binary keys with special utf-8 BOM (Byte Order Mark) values 0xBB 0xBF 0xEF
1633    static const char key2[] = "\xff\xfe\xbb\xbf\xef";
1634    static const char val2[] = "some utf-8 bom value";
1635    check((ret = store(h, h1, NULL, OPERATION_SET, key2, val2, &i)) == ENGINE_SUCCESS,
1636          "Failed set binary utf-8 bom key");
1637    check_key_value(h, h1, key2, val2, strlen(val2));
1638    h1->release(h, NULL, i);
1639    // binary keys with special utf-16BE BOM values "U+FEFF"
1640    static const char key3[] = "U+\xfe\xff\xefU+\xff\xfe";
1641    static const char val3[] = "some utf-16 bom value";
1642    check((ret = store(h, h1, NULL, OPERATION_SET, key3, val3, &i)) == ENGINE_SUCCESS,
1643          "Failed set binary utf-16 bom key");
1644    check_key_value(h, h1, key3, val3, strlen(val3));
1645    h1->release(h, NULL, i);
1646
1647    testHarness.reload_engine(&h, &h1,
1648                              testHarness.engine_path,
1649                              testHarness.get_current_testcase()->cfg,
1650                              true, false);
1651    wait_for_warmup_complete(h, h1);
1652    check_key_value(h, h1, key0, val0, strlen(val0));
1653    check_key_value(h, h1, key1, val1, strlen(val1));
1654    check_key_value(h, h1, key2, val2, strlen(val2));
1655    check_key_value(h, h1, key3, val3, strlen(val3));
1656    return SUCCESS;
1657}
1658
1659static enum test_result test_restart_bin_val(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1660
1661
1662
1663    char binaryData[] = "abcdefg\0gfedcba";
1664    cb_assert(sizeof(binaryData) != strlen(binaryData));
1665
1666    item *i = NULL;
1667    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
1668                       binaryData, sizeof(binaryData), 82758, &i, 0, 0)
1669          == ENGINE_SUCCESS,
1670          "Failed set.");
1671    h1->release(h, NULL, i);
1672
1673    testHarness.reload_engine(&h, &h1,
1674                              testHarness.engine_path,
1675                              testHarness.get_current_testcase()->cfg,
1676                              true, false);
1677    wait_for_warmup_complete(h, h1);
1678
1679    check_key_value(h, h1, "key", binaryData, sizeof(binaryData));
1680    return SUCCESS;
1681}
1682
1683static enum test_result test_wrong_vb_get(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1684    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1685    check(ENGINE_NOT_MY_VBUCKET == verify_key(h, h1, "key", 1),
1686          "Expected wrong bucket.");
1687    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1688    return SUCCESS;
1689}
1690
1691static enum test_result test_vb_get_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1692    check(set_vbucket_state(h, h1, 1, vbucket_state_pending), "Failed to set vbucket state.");
1693    const void *cookie = testHarness.create_cookie();
1694    testHarness.set_ewouldblock_handling(cookie, false);
1695
1696    item *i = NULL;
1697    check(ENGINE_EWOULDBLOCK == h1->get(h, cookie, &i, "key", strlen("key"), 1),
1698          "Expected woodblock.");
1699    h1->release(h, NULL, i);
1700
1701    testHarness.destroy_cookie(cookie);
1702    return SUCCESS;
1703}
1704
1705static enum test_result test_vb_get_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1706    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set vbucket state.");
1707    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1708    check(ENGINE_NOT_MY_VBUCKET == verify_key(h, h1, "key", 1),
1709          "Expected not my bucket.");
1710    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1711    return SUCCESS;
1712}
1713
1714static enum test_result test_wrong_vb_incr(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1715    uint64_t cas, result;
1716    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1717    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
1718                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1719                         1) == ENGINE_NOT_MY_VBUCKET,
1720          "Expected not my vbucket.");
1721    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1722    return SUCCESS;
1723}
1724
1725static enum test_result test_vb_incr_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1726    const void *cookie = testHarness.create_cookie();
1727    testHarness.set_ewouldblock_handling(cookie, false);
1728    uint64_t cas, result;
1729    check(set_vbucket_state(h, h1, 1, vbucket_state_pending), "Failed to set vbucket state.");
1730    check(h1->arithmetic(h, cookie, "key", 3, true, false, 1, 1, 0,
1731                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1732                         1) == ENGINE_EWOULDBLOCK,
1733          "Expected woodblock.");
1734    testHarness.destroy_cookie(cookie);
1735    return SUCCESS;
1736}
1737
1738static enum test_result test_vb_incr_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1739    uint64_t cas, result;
1740    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set vbucket state.");
1741    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1742    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
1743                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1744                         1) == ENGINE_NOT_MY_VBUCKET,
1745          "Expected not my bucket.");
1746    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1747    return SUCCESS;
1748}
1749
1750static enum test_result test_wrong_vb_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1751    return test_wrong_vb_mutation(h, h1, OPERATION_SET);
1752}
1753
1754static enum test_result test_wrong_vb_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1755    return test_wrong_vb_mutation(h, h1, OPERATION_CAS);
1756}
1757
1758static enum test_result test_wrong_vb_add(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1759    return test_wrong_vb_mutation(h, h1, OPERATION_ADD);
1760}
1761
1762static enum test_result test_wrong_vb_replace(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1763    return test_wrong_vb_mutation(h, h1, OPERATION_REPLACE);
1764}
1765
1766static enum test_result test_wrong_vb_append(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1767    return test_wrong_vb_mutation(h, h1, OPERATION_APPEND);
1768}
1769
1770static enum test_result test_wrong_vb_prepend(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1771    return test_wrong_vb_mutation(h, h1, OPERATION_PREPEND);
1772}
1773
1774static enum test_result test_wrong_vb_del(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1775    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1776    check(ENGINE_NOT_MY_VBUCKET == del(h, h1, "key", 0, 1), "Expected wrong bucket.");
1777    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1778    return SUCCESS;
1779}
1780
1781static enum test_result test_expiry(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1782    const char *key = "test_expiry";
1783    const char *data = "some test data here.";
1784
1785    item *it = NULL;
1786
1787    ENGINE_ERROR_CODE rv;
1788    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 2,
1789                      PROTOCOL_BINARY_RAW_BYTES);
1790    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1791
1792    item_info info;
1793    info.nvalue = 1;
1794    if (!h1->get_item_info(h, NULL, it, &info)) {
1795        abort();
1796    }
1797    memcpy(info.value[0].iov_base, data, strlen(data));
1798
1799    uint64_t cas = 0;
1800    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
1801    check(rv == ENGINE_SUCCESS, "Set failed.");
1802    check_key_value(h, h1, key, data, strlen(data));
1803    h1->release(h, NULL, it);
1804
1805    testHarness.time_travel(5);
1806    check(h1->get(h, NULL, &it, key, strlen(key), 0) == ENGINE_KEY_ENOENT,
1807          "Item didn't expire");
1808
1809    int expired_access = get_int_stat(h, h1, "ep_expired_access");
1810    int expired_pager = get_int_stat(h, h1, "ep_expired_pager");
1811    int active_expired = get_int_stat(h, h1, "vb_active_expired");
1812    check(expired_pager == 0, "Expected zero expired item by pager");
1813    check(expired_access == 1, "Expected an expired item on access");
1814    check(active_expired == 1, "Expected an expired active item");
1815    checkeq(ENGINE_SUCCESS, store(h, h1, NULL, OPERATION_SET, key, data, &it),
1816            "Failed set.");
1817    h1->release(h, NULL, it);
1818
1819    std::stringstream ss;
1820    ss << "curr_items stat should be still 1 after ";
1821    ss << "overwriting the key that was expired, but not purged yet";
1822    checkeq(1, get_int_stat(h, h1, "curr_items"), ss.str().c_str());
1823
1824    return SUCCESS;
1825}
1826
1827static enum test_result test_expiry_loader(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1828    const char *key = "test_expiry_loader";
1829    const char *data = "some test data here.";
1830
1831    item *it = NULL;
1832
1833    ENGINE_ERROR_CODE rv;
1834    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 2,
1835                      PROTOCOL_BINARY_RAW_BYTES);
1836    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1837
1838    item_info info;
1839    info.nvalue = 1;
1840    if (!h1->get_item_info(h, NULL, it, &info)) {
1841        abort();
1842    }
1843    memcpy(info.value[0].iov_base, data, strlen(data));
1844
1845    uint64_t cas = 0;
1846    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
1847    check(rv == ENGINE_SUCCESS, "Set failed.");
1848    check_key_value(h, h1, key, data, strlen(data));
1849    h1->release(h, NULL, it);
1850
1851    testHarness.time_travel(3);
1852
1853    check(h1->get(h, NULL, &it, key, strlen(key), 0) == ENGINE_KEY_ENOENT,
1854          "Item didn't expire");
1855
1856    // Restart the engine to ensure the above expired item is not loaded
1857    testHarness.reload_engine(&h, &h1,
1858                              testHarness.engine_path,
1859                              testHarness.get_current_testcase()->cfg,
1860                              true, false);
1861    wait_for_warmup_complete(h, h1);
1862    cb_assert(0 == get_int_stat(h, h1, "ep_warmup_value_count", "warmup"));
1863
1864    return SUCCESS;
1865}
1866
1867static enum test_result test_expiration_on_warmup(ENGINE_HANDLE *h,
1868                                                  ENGINE_HANDLE_V1 *h1) {
1869    const char *key = "KEY";
1870    const char *data = "VALUE";
1871
1872    item *it = NULL;
1873
1874    ENGINE_ERROR_CODE rv;
1875    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 3,
1876                      PROTOCOL_BINARY_RAW_BYTES);
1877    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1878
1879    item_info info;
1880    info.nvalue = 1;
1881    if (!h1->get_item_info(h, NULL, it, &info)) {
1882        abort();
1883    }
1884    memcpy(info.value[0].iov_base, data, strlen(data));
1885
1886    uint64_t cas = 0;
1887    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
1888    check(rv == ENGINE_SUCCESS, "Set failed.");
1889    check_key_value(h, h1, key, data, strlen(data));
1890    h1->release(h, NULL, it);
1891    wait_for_flusher_to_settle(h, h1);
1892
1893    check(get_int_stat(h, h1, "curr_items") == 1, "Failed store item");
1894    testHarness.time_travel(5);
1895
1896    // Restart the engine to ensure the above item is expired
1897    testHarness.reload_engine(&h, &h1,
1898                              testHarness.engine_path,
1899                              testHarness.get_current_testcase()->cfg,
1900                              true, false);
1901    wait_for_warmup_complete(h, h1);
1902    wait_for_flusher_to_settle(h, h1);
1903    check(get_int_stat(h, h1, "curr_items") == 0,
1904            "The item should have been expired.");
1905
1906    return SUCCESS;
1907
1908}
1909
1910static enum test_result test_bug3454(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1911    const char *key = "test_expiry_duplicate_warmup";
1912    const char *data = "some test data here.";
1913
1914    item *it = NULL;
1915
1916    ENGINE_ERROR_CODE rv;
1917    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 5,
1918                      PROTOCOL_BINARY_RAW_BYTES);
1919    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1920
1921    item_info info;
1922    info.nvalue = 1;
1923    if (!h1->get_item_info(h, NULL, it, &info)) {
1924        abort();
1925    }
1926    memcpy(info.value[0].iov_base, data, strlen(data));
1927
1928    uint64_t cas = 0;
1929    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
1930    check(rv == ENGINE_SUCCESS, "Set failed.");
1931    check_key_value(h, h1, key, data, strlen(data));
1932    h1->release(h, NULL, it);
1933    wait_for_flusher_to_settle(h, h1);
1934
1935    // Advance the ep_engine time by 10 sec for the above item to be expired.
1936    testHarness.time_travel(10);
1937    check(h1->get(h, NULL, &it, key, strlen(key), 0) == ENGINE_KEY_ENOENT,
1938          "Item didn't expire");
1939
1940    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 0,
1941                      PROTOCOL_BINARY_RAW_BYTES);
1942    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1943
1944    info.nvalue = 1;
1945    if (!h1->get_item_info(h, NULL, it, &info)) {
1946        abort();
1947    }
1948    memcpy(info.value[0].iov_base, data, strlen(data));
1949
1950    cas = 0;
1951    // Add a new item with the same key.
1952    rv = h1->store(h, NULL, it, &cas, OPERATION_ADD, 0);
1953    check(rv == ENGINE_SUCCESS, "Add failed.");
1954    check_key_value(h, h1, key, data, strlen(data));
1955    h1->release(h, NULL, it);
1956
1957    check(h1->get(h, NULL, &it, key, strlen(key), 0) == ENGINE_SUCCESS,
1958          "Item shouldn't expire");
1959    h1->release(h, NULL, it);
1960
1961    // Restart the engine to ensure the above unexpired new item is loaded
1962    testHarness.reload_engine(&h, &h1,
1963                              testHarness.engine_path,
1964                              testHarness.get_current_testcase()->cfg,
1965                              true, false);
1966    wait_for_warmup_complete(h, h1);
1967    cb_assert(1 == get_int_stat(h, h1, "ep_warmup_value_count", "warmup"));
1968    cb_assert(0 == get_int_stat(h, h1, "ep_warmup_dups", "warmup"));
1969
1970    return SUCCESS;
1971}
1972
1973static enum test_result test_bug3522(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1974    const char *key = "test_expiry_no_items_warmup";
1975    const char *data = "some test data here.";
1976
1977    item *it = NULL;
1978
1979    ENGINE_ERROR_CODE rv;
1980    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 0,
1981                      PROTOCOL_BINARY_RAW_BYTES);
1982    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1983
1984    item_info info;
1985    info.nvalue = 1;
1986    if (!h1->get_item_info(h, NULL, it, &info)) {
1987        abort();
1988    }
1989    memcpy(info.value[0].iov_base, data, strlen(data));
1990
1991    uint64_t cas = 0;
1992    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
1993    check(rv == ENGINE_SUCCESS, "Set failed.");
1994    check_key_value(h, h1, key, data, strlen(data));
1995    h1->release(h, NULL, it);
1996    wait_for_flusher_to_settle(h, h1);
1997
1998    // Add a new item with the same key and 2 sec of expiration.
1999    const char *new_data = "new data here.";
2000    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(new_data), 0, 2,
2001                      PROTOCOL_BINARY_RAW_BYTES);
2002    check(rv == ENGINE_SUCCESS, "Allocation failed.");
2003
2004    info.nvalue = 1;
2005    if (!h1->get_item_info(h, NULL, it, &info)) {
2006        abort();
2007    }
2008    memcpy(info.value[0].iov_base, new_data, strlen(new_data));
2009
2010    int pager_runs = get_int_stat(h, h1, "ep_num_expiry_pager_runs");
2011    cas = 0;
2012    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
2013    check(rv == ENGINE_SUCCESS, "Set failed.");
2014    check_key_value(h, h1, key, new_data, strlen(new_data));
2015    h1->release(h, NULL, it);
2016    testHarness.time_travel(3);
2017    wait_for_stat_change(h, h1, "ep_num_expiry_pager_runs", pager_runs);
2018    wait_for_flusher_to_settle(h, h1);
2019
2020    // Restart the engine.
2021    testHarness.reload_engine(&h, &h1,
2022                              testHarness.engine_path,
2023                              testHarness.get_current_testcase()->cfg,
2024                              true, false);
2025    wait_for_warmup_complete(h, h1);
2026    // TODO: modify this for a better test case
2027    cb_assert(0 == get_int_stat(h, h1, "ep_warmup_dups", "warmup"));
2028
2029    return SUCCESS;
2030}
2031
2032static enum test_result test_get_replica_active_state(ENGINE_HANDLE *h,
2033                                                      ENGINE_HANDLE_V1 *h1) {
2034    protocol_binary_request_header *pkt;
2035    pkt = prepare_get_replica(h, h1, vbucket_state_active);
2036    check(h1->unknown_command(h, NULL, pkt, add_response) ==
2037          ENGINE_SUCCESS, "Get Replica Failed");
2038    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2039          "Expected PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET response.");
2040
2041    return SUCCESS;
2042}
2043
2044static enum test_result test_get_replica_pending_state(ENGINE_HANDLE *h,
2045                                                       ENGINE_HANDLE_V1 *h1) {
2046    protocol_binary_request_header *pkt;
2047
2048    const void *cookie = testHarness.create_cookie();
2049    testHarness.set_ewouldblock_handling(cookie, false);
2050    pkt = prepare_get_replica(h, h1, vbucket_state_pending);
2051    check(h1->unknown_command(h, cookie, pkt, add_response) ==
2052          ENGINE_EWOULDBLOCK, "Should have returned error for pending state");
2053    testHarness.destroy_cookie(cookie);
2054    return SUCCESS;
2055}
2056
2057static enum test_result test_get_replica_dead_state(ENGINE_HANDLE *h,
2058                                                    ENGINE_HANDLE_V1 *h1) {
2059    protocol_binary_request_header *pkt;
2060    pkt = prepare_get_replica(h, h1, vbucket_state_dead);
2061    check(h1->unknown_command(h, NULL, pkt, add_response) ==
2062          ENGINE_SUCCESS, "Get Replica Failed");
2063    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2064          "Expected PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET response.");
2065
2066    return SUCCESS;
2067}
2068
2069static enum test_result test_get_replica(ENGINE_HANDLE *h,
2070                                         ENGINE_HANDLE_V1 *h1) {
2071    protocol_binary_request_header *pkt;
2072    pkt = prepare_get_replica(h, h1, vbucket_state_replica);
2073    check(h1->unknown_command(h, NULL, pkt, add_response) == ENGINE_SUCCESS,
2074                              "Get Replica Failed");
2075    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
2076          "Expected PROTOCOL_BINARY_RESPONSE_SUCCESS response.");
2077    check(strcmp("replicadata", last_body) == 0,
2078                 "Should have returned identical value");
2079
2080    return SUCCESS;
2081}
2082
2083static enum test_result test_get_replica_non_resident(ENGINE_HANDLE *h,
2084                                                      ENGINE_HANDLE_V1 *h1) {
2085
2086    item *i = NULL;
2087    check(store(h, h1, NULL, OPERATION_SET, "key", "value", &i, 0, 0)
2088          == ENGINE_SUCCESS, "Store Failed");
2089    h1->release(h, NULL, i);
2090    wait_for_flusher_to_settle(h, h1);
2091    wait_for_stat_to_be(h, h1, "ep_total_persisted", 1);
2092
2093    evict_key(h, h1, "key", 0, "Ejected.");
2094    check(set_vbucket_state(h, h1, 0, vbucket_state_replica),
2095          "Failed to set vbucket to replica");
2096
2097    get_replica(h, h1, "key", 0);
2098    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "Expected success");
2099
2100    return SUCCESS;
2101}
2102
2103static enum test_result test_get_replica_invalid_key(ENGINE_HANDLE *h,
2104                                                     ENGINE_HANDLE_V1 *h1) {
2105    protocol_binary_request_header *pkt;
2106    bool makeinvalidkey = true;
2107    pkt = prepare_get_replica(h, h1, vbucket_state_replica, makeinvalidkey);
2108    check(h1->unknown_command(h, NULL, pkt, add_response) ==
2109          ENGINE_SUCCESS, "Get Replica Failed");
2110    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2111          "Expected PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET response.");
2112    return SUCCESS;
2113}
2114
2115static enum test_result test_vb_del_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2116    const void *cookie = testHarness.create_cookie();
2117    testHarness.set_ewouldblock_handling(cookie, false);
2118    check(set_vbucket_state(h, h1, 1, vbucket_state_pending), "Failed to set vbucket state.");
2119    check(ENGINE_EWOULDBLOCK == del(h, h1, "key", 0, 1, cookie),
2120          "Expected woodblock.");
2121    testHarness.destroy_cookie(cookie);
2122    return SUCCESS;
2123}
2124
2125static enum test_result test_vb_del_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2126    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set vbucket state.");
2127    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
2128    check(ENGINE_NOT_MY_VBUCKET == del(h, h1, "key", 0, 1),
2129          "Expected not my vbucket.");
2130    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
2131    return SUCCESS;
2132}
2133
2134static enum test_result test_touch(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2135    // key is a mandatory field!
2136    touch(h, h1, NULL, 0, (time(NULL) + 10));
2137    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2138
2139    // extlen is a mandatory field!
2140    protocol_binary_request_header *request;
2141    request = createPacket(PROTOCOL_BINARY_CMD_TOUCH, 0, 0, NULL, 0, "akey", 4);
2142    check(h1->unknown_command(h, NULL, request, add_response) == ENGINE_SUCCESS,
2143          "Failed to call touch");
2144    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2145    free(request);
2146
2147    // Try to touch an unknown item...
2148    touch(h, h1, "mykey", 0, (time(NULL) + 10));
2149    check(last_status == PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, "Testing unknown key");
2150
2151    // illegal vbucket
2152    touch(h, h1, "mykey", 5, (time(NULL) + 10));
2153    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET, "Testing illegal vbucket");
2154
2155    // Store the item!
2156    item *itm = NULL;
2157    check(store(h, h1, NULL, OPERATION_SET, "mykey", "somevalue", &itm) == ENGINE_SUCCESS,
2158          "Failed set.");
2159    h1->release(h, NULL, itm);
2160
2161    check_key_value(h, h1, "mykey", "somevalue", strlen("somevalue"));
2162
2163    touch(h, h1, "mykey", 0, (time(NULL) + 10));
2164    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch mykey");
2165
2166    // time-travel 9 secs..
2167    testHarness.time_travel(9);
2168
2169    // The item should still exist
2170    check_key_value(h, h1, "mykey", "somevalue", 9);
2171
2172    // time-travel 2 secs..
2173    testHarness.time_travel(2);
2174
2175    // The item should have expired now...
2176    check(h1->get(h, NULL, &itm, "mykey", 5, 0) == ENGINE_KEY_ENOENT, "Item should be gone");
2177    return SUCCESS;
2178}
2179
2180static enum test_result test_touch_mb7342(ENGINE_HANDLE *h,
2181                                          ENGINE_HANDLE_V1 *h1) {
2182    const char *key = "MB-7342";
2183    // Store the item!
2184    item *itm = NULL;
2185    check(store(h, h1, NULL, OPERATION_SET, key, "v", &itm) == ENGINE_SUCCESS,
2186          "Failed set.");
2187    h1->release(h, NULL, itm);
2188
2189    touch(h, h1, key, 0, 0);
2190    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch key");
2191
2192    check_key_value(h, h1, key, "v", 1);
2193
2194    // Travel a loong time to see if the object is still there (the default
2195    // store sets an exp time of 3600
2196    testHarness.time_travel(3700);
2197
2198    check_key_value(h, h1, key, "v", 1);
2199
2200    return SUCCESS;
2201}
2202
2203static enum test_result test_touch_mb10277(ENGINE_HANDLE *h,
2204                                            ENGINE_HANDLE_V1 *h1) {
2205    const char *key = "MB-10277";
2206    // Store the item!
2207    item *itm = NULL;
2208    check(store(h, h1, NULL, OPERATION_SET, key, "v", &itm) == ENGINE_SUCCESS,
2209          "Failed set.");
2210    h1->release(h, NULL, itm);
2211    wait_for_flusher_to_settle(h, h1);
2212    evict_key(h, h1, key, 0, "Ejected.");
2213
2214    touch(h, h1, key, 0, 3600); // A new expiration time remains in the same.
2215    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch key");
2216
2217    return SUCCESS;
2218}
2219
2220static enum test_result test_gat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2221    // key is a mandatory field!
2222    gat(h, h1, NULL, 0, 10);
2223    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2224
2225    // extlen is a mandatory field!
2226    protocol_binary_request_header *request;
2227    request = createPacket(PROTOCOL_BINARY_CMD_GAT, 0, 0, NULL, 0, "akey", 4);
2228    check(h1->unknown_command(h, NULL, request, add_response) == ENGINE_SUCCESS,
2229          "Failed to call gat");
2230    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2231    free(request);
2232
2233    // Try to gat an unknown item...
2234    gat(h, h1, "mykey", 0, 10);
2235    check(last_status == PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, "Testing unknown key");
2236
2237    // illegal vbucket
2238    gat(h, h1, "mykey", 5, 10);
2239    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET, "Testing illegal vbucket");
2240
2241    // Store the item!
2242    item *itm = NULL;
2243    check(store(h, h1, NULL, OPERATION_SET, "mykey", "{\"some\":\"value\"}",
2244                &itm, 0, 0, 3600, PROTOCOL_BINARY_DATATYPE_JSON) == ENGINE_SUCCESS,
2245          "Failed set.");
2246    h1->release(h, NULL, itm);
2247
2248    check_key_value(h, h1, "mykey", "{\"some\":\"value\"}",
2249            strlen("{\"some\":\"value\"}"));
2250
2251    gat(h, h1, "mykey", 0, 10);
2252    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "gat mykey");
2253    check(last_datatype == PROTOCOL_BINARY_DATATYPE_JSON, "Expected datatype to be JSON");
2254    check(memcmp(last_body, "{\"some\":\"value\"}", sizeof("{\"some\":\"value\"}")) == 0,
2255          "Invalid data returned");
2256
2257    // time-travel 9 secs..
2258    testHarness.time_travel(9);
2259
2260    // The item should still exist
2261    check_key_value(h, h1, "mykey", "{\"some\":\"value\"}",
2262                    strlen("{\"some\":\"value\"}"));
2263
2264    // time-travel 2 secs..
2265    testHarness.time_travel(2);
2266
2267    // The item should have expired now...
2268    check(h1->get(h, NULL, &itm, "mykey", 5, 0) == ENGINE_KEY_ENOENT, "Item should be gone");
2269    return SUCCESS;
2270}
2271
2272static enum test_result test_gatq(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2273    // key is a mandatory field!
2274    gat(h, h1, NULL, 0, 10, true);
2275    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2276
2277    // extlen is a mandatory field!
2278    protocol_binary_request_header *request;
2279    request = createPacket(PROTOCOL_BINARY_CMD_GATQ, 0, 0, NULL, 0, "akey", 4);
2280    check(h1->unknown_command(h, NULL, request, add_response) == ENGINE_SUCCESS,
2281          "Failed to call gatq");
2282    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2283    free(request);
2284
2285    // Try to gatq an unknown item...
2286    last_status = static_cast<protocol_binary_response_status>(0xffff);
2287    gat(h, h1, "mykey", 0, 10, true);
2288
2289    // We should not have sent any response!
2290    check(last_status == (protocol_binary_response_status)0xffff, "Testing unknown key");
2291
2292    // illegal vbucket
2293    gat(h, h1, "mykey", 5, 10, true);
2294    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2295          "Testing illegal vbucket");
2296
2297    // Store the item!
2298    item *itm = NULL;
2299    check(store(h, h1, NULL, OPERATION_SET, "mykey", "{\"some\":\"value\"}",
2300                &itm, 0, 0, 3600, PROTOCOL_BINARY_DATATYPE_JSON) == ENGINE_SUCCESS,
2301          "Failed set.");
2302    h1->release(h, NULL, itm);
2303
2304    check_key_value(h, h1, "mykey", "{\"some\":\"value\"}",
2305                    strlen("{\"some\":\"value\"}"));
2306
2307    gat(h, h1, "mykey", 0, 10, true);
2308    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "gat mykey");
2309    check(last_datatype == PROTOCOL_BINARY_DATATYPE_JSON, "Expected datatype to be JSON");
2310    check(memcmp(last_body, "{\"some\":\"value\"}", sizeof("{\"some\":\"value\"}")) == 0,
2311          "Invalid data returned");
2312
2313    // time-travel 9 secs..
2314    testHarness.time_travel(9);
2315
2316    // The item should still exist
2317    check_key_value(h, h1, "mykey", "{\"some\":\"value\"}",
2318                    strlen("{\"some\":\"value\"}"));
2319
2320    // time-travel 2 secs..
2321    testHarness.time_travel(2);
2322
2323    // The item should have expired now...
2324    check(h1->get(h, NULL, &itm, "mykey", 5, 0) == ENGINE_KEY_ENOENT, "Item should be gone");
2325    return SUCCESS;
2326}
2327
2328static enum test_result test_mb5215(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2329    item *itm = NULL;
2330    check(store(h, h1, NULL, OPERATION_SET, "coolkey", "cooler", &itm)
2331          == ENGINE_SUCCESS, "Failed set.");
2332    h1->release(h, NULL, itm);
2333
2334    check_key_value(h, h1, "coolkey", "cooler", strlen("cooler"));
2335
2336    // set new exptime to 111
2337    int expTime = time(NULL) + 111;
2338
2339    touch(h, h1, "coolkey", 0, expTime);
2340    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch coolkey");
2341
2342    //reload engine
2343    testHarness.reload_engine(&h, &h1,
2344                              testHarness.engine_path,
2345                              testHarness.get_current_testcase()->cfg,
2346                              true, false);
2347    wait_for_warmup_complete(h, h1);
2348
2349    //verify persisted expiration time
2350    const char *statkey = "key coolkey 0";
2351    int newExpTime;
2352    check(h1->get(h, NULL, &itm, "coolkey", 7, 0) == ENGINE_SUCCESS,
2353          "Missing key");
2354    h1->release(h, NULL, itm);
2355    newExpTime = get_int_stat(h, h1, "key_exptime", statkey);
2356    check(newExpTime == expTime, "Failed to persist new exptime");
2357
2358    // evict key, touch expiration time, and verify
2359    evict_key(h, h1, "coolkey", 0, "Ejected.");
2360
2361    expTime = time(NULL) + 222;
2362    touch(h, h1, "coolkey", 0, expTime);
2363    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch coolkey");
2364
2365    testHarness.reload_engine(&h, &h1,
2366                              testHarness.engine_path,
2367                              testHarness.get_current_testcase()->cfg,
2368                              true, false);
2369    wait_for_warmup_complete(h, h1);
2370
2371    check(h1->get(h, NULL, &itm, "coolkey", 7, 0) == ENGINE_SUCCESS,
2372          "Missing key");
2373    h1->release(h, NULL, itm);
2374    newExpTime = get_int_stat(h, h1, "key_exptime", statkey);
2375    check(newExpTime == expTime, "Failed to persist new exptime");
2376
2377    return SUCCESS;
2378}
2379
2380static enum test_result test_alloc_limit(ENGINE_HANDLE *h,
2381                                         ENGINE_HANDLE_V1 *h1) {
2382    item *it = NULL;
2383    ENGINE_ERROR_CODE rv;
2384
2385    rv = h1->allocate(h, NULL, &it, "key", 3, 20 * 1024 * 1024, 0, 0,
2386                      PROTOCOL_BINARY_RAW_BYTES);
2387    check(rv == ENGINE_SUCCESS, "Allocated 20MB item");
2388    h1->release(h, NULL, it);
2389
2390    rv = h1->allocate(h, NULL, &it, "key", 3, (20 * 1024 * 1024) + 1, 0, 0,
2391                      PROTOCOL_BINARY_RAW_BYTES);
2392    check(rv == ENGINE_E2BIG, "Object too big");
2393
2394    return SUCCESS;
2395}
2396
2397static enum test_result test_whitespace_db(ENGINE_HANDLE *h,
2398                                           ENGINE_HANDLE_V1 *h1) {
2399    vals.clear();
2400    check(h1->get_stats(h, NULL, NULL, 0, add_stats) == ENGINE_SUCCESS,
2401          "Failed to get stats.");
2402    if (vals["ep_dbname"] != std::string(WHITESPACE_DB)) {
2403        std::cerr << "Expected dbname = ``" << WHITESPACE_DB << "''"
2404                  << ", got ``" << vals["ep_dbname"] << "''" << std::endl;
2405        return FAIL;
2406    }
2407
2408    check(access(WHITESPACE_DB, F_OK) != -1, "I expected the whitespace db to exist");
2409    return SUCCESS;
2410}
2411
2412static enum test_result test_memory_limit(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2413    set_param(h, h1, protocol_binary_engine_param_flush, "mutation_mem_threshold", "95");
2414    int used = get_int_stat(h, h1, "mem_used");
2415    double mem_threshold =
2416        static_cast<double>(get_int_stat(h, h1, "ep_mutation_mem_threshold")) / 100;
2417    int max = static_cast<int>(get_int_stat(h, h1, "ep_max_size") * mem_threshold);
2418    check(get_int_stat(h, h1, "ep_oom_errors") == 0 &&
2419          get_int_stat(h, h1, "ep_tmp_oom_errors") == 0, "Expected no OOM errors.");
2420    cb_assert(used < max);
2421
2422    char data[8192];
2423    memset(data, 'x', sizeof(data));
2424    size_t vlen = max - used - 192;
2425    data[vlen] = 0x00;
2426
2427    item *i = NULL;
2428    // So if we add an item,
2429    check(store(h, h1, NULL, OPERATION_SET, "key", data, &i) == ENGINE_SUCCESS,
2430          "store failure");
2431    check_key_value(h, h1, "key", data, vlen);
2432    h1->release(h, NULL, i);
2433
2434    // There should be no room for another.
2435    ENGINE_ERROR_CODE second = store(h, h1, NULL, OPERATION_SET, "key2", data, &i);
2436    check(second == ENGINE_ENOMEM || second == ENGINE_TMPFAIL,
2437          "should have failed second set");
2438    h1->release(h, NULL, i);
2439    check(get_int_stat(h, h1, "ep_oom_errors") == 1 ||
2440          get_int_stat(h, h1, "ep_tmp_oom_errors") == 1, "Expected an OOM error.");
2441
2442    ENGINE_ERROR_CODE overwrite = store(h, h1, NULL, OPERATION_SET, "key", data, &i);
2443    check(overwrite == ENGINE_ENOMEM || overwrite == ENGINE_TMPFAIL,
2444          "should have failed second override");
2445    h1->release(h, NULL, i);
2446    check(get_int_stat(h, h1, "ep_oom_errors") == 2 ||
2447          get_int_stat(h, h1, "ep_tmp_oom_errors") == 2, "Expected another OOM error.");
2448    check_key_value(h, h1, "key", data, vlen);
2449    check(ENGINE_SUCCESS != verify_key(h, h1, "key2"), "Expected a failure in GET");
2450    int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
2451    // Until we remove that item
2452    check(del(h, h1, "key", 0, 0) == ENGINE_SUCCESS, "Failed remove with value.");
2453    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
2454    testHarness.time_travel(65);
2455    wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
2456
2457    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue2", &i) == ENGINE_SUCCESS,
2458          "should have succeded on the last set");
2459    check_key_value(h, h1, "key2", "somevalue2", 10);
2460    h1->release(h, NULL, i);
2461    return SUCCESS;
2462}
2463
2464static enum test_result test_vbucket_get_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2465    return verify_vbucket_missing(h, h1, 1) ? SUCCESS : FAIL;
2466}
2467
2468static enum test_result test_vbucket_get(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2469    return verify_vbucket_state(h, h1, 0, vbucket_state_active) ? SUCCESS : FAIL;
2470}
2471
2472static enum test_result test_vbucket_create(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2473    if (!verify_vbucket_missing(h, h1, 1)) {
2474        fprintf(stderr, "vbucket wasn't missing.\n");
2475        return FAIL;
2476    }
2477
2478    if (!set_vbucket_state(h, h1, 1, vbucket_state_active)) {
2479        fprintf(stderr, "set state failed.\n");
2480        return FAIL;
2481    }
2482
2483    return verify_vbucket_state(h, h1, 1, vbucket_state_active) ? SUCCESS : FAIL;
2484}
2485
2486static enum test_result test_vbucket_compact(ENGINE_HANDLE *h,
2487                                             ENGINE_HANDLE_V1 *h1) {
2488    const char *key = "Carss";
2489    const char *value = "pollute";
2490    if (!verify_vbucket_missing(h, h1, 0)) {
2491        fprintf(stderr, "vbucket wasn't missing.\n");
2492        return FAIL;
2493    }
2494
2495    if (!set_vbucket_state(h, h1, 0, vbucket_state_active)) {
2496        fprintf(stderr, "set state failed.\n");
2497        return FAIL;
2498    }
2499
2500    check(verify_vbucket_state(h, h1, 0, vbucket_state_active),
2501            "VBucket state not active");
2502
2503    // Set two keys - one to be expired and other to remain...
2504    item *itm = NULL;
2505    check(store(h, h1, NULL, OPERATION_SET, key, value, &itm)
2506          == ENGINE_SUCCESS, "Failed set.");
2507    h1->release(h, NULL, itm);
2508
2509    check_key_value(h, h1, key, value, strlen(value));
2510
2511    // Set a non-expiring key...
2512    check(store(h, h1, NULL, OPERATION_SET, "trees", "cleanse", &itm)
2513          == ENGINE_SUCCESS, "Failed set.");
2514    h1->release(h, NULL, itm);
2515
2516    check_key_value(h, h1, "trees", "cleanse", strlen("cleanse"));
2517
2518    touch(h, h1, key, 0, 11);
2519    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch Carss");
2520
2521    testHarness.time_travel(12);
2522    wait_for_flusher_to_settle(h, h1);
2523
2524    check(get_int_stat(h, h1, "vb_0:purge_seqno", "vbucket-seqno") == 0,
2525            "purge_seqno not found to be zero before compaction");
2526
2527    // Compaction on VBucket
2528    compact_db(h, h1, 0, 2, 3, 1);
2529
2530    check(get_int_stat(h, h1, "ep_pending_compactions") == 0,
2531    "ep_pending_compactions stat did not tick down after compaction command");
2532
2533    // the key tree and its value should be intact...
2534    check(verify_key(h, h1, "trees") == ENGINE_SUCCESS,
2535          "key trees should be found.");
2536    // the key Carrs should have disappeared...
2537    int val = verify_key(h, h1, "Carss");
2538    check(val == ENGINE_KEY_ENOENT, "Key Carss has not expired.");
2539
2540    check(get_int_stat(h, h1, "vb_0:purge_seqno", "vbucket-seqno") == 3,
2541        "purge_seqno didn't match expected value");
2542
2543    return SUCCESS;
2544}
2545
2546struct comp_thread_ctx {
2547    ENGINE_HANDLE *h;
2548    ENGINE_HANDLE_V1 *h1;
2549    uint16_t vbid;
2550};
2551
2552extern "C" {
2553    static void compaction_thread(void *arg) {
2554        struct comp_thread_ctx *ctx = static_cast<comp_thread_ctx *>(arg);
2555        compact_db(ctx->h, ctx->h1, ctx->vbid, 0, 0, 0);
2556    }
2557}
2558
2559static enum test_result test_multiple_vb_compactions(ENGINE_HANDLE *h,
2560                                                     ENGINE_HANDLE_V1 *h1) {
2561    for (uint16_t i = 0; i < 4; ++i) {
2562        if (!set_vbucket_state(h, h1, i, vbucket_state_active)) {
2563            fprintf(stderr, "set state failed for vbucket %d.\n", i);
2564            return FAIL;
2565        }
2566        check(verify_vbucket_state(h, h1, i, vbucket_state_active),
2567              "VBucket state not active");
2568    }
2569
2570    std::vector<std::string> keys;
2571    for (int j = 0; j < 20000; ++j) {
2572        std::stringstream ss;
2573        ss << "key" << j;
2574        std::string key(ss.str());
2575        keys.push_back(key);
2576    }
2577
2578    int count = 0;
2579    std::vector<std::string>::iterator it;
2580    for (it = keys.begin(); it != keys.end(); ++it) {
2581        uint16_t vbid = count % 4;
2582        item *i;
2583        check(store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str(), &i, 0, vbid)
2584              == ENGINE_SUCCESS, "Failed to store a value");
2585        h1->release(h, NULL, i);
2586        ++count;
2587    }
2588
2589    // Compact multiple vbuckets.
2590    const int n_threads = 4;
2591    cb_thread_t threads[n_threads];
2592    struct comp_thread_ctx ctx[n_threads];
2593
2594    for (int i = 0; i < n_threads; i++) {
2595        ctx[i].h = h;
2596        ctx[i].h1 = h1;
2597        ctx[i].vbid = static_cast<uint16_t>(i);
2598        int r = cb_create_thread(&threads[i], compaction_thread, &ctx[i], 0);
2599        cb_assert(r == 0);
2600    }
2601
2602    for (int i = 0; i < n_threads; i++) {
2603        int r = cb_join_thread(threads[i]);
2604        cb_assert(r == 0);
2605    }
2606
2607    check(get_int_stat(h, h1, "ep_pending_compactions") == 0,
2608    "ep_pending_compactions stat did not tick down after compaction command");
2609
2610    return SUCCESS;
2611}
2612
2613static enum test_result vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
2614                                             const char* value = NULL) {
2615    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed to set vbucket state.");
2616
2617    vbucketDelete(h, h1, 2, value);
2618    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2619          "Expected failure deleting non-existent bucket.");
2620
2621    check(set_vbucket_state(h, h1, 1, vbucket_state_dead), "Failed set set vbucket 1 state.");
2622
2623    vbucketDelete(h, h1, 1, value);
2624    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
2625          "Expected failure deleting non-existent bucket.");
2626
2627    check(verify_vbucket_missing(h, h1, 1),
2628          "vbucket 0 was not missing after deleting it.");
2629
2630    return SUCCESS;
2631}
2632
2633static enum test_result test_vbucket_destroy_stats(ENGINE_HANDLE *h,
2634                                                   ENGINE_HANDLE_V1 *h1) {
2635
2636    int mem_used = get_int_stat(h, h1, "mem_used");
2637    int cacheSize = get_int_stat(h, h1, "ep_total_cache_size");
2638    int overhead = get_int_stat(h, h1, "ep_overhead");
2639    int nonResident = get_int_stat(h, h1, "ep_num_non_resident");
2640
2641    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed to set vbucket state.");
2642
2643    std::vector<std::string> keys;
2644    for (int j = 0; j < 2000; ++j) {
2645        std::stringstream ss;
2646        ss << "key" << j;
2647        std::string key(ss.str());
2648        keys.push_back(key);
2649    }
2650
2651    int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
2652    std::vector<std::string>::iterator it;
2653    for (it = keys.begin(); it != keys.end(); ++it) {
2654        item *i;
2655        check(store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str(), &i, 0, 1)
2656              == ENGINE_SUCCESS, "Failed to store a value");
2657        h1->release(h, NULL, i);
2658    }
2659    wait_for_flusher_to_settle(h, h1);
2660    testHarness.time_travel(65);
2661    wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
2662
2663    check(set_vbucket_state(h, h1, 1, vbucket_state_dead), "Failed set set vbucket 1 state.");
2664
2665    int vbucketDel = get_int_stat(h, h1, "ep_vbucket_del");
2666    vbucketDelete(h, h1, 1);
2667    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
2668          "Expected failure deleting non-existent bucket.");
2669
2670    check(verify_vbucket_missing(h, h1, 1),
2671          "vbucket 1 was not missing after deleting it.");
2672
2673    wait_for_stat_change(h, h1, "ep_vbucket_del", vbucketDel);
2674
2675    wait_for_stat_to_be(h, h1, "mem_used", mem_used);
2676    wait_for_stat_to_be(h, h1, "ep_total_cache_size", cacheSize);
2677    wait_for_stat_to_be(h, h1, "ep_overhead", overhead);
2678    wait_for_stat_to_be(h, h1, "ep_num_non_resident", nonResident);
2679
2680    return SUCCESS;
2681}
2682
2683static enum test_result vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
2684                                                const char* value = NULL) {
2685    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed to set vbucket state.");
2686
2687    // Store a value so the restart will try to resurrect it.
2688    item *i = NULL;
2689    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i, 0, 1)
2690          == ENGINE_SUCCESS, "Failed to set a value");
2691    check_key_value(h, h1, "key", "somevalue", 9, 1);
2692    h1->release(h, NULL, i);
2693
2694    // Reload to get a flush forced.
2695    testHarness.reload_engine(&h, &h1,
2696                              testHarness.engine_path,
2697                              testHarness.get_current_testcase()->cfg,
2698                              true, false);
2699    wait_for_warmup_complete(h, h1);
2700
2701    check(verify_vbucket_state(h, h1, 1, vbucket_state_active),
2702          "Bucket state was what it was initially, after restart.");
2703    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed to set vbucket state.");
2704    check_key_value(h, h1, "key", "somevalue", 9, 1);
2705
2706    check(set_vbucket_state(h, h1, 1, vbucket_state_dead), "Failed set set vbucket 1 state.");
2707
2708    vbucketDelete(h, h1, 1, value);
2709    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
2710          "Expected failure deleting non-existent bucket.");
2711
2712    check(verify_vbucket_missing(h, h1, 1),
2713          "vbucket 1 was not missing after deleting it.");
2714
2715    testHarness.reload_engine(&h, &h1,
2716                              testHarness.engine_path,
2717                              testHarness.get_current_testcase()->cfg,
2718                              true, false);
2719    wait_for_warmup_complete(h, h1);
2720
2721    if (verify_vbucket_state(h, h1, 1, vbucket_state_pending, true)) {
2722        std::cerr << "Bucket came up in pending state after delete." << std::endl;
2723        abort();
2724    }
2725
2726    check(verify_vbucket_missing(h, h1, 1),
2727          "vbucket 1 was not missing after restart.");
2728
2729    return SUCCESS;
2730}
2731
2732static enum test_result test_async_vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2733    return vbucket_destroy(h, h1);
2734}
2735
2736static enum test_result test_sync_vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2737    return vbucket_destroy(h, h1, "async=0");
2738}
2739
2740static enum test_result test_async_vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2741    return vbucket_destroy_restart(h, h1);
2742}
2743
2744static enum test_result test_sync_vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2745    return vbucket_destroy_restart(h, h1, "async=0");
2746}
2747
2748static enum test_result test_vb_set_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2749    return test_pending_vb_mutation(h, h1, OPERATION_SET);
2750}
2751
2752static enum test_result test_vb_add_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2753    return test_pending_vb_mutation(h, h1, OPERATION_ADD);
2754}
2755
2756static enum test_result test_vb_cas_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2757    return test_pending_vb_mutation(h, h1, OPERATION_CAS);
2758}
2759
2760static enum test_result test_vb_append_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2761    return test_pending_vb_mutation(h, h1, OPERATION_APPEND);
2762}
2763
2764static enum test_result test_vb_prepend_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2765    return test_pending_vb_mutation(h, h1, OPERATION_PREPEND);
2766}
2767
2768static enum test_result test_vb_set_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2769    return test_replica_vb_mutation(h, h1, OPERATION_SET);
2770}
2771
2772static enum test_result test_vb_replace_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2773    return test_replica_vb_mutation(h, h1, OPERATION_REPLACE);
2774}
2775
2776static enum test_result test_vb_replace_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2777    return test_pending_vb_mutation(h, h1, OPERATION_REPLACE);
2778}
2779
2780static enum test_result test_vb_add_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2781    return test_replica_vb_mutation(h, h1, OPERATION_ADD);
2782}
2783
2784static enum test_result test_vb_cas_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2785    return test_replica_vb_mutation(h, h1, OPERATION_CAS);
2786}
2787
2788static enum test_result test_vb_append_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2789    return test_replica_vb_mutation(h, h1, OPERATION_APPEND);
2790}
2791
2792static enum test_result test_vb_prepend_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2793    return test_replica_vb_mutation(h, h1, OPERATION_PREPEND);
2794}
2795
2796static enum test_result test_stats_seqno(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2797    check(set_vbucket_state(h, h1, 1, vbucket_state_active),
2798          "Failed to set vbucket state.");
2799
2800    int num_keys = 100;
2801    for (int ii = 0; ii < num_keys; ++ii) {
2802        std::stringstream ss;
2803        ss << "key" << ii;
2804        check(store(h, h1, NULL, OPERATION_SET, ss.str().c_str(),
2805                    "value", NULL, 0, 0) == ENGINE_SUCCESS,
2806              "Failed to store an item.");
2807    }
2808
2809    check(get_int_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno") == 100,
2810          "Invalid seqno");
2811    check(get_int_stat(h, h1, "vb_1:high_seqno", "vbucket-seqno") == 0,
2812          "Invalid seqno");
2813    check(get_int_stat(h, h1, "vb_1:high_seqno", "vbucket-seqno 1") == 0,
2814          "Invalid seqno");
2815
2816    uint64_t vb_uuid = get_ull_stat(h, h1, "vb_1:0:id", "failovers");
2817    check(get_ull_stat(h, h1, "vb_1:uuid", "vbucket-seqno 1") == vb_uuid,
2818          "Invalid uuid");
2819    check(vals.size() == 3, "Expected three stats");
2820
2821    // Check invalid vbucket
2822    check(h1->get_stats(h, NULL, "vbucket-seqno 2", 15, add_stats)
2823          == ENGINE_NOT_MY_VBUCKET, "Expected not my vbucket");
2824
2825    // Check bad vbucket parameter (not numeric)
2826    check(h1->get_stats(h, NULL, "vbucket-seqno tt2", 17, add_stats)
2827          == ENGINE_EINVAL, "Expected invalid");
2828
2829    // Check extra spaces at the end
2830    check(h1->get_stats(h, NULL, "vbucket-seqno    ", 17, add_stats)
2831          == ENGINE_EINVAL, "Expected invalid");
2832
2833    return SUCCESS;
2834}
2835
2836static enum test_result test_stats_diskinfo(ENGINE_HANDLE *h,
2837                                            ENGINE_HANDLE_V1 *h1) {
2838    check(set_vbucket_state(h, h1, 1, vbucket_state_active),
2839          "Failed to set vbucket state.");
2840
2841    int num_keys = 100;
2842    for (int ii = 0; ii < num_keys; ++ii) {
2843        std::stringstream ss;
2844        ss << "key" << ii;
2845        check(store(h, h1, NULL, OPERATION_SET, ss.str().c_str(),
2846                    "value", NULL, 0, 1) == ENGINE_SUCCESS,
2847              "Failed to store an item.");
2848    }
2849    wait_for_flusher_to_settle(h, h1);
2850
2851    size_t file_size = get_int_stat(h, h1, "ep_db_file_size", "diskinfo");
2852    size_t data_size = get_int_stat(h, h1, "ep_db_data_size", "diskinfo");
2853    check(file_size > 0, "DB file size should be greater than 0");
2854    check(data_size > 0, "DB data size should be greater than 0");
2855    check(file_size >= data_size, "DB file size should be >= DB data size");
2856    check(get_int_stat(h, h1, "vb_1:data_size", "diskinfo detail") > 0,
2857          "VB 1 data size should be greater than 0");
2858
2859    check(h1->get_stats(h, NULL, "diskinfo ", 9, add_stats)
2860          == ENGINE_EINVAL, "Expected invalid");
2861
2862    check(h1->get_stats(h, NULL, "diskinfo detai", 14, add_stats)
2863          == ENGINE_EINVAL, "Expected invalid");
2864
2865    check(h1->get_stats(h, NULL, "diskinfo detaillll", 18, add_stats)
2866          == ENGINE_EINVAL, "Expected invalid");
2867
2868    return SUCCESS;
2869}
2870
2871static void notifier_request(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
2872                             const void* cookie, uint32_t opaque,
2873                             uint16_t vbucket, uint64_t start,
2874                             bool shouldSucceed) {
2875
2876    uint32_t flags = 0;
2877    uint64_t rollback = 0;
2878    uint64_t vb_uuid = get_ull_stat(h, h1, "vb_0:0:id", "failovers");
2879    uint64_t snap_start_seqno = get_ull_stat(h, h1, "vb_0:0:seq", "failovers");
2880    uint64_t snap_end_seqno = snap_start_seqno;
2881    ENGINE_ERROR_CODE err = h1->dcp.stream_req(h, cookie, flags, opaque,
2882                                               vbucket, start, 0, vb_uuid,
2883                                               snap_start_seqno, snap_end_seqno,
2884                                               &rollback,
2885                                               mock_upr_add_failover_log);
2886    check(err == ENGINE_SUCCESS, "Failed to initiate stream request");
2887
2888    std::string type = get_str_stat(h, h1, "eq_dcpq:unittest:type", "dcp");
2889    check(type.compare("notifier") == 0, "Consumer not found");
2890
2891    check((uint32_t)get_int_stat(h, h1, "eq_dcpq:unittest:stream_0_flags", "dcp")
2892          == flags, "Flags didn't match");
2893    check((uint32_t)get_int_stat(h, h1, "eq_dcpq:unittest:stream_0_opaque", "dcp")
2894          == opaque, "Opaque didn't match");
2895    check((uint64_t)get_ull_stat(h, h1, "eq_dcpq:unittest:stream_0_start_seqno", "dcp")
2896          == start, "Start Seqno Didn't match");
2897    check((uint64_t)get_ull_stat(h, h1, "eq_dcpq:unittest:stream_0_end_seqno", "dcp")
2898          == 0, "End Seqno didn't match");
2899    check((uint64_t)get_ull_stat(h, h1, "eq_dcpq:unittest:stream_0_vb_uuid", "dcp")
2900          == vb_uuid, "VBucket UUID didn't match");
2901    check((uint64_t)get_ull_stat(h, h1, "eq_dcpq:unittest:stream_0_snap_start_seqno", "dcp")
2902          == snap_start_seqno, "snap start seqno didn't match");
2903}
2904
2905static enum test_result test_upr_vbtakeover_no_stream(ENGINE_HANDLE *h,
2906                                                      ENGINE_HANDLE_V1 *h1) {
2907
2908    int num_items = 10;
2909    for (int j = 0; j < num_items; ++j) {
2910        item *i = NULL;
2911        std::stringstream ss;
2912        ss << "key" << j;
2913        check(store(h, h1, NULL, OPERATION_SET, ss.str().c_str(), "data", &i)
2914              == ENGINE_SUCCESS, "Failed to store a value");
2915        h1->release(h, NULL, i);
2916    }
2917
2918    int est = get_int_stat(h, h1, "estimate", "dcp-vbtakeover 0");
2919    check(est == 10, "Invalid estimate for non-existent stream");
2920
2921    check(h1->get_stats(h, NULL, "dcp-vbtakeover 1", strlen("dcp-vbtakeover 1"),
2922                        add_stats) == ENGINE_NOT_MY_VBUCKET,
2923                        "Expected not my vbucket");
2924
2925    return SUCCESS;
2926}
2927
2928static enum test_result test_upr_notifier(ENGINE_HANDLE *h,
2929                                          ENGINE_HANDLE_V1 *h1) {
2930
2931    int num_items = 10;
2932    for (int j = 0; j < num_items; ++j) {
2933        item *i = NULL;
2934        std::stringstream ss;
2935        ss << "key" << j;
2936        check(store(h, h1, NULL, OPERATION_SET, ss.str().c_str(), "data", &i)
2937              == ENGINE_SUCCESS, "Failed to store a value");
2938        h1->release(h, NULL, i);
2939    }
2940
2941    const void *cookie = testHarness.create_cookie();
2942    uint32_t opaque = 0;
2943    uint32_t flags = DCP_OPEN_NOTIFIER;
2944    const char *name = "unittest";
2945    uint16_t nname = strlen(name);
2946
2947    check(h1->dcp.open(h, cookie, opaque, 0, flags, (void*)name, nname)
2948          == ENGINE_SUCCESS,
2949          "Failed upr notifier open connection.");
2950
2951    // Get notification for an old item
2952    notifier_request(h, h1, cookie, ++opaque, 0, 0, true);
2953    upr_step(h, h1, cookie);
2954    check(upr_last_op == PROTOCOL_BINARY_CMD_DCP_STREAM_END,
2955          "Expected stream end");
2956
2957    // Get notification when we're slightly behind
2958    notifier_request(h, h1, cookie, ++opaque, 0, 9, true);
2959    upr_step(h, h1, cookie);
2960    check(upr_last_op == PROTOCOL_BINARY_CMD_DCP_STREAM_END,
2961          "Expected stream end");
2962
2963    // Wait for notification of a future item
2964    notifier_request(h, h1, cookie, ++opaque, 0, 20, true);
2965    upr_step(h, h1, cookie);
2966
2967    for (int j = 0; j < 5; ++j) {
2968        item *i = NULL;
2969        std::stringstream ss;
2970        ss << "key" << j;
2971        check(store(h, h1, NULL, OPERATION_SET, ss.str().c_str(), "data", &i)
2972              == ENGINE_SUCCESS, "Failed to store a value");
2973        h1->release(h, NULL, i);
2974    }
2975
2976    // Shouldn't get a stream end yet
2977    upr_step(h, h1, cookie);
2978    check(upr_last_op != PROTOCOL_BINARY_CMD_DCP_STREAM_END,
2979          "Wasn't expecting a stream end");
2980
2981    for (int j = 0; j < 6; ++j) {
2982        item *i = NULL;
2983        std::stringstream ss;
2984        ss << "key" << j;
2985        check(store(h, h1, NULL, OPERATION_SET, ss.str().c_str(), "data", &i)
2986              == ENGINE_SUCCESS, "Failed to store a value");
2987        h1->release(h, NULL, i);
2988    }
2989
2990    // Should get a stream end
2991    upr_step(h, h1, cookie);
2992    check(upr_last_op == PROTOCOL_BINARY_CMD_DCP_STREAM_END,
2993          "Expected stream end");
2994
2995    testHarness.destroy_cookie(