xref: /3.0.3-GA/ep-engine/tests/ep_testsuite.cc (revision 0db302e5)
1/* -*- MODE: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/*
3 *     Copyright 2010 Couchbase, Inc
4 *
5 *   Licensed under the Apache License, Version 2.0 (the "License");
6 *   you may not use this file except in compliance with the License.
7 *   You may obtain a copy of the License at
8 *
9 *       http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *   Unless required by applicable law or agreed to in writing, software
12 *   distributed under the License is distributed on an "AS IS" BASIS,
13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *   See the License for the specific language governing permissions and
15 *   limitations under the License.
16 */
17
18// Usage: (to repeatedly run just a single test case)
19// make engine_tests IS_LOOP=-L EP_TEST_NUM=3
20
21#include "config.h"
22
23#include <stdio.h>
24#include <stdlib.h>
25#include <string.h>
26#include <sys/stat.h>
27#ifdef _MSC_VER
28#include <direct.h>
29#define mkdir(a, b) _mkdir(a)
30#else
31#include <sys/wait.h>
32#endif
33
34#include <cstdlib>
35#include <iostream>
36#include <map>
37#include <set>
38#include <sstream>
39#include <string>
40#include <vector>
41
42#include <platform/dirutils.h>
43
44#include "atomic.h"
45#include "ep-engine/command_ids.h"
46#include "ep_test_apis.h"
47#include "ep_testsuite.h"
48#include "locks.h"
49#include "mock/mccouch.h"
50#include "mock/mock_upr.h"
51#include "mutex.h"
52
53#include <snappy-c.h>
54#include <JSON_checker.h>
55
56#ifdef linux
57/* /usr/include/netinet/in.h defines macros from ntohs() to _bswap_nn to
58 * optimize the conversion functions, but the prototypes generate warnings
59 * from gcc. The conversion methods isn't the bottleneck for my app, so
60 * just remove the warnings by undef'ing the optimization ..
61 */
62#undef ntohs
63#undef ntohl
64#undef htons
65#undef htonl
66#endif
67
68// ptr_fun don't like the extern "C" thing for unlock cookie.. cast it
69// away ;)
70typedef void (*UNLOCK_COOKIE_T)(const void *cookie);
71
72extern "C" bool abort_msg(const char *expr, const char *msg, int line);
73
74
75template <typename T>
76static void checkeqfn(T exp, T got, const char *msg, const char *file, const int linenum) {
77    if (exp != got) {
78        std::stringstream ss;
79        ss << "Expected `" << exp << "', got `" << got << "' - " << msg;
80        abort_msg(ss.str().c_str(), file, linenum);
81    }
82}
83
84#define checkeq(a, b, c) checkeqfn(a, b, c, __FILE__, __LINE__)
85
86extern "C" {
87
88#define check(expr, msg) \
89    static_cast<void>((expr) ? 0 : abort_msg(#expr, msg, __LINE__))
90
91#define WHITESPACE_DB "whitespace sucks.db"
92#define MULTI_DISPATCHER_CONFIG \
93    "ht_size=129;ht_locks=3;chk_remover_stime=1;chk_period=60"
94
95struct test_harness testHarness;
96
97class ThreadData {
98public:
99    ThreadData(ENGINE_HANDLE *eh, ENGINE_HANDLE_V1 *ehv1,
100               int e=0) : h(eh), h1(ehv1), extra(e) {}
101    ENGINE_HANDLE    *h;
102    ENGINE_HANDLE_V1 *h1;
103    int               extra;
104};
105
106bool abort_msg(const char *expr, const char *msg, int line) {
107    fprintf(stderr, "%s:%d Test failed: `%s' (%s)\n",
108            __FILE__, line, msg, expr);
109    abort();
110    // UNREACHABLE
111    return false;
112}
113
114static const char *dbname_env;
115static enum test_result rmdb(void)
116{
117    const char *files[] = { WHITESPACE_DB,
118                            "/tmp/test",
119                            "/tmp/mutation.log",
120                            dbname_env,
121                            NULL };
122    int ii = 0;
123    while (files[ii] != NULL) {
124        CouchbaseDirectoryUtilities::rmrf(files[ii]);
125        if (access(files[ii], F_OK) != -1) {
126            std::cerr << "Failed to remove: " << files[ii] << " " << std::endl;
127            return FAIL;
128        }
129        ++ii;
130    }
131
132    return SUCCESS;
133}
134
135static enum test_result skipped_test_function(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
136    (void) h;
137    (void) h1;
138    return SKIPPED;
139}
140
141static bool teardown(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
142    (void)h; (void)h1;
143    vals.clear();
144    return true;
145}
146
147static const void* createTapConn(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
148                                 const char *name) {
149    const void *cookie = testHarness.create_cookie();
150    testHarness.lock_cookie(cookie);
151    TAP_ITERATOR iter = h1->get_tap_iterator(h, cookie, name,
152                                             strlen(name),
153                                             TAP_CONNECT_FLAG_DUMP, NULL,
154                                             0);
155    check(iter != NULL, "Failed to create a tap iterator");
156    return cookie;
157}
158
159static void check_key_value(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
160                            const char* key, const char* val, size_t vlen,
161                            uint16_t vbucket = 0) {
162    item_info info;
163    check(get_item_info(h, h1, &info, key, vbucket), "checking key and value");
164    check(info.nvalue == 1, "info.nvalue != 1");
165    check(vlen == info.value[0].iov_len, "Value length mismatch");
166    check(memcmp(info.value[0].iov_base, val, vlen) == 0, "Data mismatch");
167}
168
169static bool test_setup(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
170    wait_for_warmup_complete(h, h1);
171    wait_for_stat_change(h, h1, "ep_vb_snapshot_total", 0);
172
173    // warmup is complete, notify ep engine that it must now enable
174    // data traffic
175    protocol_binary_request_header *pkt = createPacket(PROTOCOL_BINARY_CMD_ENABLE_TRAFFIC);
176    check(h1->unknown_command(h, NULL, pkt, add_response) == ENGINE_SUCCESS,
177          "Failed to enable data traffic");
178    free(pkt);
179
180    return true;
181}
182
183static enum test_result test_getl(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
184    const char *key = "k1";
185    uint16_t vbucketId = 0;
186    uint32_t expiration = 25;
187
188    getl(h, h1, key, vbucketId, expiration);
189    check(last_status == PROTOCOL_BINARY_RESPONSE_KEY_ENOENT,
190          "expected the key to be missing...");
191    if (last_body != NULL && (strcmp(last_body, "NOT_FOUND") != 0)) {
192        fprintf(stderr, "Should have returned NOT_FOUND. Getl Failed");
193        abort();
194    }
195
196    item *i = NULL;
197    check(store(h, h1, NULL, OPERATION_SET, key, "{\"lock\":\"data\"}",
198                &i, 0, vbucketId, 3600, PROTOCOL_BINARY_DATATYPE_JSON)
199          == ENGINE_SUCCESS, "Failed to store an item.");
200    h1->release(h, NULL, i);
201
202    /* retry getl, should succeed */
203    getl(h, h1, key, vbucketId, expiration);
204    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
205          "Expected to be able to getl on first try");
206    check(strcmp("{\"lock\":\"data\"}", last_body) == 0, "Body was malformed.");
207    check(last_datatype == PROTOCOL_BINARY_DATATYPE_JSON,
208            "Expected datatype to be JSON");
209
210    /* wait 16 seconds */
211    testHarness.time_travel(16);
212
213    /* lock's taken so this should fail */
214    getl(h, h1, key, vbucketId, expiration);
215    check(last_status == PROTOCOL_BINARY_RESPONSE_ETMPFAIL,
216          "Expected to fail getl on second try");
217
218    if (last_body != NULL && (strcmp(last_body, "LOCK_ERROR") != 0)) {
219        fprintf(stderr, "Should have returned LOCK_ERROR. Getl Failed");
220        abort();
221    }
222
223    check(store(h, h1, NULL, OPERATION_SET, key, "lockdata2", &i, 0, vbucketId)
224          != ENGINE_SUCCESS, "Should have failed to store an item.");
225    h1->release(h, NULL, i);
226
227    /* wait another 10 seconds */
228    testHarness.time_travel(10);
229
230    /* retry set, should succeed */
231    check(store(h, h1, NULL, OPERATION_SET, key, "lockdata", &i, 0, vbucketId)
232          == ENGINE_SUCCESS, "Failed to store an item.");
233    h1->release(h, NULL, i);
234
235    /* point to wrong vbucket, to test NOT_MY_VB response */
236    getl(h, h1, key, 10, expiration);
237    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
238          "Should have received not my vbucket response");
239
240    /* acquire lock, should succeed */
241    getl(h, h1, key, vbucketId, expiration);
242    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
243          "Aquire lock should have succeeded");
244    check(last_datatype == PROTOCOL_BINARY_RAW_BYTES,
245            "Expected datatype to be RAW BYTES");
246
247    /* try an incr operation followed by a delete, both of which should fail */
248    uint64_t cas = 0;
249    uint64_t result = 0;
250
251    check(h1->arithmetic(h, NULL, key, 2, true, false, 1, 1, 0,
252                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
253                         0)  == ENGINE_TMPFAIL, "Incr failed");
254
255
256    check(del(h, h1, key, 0, 0) == ENGINE_TMPFAIL, "Delete failed");
257
258
259    /* bug MB 2699 append after getl should fail with ENGINE_TMPFAIL */
260
261    testHarness.time_travel(26);
262
263    char binaryData1[] = "abcdefg\0gfedcba";
264    char binaryData2[] = "abzdefg\0gfedcba";
265
266    check(storeCasVb11(h, h1, NULL, OPERATION_SET, key,
267                       binaryData1, sizeof(binaryData1) - 1, 82758, &i, 0, 0)
268          == ENGINE_SUCCESS,
269          "Failed set.");
270    h1->release(h, NULL, i);
271
272    /* acquire lock, should succeed */
273    getl(h, h1, key, vbucketId, expiration);
274    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
275          "Aquire lock should have succeeded");
276
277    /* append should fail */
278    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, key,
279                       binaryData2, sizeof(binaryData2) - 1, 82758, &i, 0, 0)
280          == ENGINE_TMPFAIL,
281          "Append should fail.");
282    h1->release(h, NULL, i);
283
284    /* bug MB 3252 & MB 3354.
285     * 1. Set a key with an expiry value.
286     * 2. Take a lock on the item before it expires
287     * 3. Wait for the item to expire
288     * 4. Perform a CAS operation, should fail
289     * 5. Perform a set operation, should succeed
290     */
291    const char *ekey = "test_expiry";
292    const char *edata = "some test data here.";
293
294    item *it = NULL;
295
296    check(h1->allocate(h, NULL, &it, ekey, strlen(ekey), strlen(edata), 0, 2,
297          PROTOCOL_BINARY_RAW_BYTES) == ENGINE_SUCCESS, "Allocation Failed");
298
299    item_info info;
300    info.nvalue = 1;
301    if (!h1->get_item_info(h, NULL, it, &info)) {
302        abort();
303    }
304    memcpy(info.value[0].iov_base, edata, strlen(edata));
305
306    check(h1->store(h, NULL, it, &cas, OPERATION_SET, 0) ==
307        ENGINE_SUCCESS, "Failed to Store item");
308    check_key_value(h, h1, ekey, edata, strlen(edata));
309    h1->release(h, NULL, it);
310
311    testHarness.time_travel(3);
312    cas = last_cas;
313
314    /* cas should fail */
315    check(storeCasVb11(h, h1, NULL, OPERATION_CAS, ekey,
316                       binaryData1, sizeof(binaryData1) - 1, 82758, &i, cas, 0)
317          != ENGINE_SUCCESS,
318          "CAS succeeded.");
319    h1->release(h, NULL, i);
320
321    /* but a simple store should succeed */
322    check(store(h, h1, NULL, OPERATION_SET, ekey, edata, &i, 0, vbucketId)
323          == ENGINE_SUCCESS, "Failed to store an item.");
324    h1->release(h, NULL, i);
325    return SUCCESS;
326}
327
328static enum test_result test_unl(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
329
330    const char *key = "k2";
331    uint16_t vbucketId = 0;
332
333    unl(h, h1, key, vbucketId);
334    check(last_status != PROTOCOL_BINARY_RESPONSE_SUCCESS,
335          "expected the key to be missing...");
336
337    item *i = NULL;
338    check(store(h, h1, NULL, OPERATION_SET, key, "lockdata", &i, 0, vbucketId)
339          == ENGINE_SUCCESS, "Failed to store an item.");
340    h1->release(h, NULL, i);
341
342    /* getl, should succeed */
343    getl(h, h1, key, vbucketId, 0);
344    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
345          "Expected to be able to getl on first try");
346
347    /* save the returned cas value for later */
348    uint64_t cas = last_cas;
349
350    /* lock's taken unlocking with a random cas value should fail */
351    unl(h, h1, key, vbucketId);
352    check(last_status == PROTOCOL_BINARY_RESPONSE_ETMPFAIL,
353          "Expected to fail getl on second try");
354
355    if (last_body != NULL && (strcmp(last_body, "UNLOCK_ERROR") != 0)) {
356        fprintf(stderr, "Should have returned UNLOCK_ERROR. Unl Failed");
357        abort();
358    }
359
360    unl(h, h1, key, vbucketId, cas);
361    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
362          "Expected to succed unl with correct cas");
363
364    /* acquire lock, should succeed */
365    getl(h, h1, key, vbucketId, 0);
366
367    /* wait 16 seconds */
368    testHarness.time_travel(16);
369
370    /* lock has expired, unl should fail */
371    unl(h, h1, key, vbucketId, last_cas);
372    check(last_status == PROTOCOL_BINARY_RESPONSE_ETMPFAIL,
373          "Expected to fail unl on lock timeout");
374
375    return SUCCESS;
376}
377
378static enum test_result test_wrong_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
379                                               ENGINE_STORE_OPERATION op) {
380    item *i = NULL;
381    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
382    uint64_t cas = 11;
383    if (op == OPERATION_ADD) {
384        // Add operation with cas != 0 doesn't make sense
385        cas = 0;
386    }
387    check(store(h, h1, NULL, op,
388                "key", "somevalue", &i, cas, 1) == ENGINE_NOT_MY_VBUCKET,
389        "Expected not_my_vbucket");
390    h1->release(h, NULL, i);
391    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
392    return SUCCESS;
393}
394
395static enum test_result test_pending_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
396                                                 ENGINE_STORE_OPERATION op) {
397    const void *cookie = testHarness.create_cookie();
398    testHarness.set_ewouldblock_handling(cookie, false);
399    item *i = NULL;
400    check(set_vbucket_state(h, h1, 1, vbucket_state_pending), "Failed to set vbucket state.");
401    check(verify_vbucket_state(h, h1, 1, vbucket_state_pending), "Bucket state was not set to pending.");
402    uint64_t cas = 11;
403    if (op == OPERATION_ADD) {
404        // Add operation with cas != 0 doesn't make sense..
405        cas = 0;
406    }
407    check(store(h, h1, cookie, op,
408                "key", "somevalue", &i, cas, 1) == ENGINE_EWOULDBLOCK,
409        "Expected woodblock");
410    h1->release(h, NULL, i);
411    testHarness.destroy_cookie(cookie);
412    return SUCCESS;
413}
414
415static enum test_result test_replica_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
416                                                 ENGINE_STORE_OPERATION op) {
417    item *i = NULL;
418    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set vbucket state.");
419    check(verify_vbucket_state(h, h1, 1, vbucket_state_replica), "Bucket state was not set to replica.");
420    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
421
422    uint64_t cas = 11;
423    if (op == OPERATION_ADD) {
424        // performing add with a CAS != 0 doesn't make sense...
425        cas = 0;
426    }
427    check(store(h, h1, NULL, op,
428                "key", "somevalue", &i, cas, 1) == ENGINE_NOT_MY_VBUCKET,
429        "Expected not my vbucket");
430    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
431    h1->release(h, NULL, i);
432    return SUCCESS;
433}
434
435//
436// ----------------------------------------------------------------------
437// The actual tests are below.
438// ----------------------------------------------------------------------
439//
440
441static enum test_result test_get_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
442    check(verify_key(h, h1, "k") == ENGINE_KEY_ENOENT, "Expected miss.");
443    return SUCCESS;
444}
445
446static enum test_result test_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
447    item *i = NULL;
448    check(ENGINE_SUCCESS ==
449          store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
450          "Error setting.");
451    h1->release(h, NULL, i);
452    return SUCCESS;
453}
454
455struct handle_pair {
456    ENGINE_HANDLE *h;
457    ENGINE_HANDLE_V1 *h1;
458};
459
460extern "C" {
461    static void conc_del_set_thread(void *arg) {
462        struct handle_pair *hp = static_cast<handle_pair *>(arg);
463        item *it = NULL;
464
465        for (int i = 0; i < 5000; ++i) {
466            store(hp->h, hp->h1, NULL, OPERATION_ADD,
467                  "key", "somevalue", &it);
468            hp->h1->release(hp->h, NULL, it);
469            usleep(10);
470            checkeq(ENGINE_SUCCESS,
471                    store(hp->h, hp->h1, NULL, OPERATION_SET,
472                          "key", "somevalue", &it),
473                    "Error setting.");
474            hp->h1->release(hp->h, NULL, it);
475            usleep(10);
476            // Ignoring the result here -- we're racing.
477            del(hp->h, hp->h1, "key", 0, 0);
478            usleep(10);
479        }
480    }
481}
482
483static enum test_result test_conc_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
484
485    const int n_threads = 8;
486    cb_thread_t threads[n_threads];
487    struct handle_pair hp = {h, h1};
488
489    wait_for_persisted_value(h, h1, "key", "value1");
490
491    for (int i = 0; i < n_threads; i++) {
492        int r = cb_create_thread(&threads[i], conc_del_set_thread, &hp, 0);
493        cb_assert(r == 0);
494    }
495
496    for (int i = 0; i < n_threads; i++) {
497        int r = cb_join_thread(threads[i]);
498        cb_assert(r == 0);
499    }
500
501    wait_for_flusher_to_settle(h, h1);
502
503    testHarness.reload_engine(&h, &h1,
504                              testHarness.engine_path,
505                              testHarness.get_current_testcase()->cfg,
506                              true, false);
507    wait_for_warmup_complete(h, h1);
508
509    cb_assert(0 == get_int_stat(h, h1, "ep_warmed_dups"));
510
511    return SUCCESS;
512}
513
514static enum test_result test_set_get_hit(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
515    item *i = NULL;
516    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
517          "store failure");
518    check_key_value(h, h1, "key", "somevalue", 9);
519    h1->release(h, NULL, i);
520    return SUCCESS;
521}
522
523static enum test_result test_set_get_hit_bin(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
524    char binaryData[] = "abcdefg\0gfedcba";
525    cb_assert(sizeof(binaryData) != strlen(binaryData));
526
527    item *i = NULL;
528    check(ENGINE_SUCCESS ==
529          storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
530                       binaryData, sizeof(binaryData), 82758, &i, 0, 0),
531          "Failed to set.");
532    h1->release(h, NULL, i);
533    check_key_value(h, h1, "key", binaryData, sizeof(binaryData));
534    return SUCCESS;
535}
536
537static enum test_result test_set_with_cas_non_existent(ENGINE_HANDLE *h,
538                                                       ENGINE_HANDLE_V1 *h1) {
539    const char *key = "test_expiry_flush";
540    item *i = NULL;
541
542    check(h1->allocate(h, NULL, &i, key, strlen(key), 10, 0, 0,
543          PROTOCOL_BINARY_RAW_BYTES) == ENGINE_SUCCESS, "Allocation failed.");
544
545    Item *it = reinterpret_cast<Item*>(i);
546    it->setCas(1234);
547
548    uint64_t cas = 0;
549    check(h1->store(h, NULL, i, &cas, OPERATION_SET, 0) == ENGINE_KEY_ENOENT,
550          "Expected not found");
551    h1->release(h, NULL, i);
552
553    return SUCCESS;
554}
555
556static enum test_result test_set_change_flags(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
557    item *i = NULL;
558    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
559          "Failed to set.");
560    h1->release(h, NULL, i);
561
562    item_info info;
563    uint32_t flags = 828258;
564    check(get_item_info(h, h1, &info, "key"), "Failed to get value.");
565    cb_assert(info.flags != flags);
566
567    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
568                       "newvalue", strlen("newvalue"), flags, &i, 0, 0) == ENGINE_SUCCESS,
569          "Failed to set again.");
570    h1->release(h, NULL, i);
571
572    check(get_item_info(h, h1, &info, "key"), "Failed to get value.");
573
574    return info.flags == flags ? SUCCESS : FAIL;
575}
576
577static enum test_result test_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
578    item *i = NULL;
579    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
580          "Failed to do initial set.");
581    h1->release(h, NULL, i);
582    check(store(h, h1, NULL, OPERATION_CAS, "key", "failcas", &i) != ENGINE_SUCCESS,
583          "Failed to fail initial CAS.");
584    h1->release(h, NULL, i);
585    check_key_value(h, h1, "key", "somevalue", 9);
586
587    check(h1->get(h, NULL, &i, "key", 3, 0) == ENGINE_SUCCESS,
588          "Failed to get value.");
589
590    item_info info;
591    info.nvalue = 1;
592    check(h1->get_item_info(h, NULL, i, &info), "Failed to get item info.");
593    h1->release(h, NULL, i);
594
595    check(store(h, h1, NULL, OPERATION_CAS, "key", "winCas", &i,
596                info.cas) == ENGINE_SUCCESS,
597          "Failed to store CAS");
598    h1->release(h, NULL, i);
599    check_key_value(h, h1, "key", "winCas", 6);
600
601    uint64_t cval = 99999;
602    check(store(h, h1, NULL, OPERATION_CAS, "non-existing", "winCas", &i,
603                cval) == ENGINE_KEY_ENOENT,
604          "CAS for non-existing key returned the wrong error code");
605    h1->release(h, NULL, i);
606    return SUCCESS;
607}
608
609static enum test_result test_add(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
610    item *i = NULL;
611    check(store(h, h1, NULL, OPERATION_ADD,"key", "somevalue", &i) == ENGINE_SUCCESS,
612          "Failed to add value.");
613    h1->release(h, NULL, i);
614    check(store(h, h1, NULL, OPERATION_ADD,"key", "somevalue", &i) == ENGINE_NOT_STORED,
615          "Failed to fail to re-add value.");
616    h1->release(h, NULL, i);
617
618    // This aborts on failure.
619    check_key_value(h, h1, "key", "somevalue", 9);
620
621    // Expiration above was an hour, so let's go to The Future
622    testHarness.time_travel(3800);
623
624    check(store(h, h1, NULL, OPERATION_ADD,"key", "newvalue", &i) == ENGINE_SUCCESS,
625          "Failed to add value again.");
626    h1->release(h, NULL, i);
627    check_key_value(h, h1, "key", "newvalue", 8);
628    return SUCCESS;
629}
630
631static enum test_result test_add_add_with_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
632    item *i = NULL;
633    check(store(h, h1, NULL, OPERATION_ADD, "key",
634                "somevalue", &i) == ENGINE_SUCCESS,
635          "Failed set.");
636    check_key_value(h, h1, "key", "somevalue", 9);
637    item_info info;
638    info.nvalue = 1;
639    info.nvalue = 1;
640    check(h1->get_item_info(h, NULL, i, &info) == true,
641          "Should be able to get info");
642
643    item *i2 = NULL;
644    ENGINE_ERROR_CODE ret;
645    check((ret = store(h, h1, NULL, OPERATION_ADD, "key",
646                       "somevalue", &i2, info.cas)) == ENGINE_KEY_EEXISTS,
647          "Should not be able to add the key two times");
648
649    h1->release(h, NULL, i);
650    h1->release(h, NULL, i2);
651    return SUCCESS;
652}
653
654static enum test_result test_replace(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
655    item *i = NULL;
656    check(store(h, h1, NULL, OPERATION_REPLACE,"key", "somevalue", &i) != ENGINE_SUCCESS,
657          "Failed to fail to replace non-existing value.");
658    h1->release(h, NULL, i);
659    check(store(h, h1, NULL, OPERATION_SET,"key", "somevalue", &i) == ENGINE_SUCCESS,
660          "Failed to set value.");
661    h1->release(h, NULL, i);
662    check(store(h, h1, NULL, OPERATION_REPLACE,"key", "somevalue", &i) == ENGINE_SUCCESS,
663          "Failed to replace existing value.");
664    h1->release(h, NULL, i);
665    check_key_value(h, h1, "key", "somevalue", 9);
666    return SUCCESS;
667}
668
669static enum test_result test_replace_with_eviction(ENGINE_HANDLE *h,
670                                                   ENGINE_HANDLE_V1 *h1) {
671    item *i = NULL;
672    check(store(h, h1, NULL, OPERATION_SET,"key", "somevalue", &i) == ENGINE_SUCCESS,
673          "Failed to set value.");
674    h1->release(h, NULL, i);
675    wait_for_flusher_to_settle(h, h1);
676    evict_key(h, h1, "key");
677    int numBgFetched = get_int_stat(h, h1, "ep_bg_fetched");
678
679    check(store(h, h1, NULL, OPERATION_REPLACE,"key", "somevalue1", &i) == ENGINE_SUCCESS,
680          "Failed to replace existing value.");
681
682    check(h1->get_stats(h, NULL, NULL, 0, add_stats) == ENGINE_SUCCESS,
683          "Failed to get stats.");
684    std::string eviction_policy = vals.find("ep_item_eviction_policy")->second;
685    if (eviction_policy == "full_eviction") {
686        numBgFetched++;
687    }
688
689    check(get_int_stat(h, h1, "ep_bg_fetched") == numBgFetched,
690          "Bg fetched value didn't match");
691
692    h1->release(h, NULL, i);
693    check_key_value(h, h1, "key", "somevalue1", 10);
694    return SUCCESS;
695}
696
697static enum test_result test_incr_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
698    uint64_t cas = 0, result = 0;
699    h1->arithmetic(h, NULL, "key", 3, true, false, 1, 0, 0,
700                   &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
701                   0);
702    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected to not find key");
703    return SUCCESS;
704}
705
706static enum test_result test_incr_default(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
707    uint64_t cas = 0, result = 0;
708    check(h1->arithmetic(h, NULL, "key", 3, true, true, 1, 1, 0,
709                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
710                         0) == ENGINE_SUCCESS,
711          "Failed first arith");
712    check(result == 1, "Failed result verification.");
713
714    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
715                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
716                         0) == ENGINE_SUCCESS,
717          "Failed second arith.");
718    check(result == 2, "Failed second result verification.");
719
720    check(h1->arithmetic(h, NULL, "key", 3, true, true, 1, 1, 0,
721                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
722                         0) == ENGINE_SUCCESS,
723          "Failed third arith.");
724    check(result == 3, "Failed third result verification.");
725
726    check_key_value(h, h1, "key", "3", 1);
727    return SUCCESS;
728}
729
730static enum test_result test_append(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
731    item *i = NULL;
732
733    // MB-11332: append on non-existing key should return NOT_STORED
734    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key",
735                       "foo\r\n", 5, 82758, &i, 0, 0)
736          == ENGINE_NOT_STORED,
737          "MB-11332: Failed append.");
738
739    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
740                       "\r\n", 2, 82758, &i, 0, 0)
741          == ENGINE_SUCCESS,
742          "Failed set.");
743    h1->release(h, NULL, i);
744
745    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key",
746                       "foo\r\n", 5, 82758, &i, 0, 0)
747          == ENGINE_SUCCESS,
748          "Failed append.");
749    h1->release(h, NULL, i);
750
751    check_key_value(h, h1, "key", "\r\nfoo\r\n", 7);
752
753    char binaryData1[] = "abcdefg\0gfedcba\r\n";
754    char binaryData2[] = "abzdefg\0gfedcba\r\n";
755    size_t dataSize = 20*1024*1024;
756    char *bigBinaryData3 = new char[dataSize];
757    memset(bigBinaryData3, '\0', dataSize);
758
759    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
760                       binaryData1, sizeof(binaryData1) - 1, 82758, &i, 0, 0)
761          == ENGINE_SUCCESS,
762          "Failed set.");
763    h1->release(h, NULL, i);
764
765    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key",
766                       bigBinaryData3, dataSize, 82758, &i, 0, 0)
767          == ENGINE_E2BIG,
768          "Expected append failure.");
769    h1->release(h, NULL, i);
770    delete[] bigBinaryData3;
771
772    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key",
773                       binaryData2, sizeof(binaryData2) - 1, 82758, &i, 0, 0)
774          == ENGINE_SUCCESS,
775          "Failed append.");
776    h1->release(h, NULL, i);
777
778    std::string expected;
779    expected.append(binaryData1, sizeof(binaryData1) - 1);
780    expected.append(binaryData2, sizeof(binaryData2) - 1);
781
782    check_key_value(h, h1, "key", expected.data(), expected.length());
783    return SUCCESS;
784}
785
786static enum test_result test_prepend(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
787    item *i = NULL;
788
789    // MB-11332: prepend on non-existing key should return NOT_STORED
790    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key",
791                       "foo\r\n", 5, 82758, &i, 0, 0)
792          == ENGINE_NOT_STORED,
793          "MB-11332: Failed prepend.");
794
795
796    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
797                       "\r\n", 2, 82758, &i, 0, 0)
798          == ENGINE_SUCCESS,
799          "Failed set.");
800    h1->release(h, NULL, i);
801
802    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key",
803                       "foo\r\n", 5, 82758, &i, 0, 0)
804          == ENGINE_SUCCESS,
805          "Failed append.");
806    h1->release(h, NULL, i);
807
808    check_key_value(h, h1, "key", "foo\r\n\r\n", 7);
809
810    char binaryData1[] = "abcdefg\0gfedcba\r\n";
811    char binaryData2[] = "abzdefg\0gfedcba\r\n";
812    size_t dataSize = 20*1024*1024;
813    char *bigBinaryData3 = new char[dataSize];
814    memset(bigBinaryData3, '\0', dataSize);
815
816    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
817                       binaryData1, sizeof(binaryData1) - 1, 82758, &i, 0, 0)
818          == ENGINE_SUCCESS,
819          "Failed set.");
820    h1->release(h, NULL, i);
821
822    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key",
823                       bigBinaryData3, dataSize, 82758, &i, 0, 0)
824          == ENGINE_E2BIG,
825          "Expected prepend failure.");
826    h1->release(h, NULL, i);
827    delete[] bigBinaryData3;
828
829    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key",
830                       binaryData2, sizeof(binaryData2) - 1, 82758, &i, 0, 0)
831          == ENGINE_SUCCESS,
832          "Failed append.");
833    h1->release(h, NULL, i);
834
835    std::string expected;
836    expected.append(binaryData2, sizeof(binaryData2) - 1);
837    expected.append(binaryData1, sizeof(binaryData1) - 1);
838
839    check_key_value(h, h1, "key", expected.data(), expected.length());
840    return SUCCESS;
841}
842
843static enum test_result test_append_compressed(ENGINE_HANDLE *h,
844                                               ENGINE_HANDLE_V1 *h1) {
845
846    item *i = NULL;
847
848    size_t len = snappy_max_compressed_length(2);
849    char *newBuf = (char *) malloc (len);
850    snappy_compress("\r\n", 2, newBuf, &len);
851    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key1",
852                       (const char *)newBuf, len, 82758, &i, 0, 0, 3600,
853                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
854          == ENGINE_SUCCESS, "Failed set.");
855    h1->release(h, NULL, i);
856    free (newBuf);
857
858    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key1",
859                       "foo\r\n", 5, 82758, &i, 0, 0)
860          == ENGINE_SUCCESS,
861          "Failed append uncompressed to compressed.");
862    h1->release(h, NULL, i);
863
864    size_t len1 = snappy_max_compressed_length(7);
865    char *newBuf1 = (char *) malloc (len1);
866    snappy_compress("\r\nfoo\r\n", 7, newBuf1, &len1);
867
868    item_info info;
869    check(get_item_info(h, h1, &info, "key1", 0), "checking key and value");
870    check(info.nvalue == 1, "info.nvalue != 1");
871    check(len1 == info.value[0].iov_len, "Value length mismatch");
872    check(memcmp(info.value[0].iov_base, newBuf1, len1) == 0, "Data mismatch");
873    check(info.datatype == 0x02, "Datatype mismatch");
874    free (newBuf1);
875
876    len = snappy_max_compressed_length(3);
877    newBuf = (char *) malloc (len);
878    snappy_compress("bar", 3, newBuf, &len);
879    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key1",
880                       (const char*)newBuf, len, 82758, &i, 0, 0, 3600,
881                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
882            == ENGINE_SUCCESS,
883            "Failed append compressed to compressed.");
884    h1->release(h, NULL, i);
885    free (newBuf);
886
887    len1 = snappy_max_compressed_length(10);
888    newBuf1 = (char *) malloc (len1);
889    snappy_compress("\r\nfoo\r\nbar", 10, newBuf1, &len1);
890
891    check(get_item_info(h, h1, &info, "key1", 0), "checking key and value");
892    check(info.nvalue == 1, "info.nvalue != 1");
893    check(len1 == info.value[0].iov_len, "Value length mismatch");
894    check(memcmp(info.value[0].iov_base, newBuf1, len1) == 0, "Data mismatch");
895    check(info.datatype == 0x02, "Datatype mismatch");
896    free (newBuf1);
897
898    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key2",
899                       "foo", 3, 82758, &i, 0, 0, 3600,
900                       PROTOCOL_BINARY_RAW_BYTES)
901          == ENGINE_SUCCESS, "Failed set.");
902    h1->release(h, NULL, i);
903
904    len = snappy_max_compressed_length(3);
905    newBuf = (char *) malloc (len);
906    snappy_compress("bar", 3, newBuf, &len);
907    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key2",
908                       newBuf, len, 82758, &i, 0, 0, 3600,
909                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
910            == ENGINE_SUCCESS,
911            "Failed append compressed to uncompressed.");
912    h1->release(h, NULL, i);
913    free (newBuf);
914
915    check(get_item_info(h, h1, &info, "key2", 0), "checking key and value");
916    check(info.nvalue == 1, "info.nvalue != 1");
917    check(info.value[0].iov_len == 6, "Value length mismatch");
918    check(memcmp(info.value[0].iov_base, "foobar", 6) == 0, "Data mismatch");
919    check(info.datatype == 0x00, "Datatype mismatch");
920
921    return SUCCESS;
922}
923
924static enum test_result test_prepend_compressed(ENGINE_HANDLE *h,
925                                               ENGINE_HANDLE_V1 *h1) {
926
927    item *i = NULL;
928
929    size_t len = snappy_max_compressed_length(2);
930    char *newBuf = (char *) malloc (len);
931    snappy_compress("\r\n", 2, newBuf, &len);
932    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key1",
933                       (const char *)newBuf, len, 82758, &i, 0, 0, 3600,
934                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
935          == ENGINE_SUCCESS, "Failed set.");
936    h1->release(h, NULL, i);
937    free (newBuf);
938
939    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key1",
940                       "foo\r\n", 5, 82758, &i, 0, 0)
941          == ENGINE_SUCCESS,
942          "Failed prepend uncompressed to compressed.");
943    h1->release(h, NULL, i);
944
945    size_t len1 = snappy_max_compressed_length(7);
946    char *newBuf1 = (char *) malloc (len1);
947    snappy_compress("foo\r\n\r\n", 7, newBuf1, &len1);
948
949    item_info info;
950    check(get_item_info(h, h1, &info, "key1", 0), "checking key and value");
951    check(info.nvalue == 1, "info.nvalue != 1");
952    check(len1 == info.value[0].iov_len, "Value length mismatch");
953    check(memcmp(info.value[0].iov_base, newBuf1, len1) == 0, "Data mismatch");
954    check(info.datatype == 0x02, "Datatype mismatch");
955    free (newBuf1);
956
957    len = snappy_max_compressed_length(3);
958    newBuf = (char *) malloc (len);
959    snappy_compress("bar", 3, newBuf, &len);
960    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key1",
961                       (const char*)newBuf, len, 82758, &i, 0, 0, 3600,
962                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
963            == ENGINE_SUCCESS,
964            "Failed prepend compressed to compressed.");
965    h1->release(h, NULL, i);
966    free (newBuf);
967
968    len1 = snappy_max_compressed_length(10);
969    newBuf1 = (char *) malloc (len1);
970    snappy_compress("barfoo\r\n\r\n", 10, newBuf1, &len1);
971
972    check(get_item_info(h, h1, &info, "key1", 0), "checking key and value");
973    check(info.nvalue == 1, "info.nvalue != 1");
974    check(len1 == info.value[0].iov_len, "Value length mismatch");
975    check(memcmp(info.value[0].iov_base, newBuf1, len1) == 0, "Data mismatch");
976    check(info.datatype == 0x02, "Datatype mismatch");
977    free (newBuf1);
978
979    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key2",
980                       "foo", 3, 82758, &i, 0, 0, 3600,
981                       PROTOCOL_BINARY_RAW_BYTES)
982          == ENGINE_SUCCESS, "Failed set.");
983    h1->release(h, NULL, i);
984
985    len = snappy_max_compressed_length(3);
986    newBuf = (char *) malloc (len);
987    snappy_compress("bar", 3, newBuf, &len);
988    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key2",
989                       newBuf, len, 82758, &i, 0, 0, 3600,
990                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
991            == ENGINE_SUCCESS,
992            "Failed prepend compressed to uncompressed.");
993    h1->release(h, NULL, i);
994    free (newBuf);
995
996    check(get_item_info(h, h1, &info, "key2", 0), "checking key and value");
997    check(info.nvalue == 1, "info.nvalue != 1");
998    check(info.value[0].iov_len == 6, "Value length mismatch");
999    check(memcmp(info.value[0].iov_base, "barfoo", 6) == 0, "Data mismatch");
1000    check(info.datatype == 0x00, "Datatype mismatch");
1001
1002    return SUCCESS;
1003}
1004
1005static enum test_result test_append_prepend_to_json(ENGINE_HANDLE *h,
1006                                                    ENGINE_HANDLE_V1 *h1) {
1007    item *i = NULL;
1008    item_info info;
1009
1010    const char* key1 = "foo1";
1011    const char* key2 = "foo2";
1012    const char* value1 = "{\"foo1\":\"bar1\"}";
1013    const char* value2 = "{\"foo2\":\"bar2\"}";
1014
1015    // APPEND
1016    check(storeCasVb11(h, h1, NULL, OPERATION_SET, key1,
1017                       value1, strlen(value1), 82758, &i, 0, 0,
1018                       3600, PROTOCOL_BINARY_DATATYPE_JSON)
1019          == ENGINE_SUCCESS, "Failed set.");
1020    h1->release(h, NULL, i);
1021
1022    check(h1->get(h, NULL, &i, key1, strlen(key1), 0) == ENGINE_SUCCESS,
1023            "Unable to get stored item");
1024    h1->release(h, NULL, i);
1025    info.nvalue = 1;
1026    h1->get_item_info(h, NULL, i, &info);
1027    check(checkUTF8JSON((const unsigned char*)info.value[0].iov_base,
1028                        (int)info.value[0].iov_len) == 1, "Expected JSON");
1029    check(info.datatype == PROTOCOL_BINARY_DATATYPE_JSON, "Invalid datatype");
1030
1031    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, key1,
1032                       value2, strlen(value2), 82758, &i, 0, 0)
1033          == ENGINE_SUCCESS,
1034          "Failed append.");
1035    h1->release(h, NULL, i);
1036
1037    check(h1->get(h, NULL, &i, key1, strlen(key1), 0) == ENGINE_SUCCESS,
1038            "Unable to get stored item");
1039    info.nvalue = 1;
1040    h1->get_item_info(h, NULL, i, &info);
1041    h1->release(h, NULL, i);
1042    check(checkUTF8JSON((const unsigned char*)info.value[0].iov_base,
1043                        (int)info.value[0].iov_len) == 0, "Expected Binary");
1044    check(info.datatype == PROTOCOL_BINARY_RAW_BYTES,
1045                "Invalid datatype after append");
1046
1047    // PREPEND
1048    check(storeCasVb11(h, h1, NULL, OPERATION_SET, key2,
1049                       value1, strlen(value1), 82758, &i, 0, 0,
1050                       3600, PROTOCOL_BINARY_DATATYPE_JSON)
1051          == ENGINE_SUCCESS, "Failed set.");
1052    h1->release(h, NULL, i);
1053
1054    check(h1->get(h, NULL, &i, key2, strlen(key2), 0) == ENGINE_SUCCESS,
1055            "Unable to get stored item");
1056    info.nvalue = 1;
1057    h1->get_item_info(h, NULL, i, &info);
1058    h1->release(h, NULL, i);
1059    check(checkUTF8JSON((const unsigned char*)info.value[0].iov_base,
1060                        (int)info.value[0].iov_len) == 1, "Expected JSON");
1061    check(info.datatype == PROTOCOL_BINARY_DATATYPE_JSON, "Invalid datatype");
1062
1063    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, key2,
1064                       value2, strlen(value2), 82758, &i, 0, 0)
1065          == ENGINE_SUCCESS,
1066          "Failed prepend.");
1067    h1->release(h, NULL, i);
1068
1069    check(h1->get(h, NULL, &i, key2, strlen(key2), 0) == ENGINE_SUCCESS,
1070            "Unable to get stored item");
1071    h1->release(h, NULL, i);
1072    info.nvalue = 1;
1073    h1->get_item_info(h, NULL, i, &info);
1074    check(checkUTF8JSON((const unsigned char*)info.value[0].iov_base,
1075                        (int)info.value[0].iov_len) == 0, "Expected Binary");
1076    check(info.datatype == PROTOCOL_BINARY_RAW_BYTES,
1077                "Invalid datatype after prepend");
1078
1079    return SUCCESS;
1080}
1081
1082static enum test_result test_incr(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1083    uint64_t cas = 0, result = 0;
1084    item *i = NULL;
1085    check(store(h, h1, NULL, OPERATION_ADD,"key", "1", &i) == ENGINE_SUCCESS,
1086          "Failed to add value.");
1087    h1->release(h, NULL, i);
1088
1089    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
1090                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1091                         0) == ENGINE_SUCCESS,
1092          "Failed to incr value.");
1093
1094    check_key_value(h, h1, "key", "2", 1);
1095    return SUCCESS;
1096}
1097
1098static enum test_result test_bug2799(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1099    uint64_t cas = 0, result = 0;
1100    item *i = NULL;
1101    check(store(h, h1, NULL, OPERATION_ADD, "key", "1", &i) == ENGINE_SUCCESS,
1102          "Failed to add value.");
1103    h1->release(h, NULL, i);
1104
1105    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
1106                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1107                         0) == ENGINE_SUCCESS,
1108          "Failed to incr value.");
1109
1110    check_key_value(h, h1, "key", "2", 1);
1111
1112    testHarness.time_travel(3617);
1113
1114    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1115    return SUCCESS;
1116}
1117
1118static enum test_result test_flush(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1119    item *i = NULL;
1120    // First try to delete something we know to not be there.
1121    check(del(h, h1, "key", 0, 0) == ENGINE_KEY_ENOENT, "Failed to fail initial delete.");
1122    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1123          "Failed set.");
1124    h1->release(h, NULL, i);
1125    check_key_value(h, h1, "key", "somevalue", 9);
1126    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS,
1127          "Failed to flush");
1128    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1129
1130    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1131          "Failed post-flush set.");
1132    h1->release(h, NULL, i);
1133    check_key_value(h, h1, "key", "somevalue", 9);
1134
1135    return SUCCESS;
1136}
1137
1138static enum test_result test_flush_disabled(ENGINE_HANDLE *h,
1139                                            ENGINE_HANDLE_V1 *h1) {
1140    item *i = NULL;
1141    // start an engine with disabled flush, the flush() should be noop and
1142    // we expect to see the key after flush()
1143
1144    // store a key and check its existence
1145    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1146          "Failed set.");
1147    h1->release(h, NULL, i);
1148    check_key_value(h, h1, "key", "somevalue", 9);
1149    // expect error msg engine does not support operation
1150    check(h1->flush(h, NULL, 0) == ENGINE_ENOTSUP, "Flush should be disabled");
1151    //check the key
1152    check(ENGINE_SUCCESS == verify_key(h, h1, "key"), "Expected key");
1153
1154    // restart engine with flush enabled and redo the test, we expect flush to succeed
1155    std::string param = "flushall_enabled=false";
1156    std::string config = testHarness.get_current_testcase()->cfg;
1157    size_t found = config.find(param);
1158    if(found != config.npos) {
1159        config.replace(found, param.size(), "flushall_enabled=true");
1160    }
1161    testHarness.reload_engine(&h, &h1,
1162                              testHarness.engine_path,
1163                              config.c_str(),
1164                              true, false);
1165    wait_for_warmup_complete(h, h1);
1166
1167    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS, "Flush should be enabled");
1168
1169    //expect missing key
1170    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1171
1172    return SUCCESS;
1173}
1174
1175static enum test_result test_flush_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1176    item *i = NULL;
1177    int mem_used = get_int_stat(h, h1, "mem_used");
1178    int overhead = get_int_stat(h, h1, "ep_overhead");
1179    int cacheSize = get_int_stat(h, h1, "ep_total_cache_size");
1180    int nonResident = get_int_stat(h, h1, "ep_num_non_resident");
1181
1182    int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
1183    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1184          "Failed set.");
1185    h1->release(h, NULL, i);
1186    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i) == ENGINE_SUCCESS,
1187          "Failed set.");
1188    h1->release(h, NULL, i);
1189    testHarness.time_travel(65);
1190    wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
1191
1192    check(ENGINE_SUCCESS == verify_key(h, h1, "key"), "Expected key");
1193    check(ENGINE_SUCCESS == verify_key(h, h1, "key2"), "Expected key2");
1194
1195    check_key_value(h, h1, "key", "somevalue", 9);
1196    check_key_value(h, h1, "key2", "somevalue", 9);
1197
1198    int mem_used2 = get_int_stat(h, h1, "mem_used");
1199    int overhead2 = get_int_stat(h, h1, "ep_overhead");
1200    int cacheSize2 = get_int_stat(h, h1, "ep_total_cache_size");
1201
1202    cb_assert(mem_used2 > mem_used);
1203    // "mem_used2 - overhead2" (i.e., ep_kv_size) should be greater than the hashtable cache size
1204    // due to the checkpoint overhead
1205    cb_assert(mem_used2 - overhead2 > cacheSize2);
1206
1207    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS, "Failed to flush");
1208    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1209    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key2"), "Expected missing key");
1210
1211    wait_for_flusher_to_settle(h, h1);
1212
1213    mem_used2 = get_int_stat(h, h1, "mem_used");
1214    overhead2 = get_int_stat(h, h1, "ep_overhead");
1215    cacheSize2 = get_int_stat(h, h1, "ep_total_cache_size");
1216    int nonResident2 = get_int_stat(h, h1, "ep_num_non_resident");
1217
1218    cb_assert(mem_used2 == mem_used);
1219    cb_assert(overhead2 == overhead);
1220    cb_assert(nonResident2 == nonResident);
1221    cb_assert(cacheSize2 == cacheSize);
1222
1223    return SUCCESS;
1224}
1225
1226static enum test_result test_flush_multiv(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1227    item *i = NULL;
1228    check(set_vbucket_state(h, h1, 2, vbucket_state_active), "Failed to set vbucket state.");
1229    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1230          "Failed set.");
1231    h1->release(h, NULL, i);
1232    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i,
1233                0, 2) == ENGINE_SUCCESS,
1234          "Failed set in vb2.");
1235    h1->release(h, NULL, i);
1236
1237    check(ENGINE_SUCCESS == verify_key(h, h1, "key"), "Expected key");
1238    check(ENGINE_SUCCESS == verify_key(h, h1, "key2", 2), "Expected key2");
1239
1240    check_key_value(h, h1, "key", "somevalue", 9);
1241    check_key_value(h, h1, "key2", "somevalue", 9, 2);
1242
1243    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS, "Failed to flush");
1244
1245    vals.clear();
1246    check(h1->get_stats(h, NULL, NULL, 0, add_stats) == ENGINE_SUCCESS,
1247          "Failed to get stats.");
1248    check(vals.find("ep_flush_all") != vals.end(), "Failed to get the status of flush_all");
1249
1250    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1251    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key2", 2), "Expected missing key");
1252
1253    return SUCCESS;
1254}
1255
1256static int checkCurrItemsAfterShutdown(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1257                                       int numItems2Load, bool shutdownForce) {
1258    std::vector<std::string> keys;
1259    for (int index = 0; index < numItems2Load; ++index) {
1260        std::stringstream s;
1261        s << "keys_2_load-" << index;
1262        std::string key(s.str());
1263        keys.push_back(key);
1264    }
1265
1266    check(get_int_stat(h, h1, "ep_total_persisted") == 0,
1267          "Expected ep_total_persisted equals 0");
1268    check(get_int_stat(h, h1, "curr_items") == 0,
1269          "Expected curr_items equals 0");
1270
1271    // stop flusher before loading new items
1272    protocol_binary_request_header *pkt = createPacket(PROTOCOL_BINARY_CMD_STOP_PERSISTENCE);
1273    check(h1->unknown_command(h, NULL, pkt, add_response) == ENGINE_SUCCESS,
1274          "CMD_STOP_PERSISTENCE failed!");
1275    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
1276          "Failed to stop persistence!");
1277    free(pkt);
1278
1279    std::vector<std::string>::iterator itr;
1280    for (itr = keys.begin(); itr != keys.end(); ++itr) {
1281        item *i;
1282        check(store(h, h1, NULL, OPERATION_SET, itr->c_str(), "oracle", &i, 0, 0)
1283              == ENGINE_SUCCESS, "Failed to store a value");
1284        h1->release(h, NULL, i);
1285    }
1286
1287    check(get_int_stat(h, h1, "ep_total_persisted") == 0,
1288          "Incorrect ep_total_persisted, expected 0");
1289    std::stringstream ss;
1290    ss << "Incorrect curr_items, expected " << numItems2Load;
1291    std::string errmsg(ss.str());
1292    check(get_int_stat(h, h1, "curr_items") == numItems2Load,
1293          errmsg.c_str());
1294
1295    // resume flusher before shutdown + warmup
1296    pkt = createPacket(PROTOCOL_BINARY_CMD_START_PERSISTENCE);
1297    check(h1->unknown_command(h, NULL, pkt, add_response) == ENGINE_SUCCESS,
1298          "CMD_START_PERSISTENCE failed!");
1299    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
1300          "Failed to start persistence!");
1301    free(pkt);
1302
1303    // shutdown engine force and restart
1304    testHarness.reload_engine(&h, &h1,
1305                              testHarness.engine_path,
1306                              testHarness.get_current_testcase()->cfg,
1307                              true, shutdownForce);
1308    wait_for_warmup_complete(h, h1);
1309    return get_int_stat(h, h1, "curr_items");
1310}
1311
1312static enum test_result test_flush_shutdown_force(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1313    int numItems2load = 3000;
1314    bool shutdownForce = true;
1315    int currItems = checkCurrItemsAfterShutdown(h, h1, numItems2load, shutdownForce);
1316    check (currItems <= numItems2load,
1317           "Number of curr items should be <= 3000, unless previous "
1318           "shutdown force had to wait for the flusher");
1319    return SUCCESS;
1320}
1321
1322static enum test_result test_flush_shutdown_noforce(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1323    int numItems2load = 3000;
1324    bool shutdownForce = false;
1325    int currItems = checkCurrItemsAfterShutdown(h, h1, numItems2load, shutdownForce);
1326    check (currItems == numItems2load,
1327           "Number of curr items should be equal to 3000, unless previous "
1328           "shutdown did not wait for the flusher");
1329    return SUCCESS;
1330}
1331
1332static enum test_result test_flush_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1333    item *i = NULL;
1334    // First try to delete something we know to not be there.
1335    check(del(h, h1, "key", 0, 0) == ENGINE_KEY_ENOENT, "Failed to fail initial delete.");
1336    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1337          "Failed set.");
1338    h1->release(h, NULL, i);
1339    check_key_value(h, h1, "key", "somevalue", 9);
1340
1341    // Restart once to ensure written to disk.
1342    testHarness.reload_engine(&h, &h1,
1343                              testHarness.engine_path,
1344                              testHarness.get_current_testcase()->cfg,
1345                              true, false);
1346    wait_for_warmup_complete(h, h1);
1347
1348    // Read value from disk.
1349    check_key_value(h, h1, "key", "somevalue", 9);
1350
1351    // Flush
1352    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS,
1353          "Failed to flush");
1354
1355    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i) == ENGINE_SUCCESS,
1356          "Failed post-flush set.");
1357    h1->release(h, NULL, i);
1358    check_key_value(h, h1, "key2", "somevalue", 9);
1359
1360    // Restart again, ensure written to disk.
1361    testHarness.reload_engine(&h, &h1,
1362                              testHarness.engine_path,
1363                              testHarness.get_current_testcase()->cfg,
1364                              true, false);
1365    wait_for_warmup_complete(h, h1);
1366
1367    check(store(h, h1, NULL, OPERATION_SET, "key3", "somevalue", &i) == ENGINE_SUCCESS,
1368          "Failed post-flush, post-restart set.");
1369    h1->release(h, NULL, i);
1370    check_key_value(h, h1, "key3", "somevalue", 9);
1371
1372    // Read value again, should not be there.
1373    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1374    return SUCCESS;
1375}
1376
1377static enum test_result test_flush_multiv_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1378    item *i = NULL;
1379    check(set_vbucket_state(h, h1, 2, vbucket_state_active), "Failed to set vbucket state.");
1380    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1381          "Failed set.");
1382    h1->release(h, NULL, i);
1383    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i,
1384                0, 2) == ENGINE_SUCCESS,
1385          "Failed set in vb2.");
1386    h1->release(h, NULL, i);
1387
1388    // Restart once to ensure written to disk.
1389    testHarness.reload_engine(&h, &h1,
1390                              testHarness.engine_path,
1391                              testHarness.get_current_testcase()->cfg,
1392                              true, false);
1393    wait_for_warmup_complete(h, h1);
1394
1395    // Read value from disk.
1396    check_key_value(h, h1, "key", "somevalue", 9);
1397
1398    // Flush
1399    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS,
1400          "Failed to flush");
1401
1402    // Restart again, ensure written to disk.
1403    testHarness.reload_engine(&h, &h1,
1404                              testHarness.engine_path,
1405                              testHarness.get_current_testcase()->cfg,
1406                              true, false);
1407    wait_for_warmup_complete(h, h1);
1408
1409    // Read value again, should not be there.
1410    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1411    check(verify_vbucket_missing(h, h1, 2), "Bucket 2 came back.");
1412    return SUCCESS;
1413}
1414
1415static enum test_result test_delete(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1416    item *i = NULL;
1417    // First try to delete something we know to not be there.
1418    check(del(h, h1, "key", 0, 0) == ENGINE_KEY_ENOENT, "Failed to fail initial delete.");
1419    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1420          "Failed set.");
1421    Item *it = reinterpret_cast<Item*>(i);
1422    uint64_t orig_cas = it->getCas();
1423    h1->release(h, NULL, i);
1424    check_key_value(h, h1, "key", "somevalue", 9);
1425
1426    uint64_t cas = 0;
1427    check(h1->remove(h, NULL, "key", 3, &cas, 0) == ENGINE_SUCCESS,
1428          "Failed remove with value.");
1429    check(orig_cas + 1 == cas, "Cas mismatch on delete");
1430    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1431
1432    // Can I time travel to an expired object and delete it?
1433    checkeq(ENGINE_SUCCESS, store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1434            "Failed set.");
1435    h1->release(h, NULL, i);
1436    testHarness.time_travel(3617);
1437    checkeq(ENGINE_KEY_ENOENT, del(h, h1, "key", 0, 0),
1438            "Did not get ENOENT removing an expired object.");
1439    checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"), "Expected missing key");
1440
1441    return SUCCESS;
1442}
1443
1444static enum test_result test_set_delete(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1445    item *i = NULL;
1446    checkeq(ENGINE_SUCCESS, store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1447            "Failed set.");
1448    h1->release(h, NULL, i);
1449    check_key_value(h, h1, "key", "somevalue", 9);
1450    checkeq(ENGINE_SUCCESS, del(h, h1, "key", 0, 0),
1451            "Failed remove with value.");
1452    checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"), "Expected missing key");
1453    wait_for_flusher_to_settle(h, h1);
1454    wait_for_stat_to_be(h, h1, "curr_items", 0);
1455    return SUCCESS;
1456}
1457
1458static enum test_result test_set_delete_invalid_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1459    item *i = NULL;
1460    check(store(h, h1, NULL, OPERATION_SET, "key",
1461                "somevalue", &i) == ENGINE_SUCCESS,
1462          "Failed set.");
1463    check_key_value(h, h1, "key", "somevalue", 9);
1464    item_info info;
1465    info.nvalue = 1;
1466    check(h1->get_item_info(h, NULL, i, &info) == true,
1467          "Should be able to get info");
1468    h1->release(h, NULL, i);
1469
1470    check(del(h, h1, "key", info.cas + 1, 0) == ENGINE_KEY_EEXISTS,
1471          "Didn't expect to be able to remove the item with wrong cas");
1472    return SUCCESS;
1473}
1474
1475static enum test_result test_bug2509(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1476    for (int j = 0; j < 10000; ++j) {
1477        item *itm = NULL;
1478        checkeq(ENGINE_SUCCESS,
1479                store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &itm),
1480                "Failed set.");
1481        h1->release(h, NULL, itm);
1482        usleep(10);
1483        checkeq(ENGINE_SUCCESS, del(h, h1, "key", 0, 0), "Failed remove with value.");
1484        usleep(10);
1485    }
1486
1487    // Restart again, to verify we don't have any duplicates.
1488    testHarness.reload_engine(&h, &h1,
1489                              testHarness.engine_path,
1490                              testHarness.get_current_testcase()->cfg,
1491                              true, false);
1492    wait_for_warmup_complete(h, h1);
1493
1494    return get_int_stat(h, h1, "ep_warmup_dups") == 0 ? SUCCESS : FAIL;
1495}
1496
1497static enum test_result test_bug7023(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1498    std::vector<std::string> keys;
1499    // Make a vbucket mess.
1500    for (int j = 0; j < 10000; ++j) {
1501        std::stringstream ss;
1502        ss << "key" << j;
1503        std::string key(ss.str());
1504        keys.push_back(key);
1505    }
1506
1507    std::vector<std::string>::iterator it;
1508    for (int j = 0; j < 5; ++j) {
1509        check(set_vbucket_state(h, h1, 0, vbucket_state_dead), "Failed set set vbucket 0 dead.");
1510        vbucketDelete(h, h1, 0);
1511        check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
1512              "Expected vbucket deletion to work.");
1513        check(set_vbucket_state(h, h1, 0, vbucket_state_active), "Failed set set vbucket 0 active.");
1514        for (it = keys.begin(); it != keys.end(); ++it) {
1515            item *i;
1516            check(store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str(), &i)
1517                  == ENGINE_SUCCESS, "Failed to store a value");
1518        }
1519    }
1520    wait_for_flusher_to_settle(h, h1);
1521
1522    // Restart again, to verify no data loss.
1523    testHarness.reload_engine(&h, &h1,
1524                              testHarness.engine_path,
1525                              testHarness.get_current_testcase()->cfg,
1526                              true, false);
1527    wait_for_warmup_complete(h, h1);
1528    return get_int_stat(h, h1, "ep_warmup_value_count", "warmup") == 10000 ? SUCCESS : FAIL;
1529}
1530
1531static enum test_result test_delete_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1532    wait_for_persisted_value(h, h1, "key", "value1");
1533
1534    check(del(h, h1, "key", 0, 0) == ENGINE_SUCCESS, "Failed remove with value.");
1535
1536    wait_for_persisted_value(h, h1, "key", "value2");
1537
1538    testHarness.reload_engine(&h, &h1,
1539                              testHarness.engine_path,
1540                              testHarness.get_current_testcase()->cfg,
1541                              true, false);
1542    wait_for_warmup_complete(h, h1);
1543
1544    check_key_value(h, h1, "key", "value2", 6);
1545    check(del(h, h1, "key", 0, 0) == ENGINE_SUCCESS, "Failed remove with value.");
1546    wait_for_flusher_to_settle(h, h1);
1547
1548    testHarness.reload_engine(&h, &h1,
1549                              testHarness.engine_path,
1550                              testHarness.get_current_testcase()->cfg,
1551                              true, false);
1552    wait_for_warmup_complete(h, h1);
1553
1554    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1555
1556    return SUCCESS;
1557}
1558
1559static enum test_result test_get_delete_missing_file(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1560    const char *key = "key";
1561    wait_for_persisted_value(h, h1, key, "value2delete");
1562
1563    // whack the db file and directory where the key is stored
1564    rmdb();
1565
1566    item *i = NULL;
1567    int errorCode = h1->get(h, NULL, &i, key, strlen(key), 0);
1568    h1->release(h, NULL, i);
1569
1570    // ep engine must be unaware of well-being of the db file as long as
1571    // the item is still in the memory
1572    check(errorCode == ENGINE_SUCCESS, "Expected success for get");
1573
1574    i = NULL;
1575    evict_key(h, h1, key);
1576    errorCode = h1->get(h, NULL, &i, key, strlen(key), 0);
1577    h1->release(h, NULL, i);
1578
1579    // ep engine must be now aware of the ill-fated db file where
1580    // the item is supposedly stored
1581    check(errorCode == ENGINE_TMPFAIL, "Expected tmp fail for get");
1582
1583    return SUCCESS;
1584}
1585
1586
1587static enum test_result test_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1588    item *i = NULL;
1589    static const char val[] = "somevalue";
1590    ENGINE_ERROR_CODE ret;
1591    check((ret = store(h, h1, NULL, OPERATION_SET, "key", val, &i)) == ENGINE_SUCCESS,
1592          "Failed set.");
1593    h1->release(h, NULL, i);
1594
1595    testHarness.reload_engine(&h, &h1,
1596                              testHarness.engine_path,
1597                              testHarness.get_current_testcase()->cfg,
1598                              true, false);
1599    wait_for_warmup_complete(h, h1);
1600    check_key_value(h, h1, "key", val, strlen(val));
1601    return SUCCESS;
1602}
1603
1604static enum test_result test_restart_session_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1605    createTapConn(h, h1, "tap_client_thread");
1606
1607    testHarness.reload_engine(&h, &h1,
1608                              testHarness.engine_path,
1609                              testHarness.get_current_testcase()->cfg,
1610                              true, false);
1611    wait_for_warmup_complete(h, h1);
1612    createTapConn(h, h1, "tap_client_thread");
1613
1614    check(h1->get_stats(h, NULL, "tap", 3, add_stats) == ENGINE_SUCCESS,
1615          "Failed to get stats.");
1616    std::string val = vals["eq_tapq:tap_client_thread:backfill_completed"];
1617    check(strcmp(val.c_str(), "true") == 0, "Don't expect the backfill upon restart");
1618    return SUCCESS;
1619}
1620
1621static enum test_result test_specialKeys(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1622    item *i = NULL;
1623    ENGINE_ERROR_CODE ret;
1624
1625    // Simplified Chinese "Couchbase"
1626    static const char key0[] = "沙发数据库";
1627    static const char val0[] = "some Chinese value";
1628    check((ret = store(h, h1, NULL, OPERATION_SET, key0, val0, &i)) == ENGINE_SUCCESS,
1629          "Failed set Chinese key");
1630    check_key_value(h, h1, key0, val0, strlen(val0));
1631    h1->release(h, NULL, i);
1632    // Traditional Chinese "Couchbase"
1633    static const char key1[] = "沙發數據庫";
1634    static const char val1[] = "some Traditional Chinese value";
1635    check((ret = store(h, h1, NULL, OPERATION_SET, key1, val1, &i)) == ENGINE_SUCCESS,
1636          "Failed set Traditional Chinese key");
1637    h1->release(h, NULL, i);
1638    // Korean "couch potato"
1639    static const char key2[] = "쇼파감자";
1640    static const char val2[] = "some Korean value";
1641    check((ret = store(h, h1, NULL, OPERATION_SET, key2, val2, &i)) == ENGINE_SUCCESS,
1642          "Failed set Korean key");
1643    h1->release(h, NULL, i);
1644    // Russian "couch potato"
1645    static const char key3[] = "лодырь, лентяй";
1646    static const char val3[] = "some Russian value";
1647    check((ret = store(h, h1, NULL, OPERATION_SET, key3, val3, &i)) == ENGINE_SUCCESS,
1648          "Failed set Russian key");
1649    h1->release(h, NULL, i);
1650    // Japanese "couch potato"
1651    static const char key4[] = "カウチポテト";
1652    static const char val4[] = "some Japanese value";
1653    check((ret = store(h, h1, NULL, OPERATION_SET, key4, val4, &i)) == ENGINE_SUCCESS,
1654          "Failed set Japanese key");
1655    h1->release(h, NULL, i);
1656    // Indian char key, and no idea what it is
1657    static const char key5[] = "हरियानवी";
1658    static const char val5[] = "some Indian value";
1659    check((ret = store(h, h1, NULL, OPERATION_SET, key5, val5, &i)) == ENGINE_SUCCESS,
1660          "Failed set Indian key");
1661    h1->release(h, NULL, i);
1662    // Portuguese translation "couch potato"
1663    static const char key6[] = "sedentário";
1664    static const char val6[] = "some Portuguese value";
1665    check((ret = store(h, h1, NULL, OPERATION_SET, key6, val6, &i)) == ENGINE_SUCCESS,
1666          "Failed set Portuguese key");
1667    h1->release(h, NULL, i);
1668    // Arabic translation "couch potato"
1669    static const char key7[] = "الحافلةالبطاطة";
1670    static const char val7[] = "some Arabic value";
1671    check((ret = store(h, h1, NULL, OPERATION_SET, key7, val7, &i)) == ENGINE_SUCCESS,
1672          "Failed set Arabic key");
1673    h1->release(h, NULL, i);
1674
1675    testHarness.reload_engine(&h, &h1,
1676                              testHarness.engine_path,
1677                              testHarness.get_current_testcase()->cfg,
1678                              true, false);
1679    wait_for_warmup_complete(h, h1);
1680    check_key_value(h, h1, key0, val0, strlen(val0));
1681    check_key_value(h, h1, key1, val1, strlen(val1));
1682    check_key_value(h, h1, key2, val2, strlen(val2));
1683    check_key_value(h, h1, key3, val3, strlen(val3));
1684    check_key_value(h, h1, key4, val4, strlen(val4));
1685    check_key_value(h, h1, key5, val5, strlen(val5));
1686    check_key_value(h, h1, key6, val6, strlen(val6));
1687    check_key_value(h, h1, key7, val7, strlen(val7));
1688    return SUCCESS;
1689}
1690
1691static enum test_result test_binKeys(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1692    item *i = NULL;
1693    ENGINE_ERROR_CODE ret;
1694
1695    // binary key with char values beyond 0x7F
1696    static const char key0[] = "\xe0\xed\xf1\x6f\x7f\xf8\xfa";
1697    static const char val0[] = "some value val8";
1698    check((ret = store(h, h1, NULL, OPERATION_SET, key0, val0, &i)) == ENGINE_SUCCESS,
1699          "Failed set binary key0");
1700    check_key_value(h, h1, key0, val0, strlen(val0));
1701    h1->release(h, NULL, i);
1702    // binary keys with char values beyond 0x7F
1703    static const char key1[] = "\xf1\xfd\xfe\xff\xf0\xf8\xef";
1704    static const char val1[] = "some value val9";
1705    check((ret = store(h, h1, NULL, OPERATION_SET, key1, val1, &i)) == ENGINE_SUCCESS,
1706          "Failed set binary key1");
1707    check_key_value(h, h1, key1, val1, strlen(val1));
1708    h1->release(h, NULL, i);
1709    // binary keys with special utf-8 BOM (Byte Order Mark) values 0xBB 0xBF 0xEF
1710    static const char key2[] = "\xff\xfe\xbb\xbf\xef";
1711    static const char val2[] = "some utf-8 bom value";
1712    check((ret = store(h, h1, NULL, OPERATION_SET, key2, val2, &i)) == ENGINE_SUCCESS,
1713          "Failed set binary utf-8 bom key");
1714    check_key_value(h, h1, key2, val2, strlen(val2));
1715    h1->release(h, NULL, i);
1716    // binary keys with special utf-16BE BOM values "U+FEFF"
1717    static const char key3[] = "U+\xfe\xff\xefU+\xff\xfe";
1718    static const char val3[] = "some utf-16 bom value";
1719    check((ret = store(h, h1, NULL, OPERATION_SET, key3, val3, &i)) == ENGINE_SUCCESS,
1720          "Failed set binary utf-16 bom key");
1721    check_key_value(h, h1, key3, val3, strlen(val3));
1722    h1->release(h, NULL, i);
1723
1724    testHarness.reload_engine(&h, &h1,
1725                              testHarness.engine_path,
1726                              testHarness.get_current_testcase()->cfg,
1727                              true, false);
1728    wait_for_warmup_complete(h, h1);
1729    check_key_value(h, h1, key0, val0, strlen(val0));
1730    check_key_value(h, h1, key1, val1, strlen(val1));
1731    check_key_value(h, h1, key2, val2, strlen(val2));
1732    check_key_value(h, h1, key3, val3, strlen(val3));
1733    return SUCCESS;
1734}
1735
1736static enum test_result test_restart_bin_val(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1737
1738
1739
1740    char binaryData[] = "abcdefg\0gfedcba";
1741    cb_assert(sizeof(binaryData) != strlen(binaryData));
1742
1743    item *i = NULL;
1744    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
1745                       binaryData, sizeof(binaryData), 82758, &i, 0, 0)
1746          == ENGINE_SUCCESS,
1747          "Failed set.");
1748    h1->release(h, NULL, i);
1749
1750    testHarness.reload_engine(&h, &h1,
1751                              testHarness.engine_path,
1752                              testHarness.get_current_testcase()->cfg,
1753                              true, false);
1754    wait_for_warmup_complete(h, h1);
1755
1756    check_key_value(h, h1, "key", binaryData, sizeof(binaryData));
1757    return SUCCESS;
1758}
1759
1760static enum test_result test_wrong_vb_get(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1761    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1762    check(ENGINE_NOT_MY_VBUCKET == verify_key(h, h1, "key", 1),
1763          "Expected wrong bucket.");
1764    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1765    return SUCCESS;
1766}
1767
1768static enum test_result test_vb_get_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1769    check(set_vbucket_state(h, h1, 1, vbucket_state_pending), "Failed to set vbucket state.");
1770    const void *cookie = testHarness.create_cookie();
1771    testHarness.set_ewouldblock_handling(cookie, false);
1772
1773    item *i = NULL;
1774    check(ENGINE_EWOULDBLOCK == h1->get(h, cookie, &i, "key", strlen("key"), 1),
1775          "Expected woodblock.");
1776    h1->release(h, NULL, i);
1777
1778    testHarness.destroy_cookie(cookie);
1779    return SUCCESS;
1780}
1781
1782static enum test_result test_vb_get_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1783    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set vbucket state.");
1784    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1785    check(ENGINE_NOT_MY_VBUCKET == verify_key(h, h1, "key", 1),
1786          "Expected not my bucket.");
1787    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1788    return SUCCESS;
1789}
1790
1791static enum test_result test_wrong_vb_incr(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1792    uint64_t cas, result;
1793    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1794    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
1795                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1796                         1) == ENGINE_NOT_MY_VBUCKET,
1797          "Expected not my vbucket.");
1798    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1799    return SUCCESS;
1800}
1801
1802static enum test_result test_vb_incr_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1803    const void *cookie = testHarness.create_cookie();
1804    testHarness.set_ewouldblock_handling(cookie, false);
1805    uint64_t cas, result;
1806    check(set_vbucket_state(h, h1, 1, vbucket_state_pending), "Failed to set vbucket state.");
1807    check(h1->arithmetic(h, cookie, "key", 3, true, false, 1, 1, 0,
1808                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1809                         1) == ENGINE_EWOULDBLOCK,
1810          "Expected woodblock.");
1811    testHarness.destroy_cookie(cookie);
1812    return SUCCESS;
1813}
1814
1815static enum test_result test_vb_incr_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1816    uint64_t cas, result;
1817    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set vbucket state.");
1818    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1819    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
1820                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1821                         1) == ENGINE_NOT_MY_VBUCKET,
1822          "Expected not my bucket.");
1823    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1824    return SUCCESS;
1825}
1826
1827static enum test_result test_wrong_vb_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1828    return test_wrong_vb_mutation(h, h1, OPERATION_SET);
1829}
1830
1831static enum test_result test_wrong_vb_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1832    return test_wrong_vb_mutation(h, h1, OPERATION_CAS);
1833}
1834
1835static enum test_result test_wrong_vb_add(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1836    return test_wrong_vb_mutation(h, h1, OPERATION_ADD);
1837}
1838
1839static enum test_result test_wrong_vb_replace(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1840    return test_wrong_vb_mutation(h, h1, OPERATION_REPLACE);
1841}
1842
1843static enum test_result test_wrong_vb_append(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1844    return test_wrong_vb_mutation(h, h1, OPERATION_APPEND);
1845}
1846
1847static enum test_result test_wrong_vb_prepend(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1848    return test_wrong_vb_mutation(h, h1, OPERATION_PREPEND);
1849}
1850
1851static enum test_result test_wrong_vb_del(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1852    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1853    check(ENGINE_NOT_MY_VBUCKET == del(h, h1, "key", 0, 1), "Expected wrong bucket.");
1854    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1855    return SUCCESS;
1856}
1857
1858static enum test_result test_expiry(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1859    const char *key = "test_expiry";
1860    const char *data = "some test data here.";
1861
1862    item *it = NULL;
1863
1864    ENGINE_ERROR_CODE rv;
1865    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 2,
1866                      PROTOCOL_BINARY_RAW_BYTES);
1867    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1868
1869    item_info info;
1870    info.nvalue = 1;
1871    if (!h1->get_item_info(h, NULL, it, &info)) {
1872        abort();
1873    }
1874    memcpy(info.value[0].iov_base, data, strlen(data));
1875
1876    uint64_t cas = 0;
1877    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
1878    check(rv == ENGINE_SUCCESS, "Set failed.");
1879    check_key_value(h, h1, key, data, strlen(data));
1880    h1->release(h, NULL, it);
1881
1882    testHarness.time_travel(5);
1883    check(h1->get(h, NULL, &it, key, strlen(key), 0) == ENGINE_KEY_ENOENT,
1884          "Item didn't expire");
1885
1886    int expired_access = get_int_stat(h, h1, "ep_expired_access");
1887    int expired_pager = get_int_stat(h, h1, "ep_expired_pager");
1888    int active_expired = get_int_stat(h, h1, "vb_active_expired");
1889    check(expired_pager == 0, "Expected zero expired item by pager");
1890    check(expired_access == 1, "Expected an expired item on access");
1891    check(active_expired == 1, "Expected an expired active item");
1892    checkeq(ENGINE_SUCCESS, store(h, h1, NULL, OPERATION_SET, key, data, &it),
1893            "Failed set.");
1894    h1->release(h, NULL, it);
1895
1896    std::stringstream ss;
1897    ss << "curr_items stat should be still 1 after ";
1898    ss << "overwriting the key that was expired, but not purged yet";
1899    checkeq(1, get_int_stat(h, h1, "curr_items"), ss.str().c_str());
1900
1901    return SUCCESS;
1902}
1903
1904static enum test_result test_expiry_loader(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1905    const char *key = "test_expiry_loader";
1906    const char *data = "some test data here.";
1907
1908    item *it = NULL;
1909
1910    ENGINE_ERROR_CODE rv;
1911    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 2,
1912                      PROTOCOL_BINARY_RAW_BYTES);
1913    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1914
1915    item_info info;
1916    info.nvalue = 1;
1917    if (!h1->get_item_info(h, NULL, it, &info)) {
1918        abort();
1919    }
1920    memcpy(info.value[0].iov_base, data, strlen(data));
1921
1922    uint64_t cas = 0;
1923    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
1924    check(rv == ENGINE_SUCCESS, "Set failed.");
1925    check_key_value(h, h1, key, data, strlen(data));
1926    h1->release(h, NULL, it);
1927
1928    testHarness.time_travel(3);
1929
1930    check(h1->get(h, NULL, &it, key, strlen(key), 0) == ENGINE_KEY_ENOENT,
1931          "Item didn't expire");
1932
1933    // Restart the engine to ensure the above expired item is not loaded
1934    testHarness.reload_engine(&h, &h1,
1935                              testHarness.engine_path,
1936                              testHarness.get_current_testcase()->cfg,
1937                              true, false);
1938    wait_for_warmup_complete(h, h1);
1939    cb_assert(0 == get_int_stat(h, h1, "ep_warmup_value_count", "warmup"));
1940
1941    return SUCCESS;
1942}
1943
1944static enum test_result test_expiration_on_warmup(ENGINE_HANDLE *h,
1945                                                  ENGINE_HANDLE_V1 *h1) {
1946    const char *key = "KEY";
1947    const char *data = "VALUE";
1948
1949    item *it = NULL;
1950
1951    ENGINE_ERROR_CODE rv;
1952    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 3,
1953                      PROTOCOL_BINARY_RAW_BYTES);
1954    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1955
1956    item_info info;
1957    info.nvalue = 1;
1958    if (!h1->get_item_info(h, NULL, it, &info)) {
1959        abort();
1960    }
1961    memcpy(info.value[0].iov_base, data, strlen(data));
1962
1963    uint64_t cas = 0;
1964    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
1965    check(rv == ENGINE_SUCCESS, "Set failed.");
1966    check_key_value(h, h1, key, data, strlen(data));
1967    h1->release(h, NULL, it);
1968    wait_for_flusher_to_settle(h, h1);
1969
1970    check(get_int_stat(h, h1, "curr_items") == 1, "Failed store item");
1971    testHarness.time_travel(5);
1972
1973    // Restart the engine to ensure the above item is expired
1974    testHarness.reload_engine(&h, &h1,
1975                              testHarness.engine_path,
1976                              testHarness.get_current_testcase()->cfg,
1977                              true, false);
1978    wait_for_warmup_complete(h, h1);
1979    wait_for_flusher_to_settle(h, h1);
1980    check(get_int_stat(h, h1, "curr_items") == 0,
1981            "The item should have been expired.");
1982
1983    return SUCCESS;
1984
1985}
1986
1987static enum test_result test_bug3454(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1988    const char *key = "test_expiry_duplicate_warmup";
1989    const char *data = "some test data here.";
1990
1991    item *it = NULL;
1992
1993    ENGINE_ERROR_CODE rv;
1994    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 5,
1995                      PROTOCOL_BINARY_RAW_BYTES);
1996    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1997
1998    item_info info;
1999    info.nvalue = 1;
2000    if (!h1->get_item_info(h, NULL, it, &info)) {
2001        abort();
2002    }
2003    memcpy(info.value[0].iov_base, data, strlen(data));
2004
2005    uint64_t cas = 0;
2006    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
2007    check(rv == ENGINE_SUCCESS, "Set failed.");
2008    check_key_value(h, h1, key, data, strlen(data));
2009    h1->release(h, NULL, it);
2010    wait_for_flusher_to_settle(h, h1);
2011
2012    // Advance the ep_engine time by 10 sec for the above item to be expired.
2013    testHarness.time_travel(10);
2014    check(h1->get(h, NULL, &it, key, strlen(key), 0) == ENGINE_KEY_ENOENT,
2015          "Item didn't expire");
2016
2017    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 0,
2018                      PROTOCOL_BINARY_RAW_BYTES);
2019    check(rv == ENGINE_SUCCESS, "Allocation failed.");
2020
2021    info.nvalue = 1;
2022    if (!h1->get_item_info(h, NULL, it, &info)) {
2023        abort();
2024    }
2025    memcpy(info.value[0].iov_base, data, strlen(data));
2026
2027    cas = 0;
2028    // Add a new item with the same key.
2029    rv = h1->store(h, NULL, it, &cas, OPERATION_ADD, 0);
2030    check(rv == ENGINE_SUCCESS, "Add failed.");
2031    check_key_value(h, h1, key, data, strlen(data));
2032    h1->release(h, NULL, it);
2033
2034    check(h1->get(h, NULL, &it, key, strlen(key), 0) == ENGINE_SUCCESS,
2035          "Item shouldn't expire");
2036    h1->release(h, NULL, it);
2037
2038    // Restart the engine to ensure the above unexpired new item is loaded
2039    testHarness.reload_engine(&h, &h1,
2040                              testHarness.engine_path,
2041                              testHarness.get_current_testcase()->cfg,
2042                              true, false);
2043    wait_for_warmup_complete(h, h1);
2044    cb_assert(1 == get_int_stat(h, h1, "ep_warmup_value_count", "warmup"));
2045    cb_assert(0 == get_int_stat(h, h1, "ep_warmup_dups", "warmup"));
2046
2047    return SUCCESS;
2048}
2049
2050static enum test_result test_bug3522(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2051    const char *key = "test_expiry_no_items_warmup";
2052    const char *data = "some test data here.";
2053
2054    item *it = NULL;
2055
2056    ENGINE_ERROR_CODE rv;
2057    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 0,
2058                      PROTOCOL_BINARY_RAW_BYTES);
2059    check(rv == ENGINE_SUCCESS, "Allocation failed.");
2060
2061    item_info info;
2062    info.nvalue = 1;
2063    if (!h1->get_item_info(h, NULL, it, &info)) {
2064        abort();
2065    }
2066    memcpy(info.value[0].iov_base, data, strlen(data));
2067
2068    uint64_t cas = 0;
2069    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
2070    check(rv == ENGINE_SUCCESS, "Set failed.");
2071    check_key_value(h, h1, key, data, strlen(data));
2072    h1->release(h, NULL, it);
2073    wait_for_flusher_to_settle(h, h1);
2074
2075    // Add a new item with the same key and 2 sec of expiration.
2076    const char *new_data = "new data here.";
2077    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(new_data), 0, 2,
2078                      PROTOCOL_BINARY_RAW_BYTES);
2079    check(rv == ENGINE_SUCCESS, "Allocation failed.");
2080
2081    info.nvalue = 1;
2082    if (!h1->get_item_info(h, NULL, it, &info)) {
2083        abort();
2084    }
2085    memcpy(info.value[0].iov_base, new_data, strlen(new_data));
2086
2087    int pager_runs = get_int_stat(h, h1, "ep_num_expiry_pager_runs");
2088    cas = 0;
2089    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
2090    check(rv == ENGINE_SUCCESS, "Set failed.");
2091    check_key_value(h, h1, key, new_data, strlen(new_data));
2092    h1->release(h, NULL, it);
2093    testHarness.time_travel(3);
2094    wait_for_stat_change(h, h1, "ep_num_expiry_pager_runs", pager_runs);
2095    wait_for_flusher_to_settle(h, h1);
2096
2097    // Restart the engine.
2098    testHarness.reload_engine(&h, &h1,
2099                              testHarness.engine_path,
2100                              testHarness.get_current_testcase()->cfg,
2101                              true, false);
2102    wait_for_warmup_complete(h, h1);
2103    // TODO: modify this for a better test case
2104    cb_assert(0 == get_int_stat(h, h1, "ep_warmup_dups", "warmup"));
2105
2106    return SUCCESS;
2107}
2108
2109static enum test_result test_get_replica_active_state(ENGINE_HANDLE *h,
2110                                                      ENGINE_HANDLE_V1 *h1) {
2111    protocol_binary_request_header *pkt;
2112    pkt = prepare_get_replica(h, h1, vbucket_state_active);
2113    check(h1->unknown_command(h, NULL, pkt, add_response) ==
2114          ENGINE_SUCCESS, "Get Replica Failed");
2115    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2116          "Expected PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET response.");
2117
2118    return SUCCESS;
2119}
2120
2121static enum test_result test_get_replica_pending_state(ENGINE_HANDLE *h,
2122                                                       ENGINE_HANDLE_V1 *h1) {
2123    protocol_binary_request_header *pkt;
2124
2125    const void *cookie = testHarness.create_cookie();
2126    testHarness.set_ewouldblock_handling(cookie, false);
2127    pkt = prepare_get_replica(h, h1, vbucket_state_pending);
2128    check(h1->unknown_command(h, cookie, pkt, add_response) ==
2129          ENGINE_EWOULDBLOCK, "Should have returned error for pending state");
2130    testHarness.destroy_cookie(cookie);
2131    return SUCCESS;
2132}
2133
2134static enum test_result test_get_replica_dead_state(ENGINE_HANDLE *h,
2135                                                    ENGINE_HANDLE_V1 *h1) {
2136    protocol_binary_request_header *pkt;
2137    pkt = prepare_get_replica(h, h1, vbucket_state_dead);
2138    check(h1->unknown_command(h, NULL, pkt, add_response) ==
2139          ENGINE_SUCCESS, "Get Replica Failed");
2140    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2141          "Expected PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET response.");
2142
2143    return SUCCESS;
2144}
2145
2146static enum test_result test_get_replica(ENGINE_HANDLE *h,
2147                                         ENGINE_HANDLE_V1 *h1) {
2148    protocol_binary_request_header *pkt;
2149    pkt = prepare_get_replica(h, h1, vbucket_state_replica);
2150    check(h1->unknown_command(h, NULL, pkt, add_response) == ENGINE_SUCCESS,
2151                              "Get Replica Failed");
2152    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
2153          "Expected PROTOCOL_BINARY_RESPONSE_SUCCESS response.");
2154    check(strcmp("replicadata", last_body) == 0,
2155                 "Should have returned identical value");
2156
2157    return SUCCESS;
2158}
2159
2160static enum test_result test_get_replica_non_resident(ENGINE_HANDLE *h,
2161                                                      ENGINE_HANDLE_V1 *h1) {
2162
2163    item *i = NULL;
2164    check(store(h, h1, NULL, OPERATION_SET, "key", "value", &i, 0, 0)
2165          == ENGINE_SUCCESS, "Store Failed");
2166    h1->release(h, NULL, i);
2167    wait_for_flusher_to_settle(h, h1);
2168    wait_for_stat_to_be(h, h1, "ep_total_persisted", 1);
2169
2170    evict_key(h, h1, "key", 0, "Ejected.");
2171    check(set_vbucket_state(h, h1, 0, vbucket_state_replica),
2172          "Failed to set vbucket to replica");
2173
2174    get_replica(h, h1, "key", 0);
2175    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "Expected success");
2176
2177    return SUCCESS;
2178}
2179
2180static enum test_result test_get_replica_invalid_key(ENGINE_HANDLE *h,
2181                                                     ENGINE_HANDLE_V1 *h1) {
2182    protocol_binary_request_header *pkt;
2183    bool makeinvalidkey = true;
2184    pkt = prepare_get_replica(h, h1, vbucket_state_replica, makeinvalidkey);
2185    check(h1->unknown_command(h, NULL, pkt, add_response) ==
2186          ENGINE_SUCCESS, "Get Replica Failed");
2187    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2188          "Expected PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET response.");
2189    return SUCCESS;
2190}
2191
2192static enum test_result test_vb_del_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2193    const void *cookie = testHarness.create_cookie();
2194    testHarness.set_ewouldblock_handling(cookie, false);
2195    check(set_vbucket_state(h, h1, 1, vbucket_state_pending), "Failed to set vbucket state.");
2196    check(ENGINE_EWOULDBLOCK == del(h, h1, "key", 0, 1, cookie),
2197          "Expected woodblock.");
2198    testHarness.destroy_cookie(cookie);
2199    return SUCCESS;
2200}
2201
2202static enum test_result test_vb_del_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2203    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set vbucket state.");
2204    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
2205    check(ENGINE_NOT_MY_VBUCKET == del(h, h1, "key", 0, 1),
2206          "Expected not my vbucket.");
2207    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
2208    return SUCCESS;
2209}
2210
2211static enum test_result test_touch(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2212    // key is a mandatory field!
2213    touch(h, h1, NULL, 0, (time(NULL) + 10));
2214    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2215
2216    // extlen is a mandatory field!
2217    protocol_binary_request_header *request;
2218    request = createPacket(PROTOCOL_BINARY_CMD_TOUCH, 0, 0, NULL, 0, "akey", 4);
2219    check(h1->unknown_command(h, NULL, request, add_response) == ENGINE_SUCCESS,
2220          "Failed to call touch");
2221    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2222    free(request);
2223
2224    // Try to touch an unknown item...
2225    touch(h, h1, "mykey", 0, (time(NULL) + 10));
2226    check(last_status == PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, "Testing unknown key");
2227
2228    // illegal vbucket
2229    touch(h, h1, "mykey", 5, (time(NULL) + 10));
2230    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET, "Testing illegal vbucket");
2231
2232    // Store the item!
2233    item *itm = NULL;
2234    check(store(h, h1, NULL, OPERATION_SET, "mykey", "somevalue", &itm) == ENGINE_SUCCESS,
2235          "Failed set.");
2236    h1->release(h, NULL, itm);
2237
2238    check_key_value(h, h1, "mykey", "somevalue", strlen("somevalue"));
2239
2240    touch(h, h1, "mykey", 0, (time(NULL) + 10));
2241    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch mykey");
2242
2243    // time-travel 9 secs..
2244    testHarness.time_travel(9);
2245
2246    // The item should still exist
2247    check_key_value(h, h1, "mykey", "somevalue", 9);
2248
2249    // time-travel 2 secs..
2250    testHarness.time_travel(2);
2251
2252    // The item should have expired now...
2253    check(h1->get(h, NULL, &itm, "mykey", 5, 0) == ENGINE_KEY_ENOENT, "Item should be gone");
2254    return SUCCESS;
2255}
2256
2257static enum test_result test_touch_mb7342(ENGINE_HANDLE *h,
2258                                          ENGINE_HANDLE_V1 *h1) {
2259    const char *key = "MB-7342";
2260    // Store the item!
2261    item *itm = NULL;
2262    check(store(h, h1, NULL, OPERATION_SET, key, "v", &itm) == ENGINE_SUCCESS,
2263          "Failed set.");
2264    h1->release(h, NULL, itm);
2265
2266    touch(h, h1, key, 0, 0);
2267    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch key");
2268
2269    check_key_value(h, h1, key, "v", 1);
2270
2271    // Travel a loong time to see if the object is still there (the default
2272    // store sets an exp time of 3600
2273    testHarness.time_travel(3700);
2274
2275    check_key_value(h, h1, key, "v", 1);
2276
2277    return SUCCESS;
2278}
2279
2280static enum test_result test_touch_mb10277(ENGINE_HANDLE *h,
2281                                            ENGINE_HANDLE_V1 *h1) {
2282    const char *key = "MB-10277";
2283    // Store the item!
2284    item *itm = NULL;
2285    check(store(h, h1, NULL, OPERATION_SET, key, "v", &itm) == ENGINE_SUCCESS,
2286          "Failed set.");
2287    h1->release(h, NULL, itm);
2288    wait_for_flusher_to_settle(h, h1);
2289    evict_key(h, h1, key, 0, "Ejected.");
2290
2291    touch(h, h1, key, 0, 3600); // A new expiration time remains in the same.
2292    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch key");
2293
2294    return SUCCESS;
2295}
2296
2297static enum test_result test_gat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2298    // key is a mandatory field!
2299    gat(h, h1, NULL, 0, 10);
2300    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2301
2302    // extlen is a mandatory field!
2303    protocol_binary_request_header *request;
2304    request = createPacket(PROTOCOL_BINARY_CMD_GAT, 0, 0, NULL, 0, "akey", 4);
2305    check(h1->unknown_command(h, NULL, request, add_response) == ENGINE_SUCCESS,
2306          "Failed to call gat");
2307    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2308    free(request);
2309
2310    // Try to gat an unknown item...
2311    gat(h, h1, "mykey", 0, 10);
2312    check(last_status == PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, "Testing unknown key");
2313
2314    // illegal vbucket
2315    gat(h, h1, "mykey", 5, 10);
2316    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET, "Testing illegal vbucket");
2317
2318    // Store the item!
2319    item *itm = NULL;
2320    check(store(h, h1, NULL, OPERATION_SET, "mykey", "{\"some\":\"value\"}",
2321                &itm, 0, 0, 3600, PROTOCOL_BINARY_DATATYPE_JSON) == ENGINE_SUCCESS,
2322          "Failed set.");
2323    h1->release(h, NULL, itm);
2324
2325    check_key_value(h, h1, "mykey", "{\"some\":\"value\"}",
2326            strlen("{\"some\":\"value\"}"));
2327
2328    gat(h, h1, "mykey", 0, 10);
2329    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "gat mykey");
2330    check(last_datatype == PROTOCOL_BINARY_DATATYPE_JSON, "Expected datatype to be JSON");
2331    check(memcmp(last_body, "{\"some\":\"value\"}", sizeof("{\"some\":\"value\"}")) == 0,
2332          "Invalid data returned");
2333
2334    // time-travel 9 secs..
2335    testHarness.time_travel(9);
2336
2337    // The item should still exist
2338    check_key_value(h, h1, "mykey", "{\"some\":\"value\"}",
2339                    strlen("{\"some\":\"value\"}"));
2340
2341    // time-travel 2 secs..
2342    testHarness.time_travel(2);
2343
2344    // The item should have expired now...
2345    check(h1->get(h, NULL, &itm, "mykey", 5, 0) == ENGINE_KEY_ENOENT, "Item should be gone");
2346    return SUCCESS;
2347}
2348
2349static enum test_result test_gatq(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2350    // key is a mandatory field!
2351    gat(h, h1, NULL, 0, 10, true);
2352    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2353
2354    // extlen is a mandatory field!
2355    protocol_binary_request_header *request;
2356    request = createPacket(PROTOCOL_BINARY_CMD_GATQ, 0, 0, NULL, 0, "akey", 4);
2357    check(h1->unknown_command(h, NULL, request, add_response) == ENGINE_SUCCESS,
2358          "Failed to call gatq");
2359    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2360    free(request);
2361
2362    // Try to gatq an unknown item...
2363    last_status = static_cast<protocol_binary_response_status>(0xffff);
2364    gat(h, h1, "mykey", 0, 10, true);
2365
2366    // We should not have sent any response!
2367    check(last_status == (protocol_binary_response_status)0xffff, "Testing unknown key");
2368
2369    // illegal vbucket
2370    gat(h, h1, "mykey", 5, 10, true);
2371    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2372          "Testing illegal vbucket");
2373
2374    // Store the item!
2375    item *itm = NULL;
2376    check(store(h, h1, NULL, OPERATION_SET, "mykey", "{\"some\":\"value\"}",
2377                &itm, 0, 0, 3600, PROTOCOL_BINARY_DATATYPE_JSON) == ENGINE_SUCCESS,
2378          "Failed set.");
2379    h1->release(h, NULL, itm);
2380
2381    check_key_value(h, h1, "mykey", "{\"some\":\"value\"}",
2382                    strlen("{\"some\":\"value\"}"));
2383
2384    gat(h, h1, "mykey", 0, 10, true);
2385    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "gat mykey");
2386    check(last_datatype == PROTOCOL_BINARY_DATATYPE_JSON, "Expected datatype to be JSON");
2387    check(memcmp(last_body, "{\"some\":\"value\"}", sizeof("{\"some\":\"value\"}")) == 0,
2388          "Invalid data returned");
2389
2390    // time-travel 9 secs..
2391    testHarness.time_travel(9);
2392
2393    // The item should still exist
2394    check_key_value(h, h1, "mykey", "{\"some\":\"value\"}",
2395                    strlen("{\"some\":\"value\"}"));
2396
2397    // time-travel 2 secs..
2398    testHarness.time_travel(2);
2399
2400    // The item should have expired now...
2401    check(h1->get(h, NULL, &itm, "mykey", 5, 0) == ENGINE_KEY_ENOENT, "Item should be gone");
2402    return SUCCESS;
2403}
2404
2405static enum test_result test_mb5215(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2406    item *itm = NULL;
2407    check(store(h, h1, NULL, OPERATION_SET, "coolkey", "cooler", &itm)
2408          == ENGINE_SUCCESS, "Failed set.");
2409    h1->release(h, NULL, itm);
2410
2411    check_key_value(h, h1, "coolkey", "cooler", strlen("cooler"));
2412
2413    // set new exptime to 111
2414    int expTime = time(NULL) + 111;
2415
2416    touch(h, h1, "coolkey", 0, expTime);
2417    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch coolkey");
2418
2419    //reload engine
2420    testHarness.reload_engine(&h, &h1,
2421                              testHarness.engine_path,
2422                              testHarness.get_current_testcase()->cfg,
2423                              true, false);
2424    wait_for_warmup_complete(h, h1);
2425
2426    //verify persisted expiration time
2427    const char *statkey = "key coolkey 0";
2428    int newExpTime;
2429    check(h1->get(h, NULL, &itm, "coolkey", 7, 0) == ENGINE_SUCCESS,
2430          "Missing key");
2431    h1->release(h, NULL, itm);
2432    newExpTime = get_int_stat(h, h1, "key_exptime", statkey);
2433    check(newExpTime == expTime, "Failed to persist new exptime");
2434
2435    // evict key, touch expiration time, and verify
2436    evict_key(h, h1, "coolkey", 0, "Ejected.");
2437
2438    expTime = time(NULL) + 222;
2439    touch(h, h1, "coolkey", 0, expTime);
2440    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch coolkey");
2441
2442    testHarness.reload_engine(&h, &h1,
2443                              testHarness.engine_path,
2444                              testHarness.get_current_testcase()->cfg,
2445                              true, false);
2446    wait_for_warmup_complete(h, h1);
2447
2448    check(h1->get(h, NULL, &itm, "coolkey", 7, 0) == ENGINE_SUCCESS,
2449          "Missing key");
2450    h1->release(h, NULL, itm);
2451    newExpTime = get_int_stat(h, h1, "key_exptime", statkey);
2452    check(newExpTime == expTime, "Failed to persist new exptime");
2453
2454    return SUCCESS;
2455}
2456
2457static enum test_result test_alloc_limit(ENGINE_HANDLE *h,
2458                                         ENGINE_HANDLE_V1 *h1) {
2459    item *it = NULL;
2460    ENGINE_ERROR_CODE rv;
2461
2462    rv = h1->allocate(h, NULL, &it, "key", 3, 20 * 1024 * 1024, 0, 0,
2463                      PROTOCOL_BINARY_RAW_BYTES);
2464    check(rv == ENGINE_SUCCESS, "Allocated 20MB item");
2465    h1->release(h, NULL, it);
2466
2467    rv = h1->allocate(h, NULL, &it, "key", 3, (20 * 1024 * 1024) + 1, 0, 0,
2468                      PROTOCOL_BINARY_RAW_BYTES);
2469    check(rv == ENGINE_E2BIG, "Object too big");
2470
2471    return SUCCESS;
2472}
2473
2474static enum test_result test_whitespace_db(ENGINE_HANDLE *h,
2475                                           ENGINE_HANDLE_V1 *h1) {
2476    vals.clear();
2477    check(h1->get_stats(h, NULL, NULL, 0, add_stats) == ENGINE_SUCCESS,
2478          "Failed to get stats.");
2479    if (vals["ep_dbname"] != std::string(WHITESPACE_DB)) {
2480        std::cerr << "Expected dbname = ``" << WHITESPACE_DB << "''"
2481                  << ", got ``" << vals["ep_dbname"] << "''" << std::endl;
2482        return FAIL;
2483    }
2484
2485    check(access(WHITESPACE_DB, F_OK) != -1, "I expected the whitespace db to exist");
2486    return SUCCESS;
2487}
2488
2489static enum test_result test_memory_limit(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2490    set_param(h, h1, protocol_binary_engine_param_flush, "mutation_mem_threshold", "95");
2491    int used = get_int_stat(h, h1, "mem_used");
2492    double mem_threshold =
2493        static_cast<double>(get_int_stat(h, h1, "ep_mutation_mem_threshold")) / 100;
2494    int max = static_cast<int>(get_int_stat(h, h1, "ep_max_size") * mem_threshold);
2495    check(get_int_stat(h, h1, "ep_oom_errors") == 0 &&
2496          get_int_stat(h, h1, "ep_tmp_oom_errors") == 0, "Expected no OOM errors.");
2497    cb_assert(used < max);
2498
2499    char data[8192];
2500    memset(data, 'x', sizeof(data));
2501    size_t vlen = max - used - 192;
2502    data[vlen] = 0x00;
2503
2504    item *i = NULL;
2505    // So if we add an item,
2506    check(store(h, h1, NULL, OPERATION_SET, "key", data, &i) == ENGINE_SUCCESS,
2507          "store failure");
2508    check_key_value(h, h1, "key", data, vlen);
2509    h1->release(h, NULL, i);
2510
2511    // There should be no room for another.
2512    ENGINE_ERROR_CODE second = store(h, h1, NULL, OPERATION_SET, "key2", data, &i);
2513    check(second == ENGINE_ENOMEM || second == ENGINE_TMPFAIL,
2514          "should have failed second set");
2515    h1->release(h, NULL, i);
2516    check(get_int_stat(h, h1, "ep_oom_errors") == 1 ||
2517          get_int_stat(h, h1, "ep_tmp_oom_errors") == 1, "Expected an OOM error.");
2518
2519    ENGINE_ERROR_CODE overwrite = store(h, h1, NULL, OPERATION_SET, "key", data, &i);
2520    check(overwrite == ENGINE_ENOMEM || overwrite == ENGINE_TMPFAIL,
2521          "should have failed second override");
2522    h1->release(h, NULL, i);
2523    check(get_int_stat(h, h1, "ep_oom_errors") == 2 ||
2524          get_int_stat(h, h1, "ep_tmp_oom_errors") == 2, "Expected another OOM error.");
2525    check_key_value(h, h1, "key", data, vlen);
2526    check(ENGINE_SUCCESS != verify_key(h, h1, "key2"), "Expected a failure in GET");
2527    int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
2528    // Until we remove that item
2529    check(del(h, h1, "key", 0, 0) == ENGINE_SUCCESS, "Failed remove with value.");
2530    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
2531    testHarness.time_travel(65);
2532    wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
2533
2534    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue2", &i) == ENGINE_SUCCESS,
2535          "should have succeded on the last set");
2536    check_key_value(h, h1, "key2", "somevalue2", 10);
2537    h1->release(h, NULL, i);
2538    return SUCCESS;
2539}
2540
2541static enum test_result test_vbucket_get_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2542    return verify_vbucket_missing(h, h1, 1) ? SUCCESS : FAIL;
2543}
2544
2545static enum test_result test_vbucket_get(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2546    return verify_vbucket_state(h, h1, 0, vbucket_state_active) ? SUCCESS : FAIL;
2547}
2548
2549static enum test_result test_vbucket_create(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2550    if (!verify_vbucket_missing(h, h1, 1)) {
2551        fprintf(stderr, "vbucket wasn't missing.\n");
2552        return FAIL;
2553    }
2554
2555    if (!set_vbucket_state(h, h1, 1, vbucket_state_active)) {
2556        fprintf(stderr, "set state failed.\n");
2557        return FAIL;
2558    }
2559
2560    return verify_vbucket_state(h, h1, 1, vbucket_state_active) ? SUCCESS : FAIL;
2561}
2562
2563static enum test_result test_vbucket_compact(ENGINE_HANDLE *h,
2564                                             ENGINE_HANDLE_V1 *h1) {
2565    const char *key = "Carss";
2566    const char *value = "pollute";
2567    if (!verify_vbucket_missing(h, h1, 0)) {
2568        fprintf(stderr, "vbucket wasn't missing.\n");
2569        return FAIL;
2570    }
2571
2572    if (!set_vbucket_state(h, h1, 0, vbucket_state_active)) {
2573        fprintf(stderr, "set state failed.\n");
2574        return FAIL;
2575    }
2576
2577    check(verify_vbucket_state(h, h1, 0, vbucket_state_active),
2578            "VBucket state not active");
2579
2580    // Set two keys - one to be expired and other to remain...
2581    item *itm = NULL;
2582    check(store(h, h1, NULL, OPERATION_SET, key, value, &itm)
2583          == ENGINE_SUCCESS, "Failed set.");
2584    h1->release(h, NULL, itm);
2585
2586    check_key_value(h, h1, key, value, strlen(value));
2587
2588    // Set a non-expiring key...
2589    check(store(h, h1, NULL, OPERATION_SET, "trees", "cleanse", &itm)
2590          == ENGINE_SUCCESS, "Failed set.");
2591    h1->release(h, NULL, itm);
2592
2593    check_key_value(h, h1, "trees", "cleanse", strlen("cleanse"));
2594
2595    touch(h, h1, key, 0, 11);
2596    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch Carss");
2597
2598    testHarness.time_travel(12);
2599    wait_for_flusher_to_settle(h, h1);
2600
2601    check(get_int_stat(h, h1, "vb_0:purge_seqno", "vbucket-seqno") == 0,
2602            "purge_seqno not found to be zero before compaction");
2603
2604    // Compaction on VBucket
2605    compact_db(h, h1, 0, 2, 3, 1);
2606
2607    check(get_int_stat(h, h1, "ep_pending_compactions") == 0,
2608    "ep_pending_compactions stat did not tick down after compaction command");
2609
2610    // the key tree and its value should be intact...
2611    check(verify_key(h, h1, "trees") == ENGINE_SUCCESS,
2612          "key trees should be found.");
2613    // the key Carrs should have disappeared...
2614    int val = verify_key(h, h1, "Carss");
2615    check(val == ENGINE_KEY_ENOENT, "Key Carss has not expired.");
2616
2617    check(get_int_stat(h, h1, "vb_0:purge_seqno", "vbucket-seqno") == 3,
2618        "purge_seqno didn't match expected value");
2619
2620    return SUCCESS;
2621}
2622
2623struct comp_thread_ctx {
2624    ENGINE_HANDLE *h;
2625    ENGINE_HANDLE_V1 *h1;
2626    uint16_t vbid;
2627};
2628
2629extern "C" {
2630    static void compaction_thread(void *arg) {
2631        struct comp_thread_ctx *ctx = static_cast<comp_thread_ctx *>(arg);
2632        compact_db(ctx->h, ctx->h1, ctx->vbid, 0, 0, 0);
2633    }
2634}
2635
2636static enum test_result test_multiple_vb_compactions(ENGINE_HANDLE *h,
2637                                                     ENGINE_HANDLE_V1 *h1) {
2638    for (uint16_t i = 0; i < 4; ++i) {
2639        if (!set_vbucket_state(h, h1, i, vbucket_state_active)) {
2640            fprintf(stderr, "set state failed for vbucket %d.\n", i);
2641            return FAIL;
2642        }
2643        check(verify_vbucket_state(h, h1, i, vbucket_state_active),
2644              "VBucket state not active");
2645    }
2646
2647    std::vector<std::string> keys;
2648    for (int j = 0; j < 20000; ++j) {
2649        std::stringstream ss;
2650        ss << "key" << j;
2651        std::string key(ss.str());
2652        keys.push_back(key);
2653    }
2654
2655    int count = 0;
2656    std::vector<std::string>::iterator it;
2657    for (it = keys.begin(); it != keys.end(); ++it) {
2658        uint16_t vbid = count % 4;
2659        item *i;
2660        check(store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str(), &i, 0, vbid)
2661              == ENGINE_SUCCESS, "Failed to store a value");
2662        h1->release(h, NULL, i);
2663        ++count;
2664    }
2665
2666    // Compact multiple vbuckets.
2667    const int n_threads = 4;
2668    cb_thread_t threads[n_threads];
2669    struct comp_thread_ctx ctx[n_threads];
2670
2671    for (int i = 0; i < n_threads; i++) {
2672        ctx[i].h = h;
2673        ctx[i].h1 = h1;
2674        ctx[i].vbid = static_cast<uint16_t>(i);
2675        int r = cb_create_thread(&threads[i], compaction_thread, &ctx[i], 0);
2676        cb_assert(r == 0);
2677    }
2678
2679    for (int i = 0; i < n_threads; i++) {
2680        int r = cb_join_thread(threads[i]);
2681        cb_assert(r == 0);
2682    }
2683
2684    check(get_int_stat(h, h1, "ep_pending_compactions") == 0,
2685    "ep_pending_compactions stat did not tick down after compaction command");
2686
2687    return SUCCESS;
2688}
2689
2690static enum test_result
2691test_multi_vb_compactions_with_workload(ENGINE_HANDLE *h,
2692                                        ENGINE_HANDLE_V1 *h1) {
2693    for (uint16_t i = 0; i < 4; ++i) {
2694        if (!set_vbucket_state(h, h1, i, vbucket_state_active)) {
2695            fprintf(stderr, "set state failed for vbucket %d.\n", i);
2696            return FAIL;
2697        }
2698        check(verify_vbucket_state(h, h1, i, vbucket_state_active),
2699              "VBucket state not active");
2700    }
2701
2702    std::vector<std::string> keys;
2703    for (int j = 0; j < 10000; ++j) {
2704        std::stringstream ss;
2705        ss << "key" << j;
2706        std::string key(ss.str());
2707        keys.push_back(key);
2708    }
2709
2710    int count = 0;
2711    std::vector<std::string>::iterator it;
2712    for (it = keys.begin(); it != keys.end(); ++it) {
2713        uint16_t vbid = count % 4;
2714        item *i;
2715        check(store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str(), &i, 0, vbid)
2716              == ENGINE_SUCCESS, "Failed to store a value");
2717        h1->release(h, NULL, i);
2718        ++count;
2719    }
2720    wait_for_flusher_to_settle(h, h1);
2721
2722    for (int i = 0; i < 2; ++i) {
2723        count = 0;
2724        for (it = keys.begin(); it != keys.end(); ++it) {
2725            uint16_t vbid = count % 4;
2726            item *i = NULL;
2727            check(h1->get(h, NULL, &i, it->c_str(), strlen(it->c_str()), vbid) ==
2728                  ENGINE_SUCCESS, "Unable to get stored item");
2729            h1->release(h, NULL, i);
2730            ++count;
2731        }
2732    }
2733    wait_for_str_stat_to_be(h, h1, "ep_workload_pattern", "read_heavy", NULL);
2734
2735    // Compact multiple vbuckets.
2736    const int n_threads = 4;
2737    cb_thread_t threads[n_threads];
2738    struct comp_thread_ctx ctx[n_threads];
2739
2740    for (int i = 0; i < n_threads; i++) {
2741        ctx[i].h = h;
2742        ctx[i].h1 = h1;
2743        ctx[i].vbid = static_cast<uint16_t>(i);
2744        int r = cb_create_thread(&threads[i], compaction_thread, &ctx[i], 0);
2745        cb_assert(r == 0);
2746    }
2747
2748    for (int i = 0; i < n_threads; i++) {
2749        int r = cb_join_thread(threads[i]);
2750        cb_assert(r == 0);
2751    }
2752
2753    check(get_int_stat(h, h1, "ep_pending_compactions") == 0,
2754    "ep_pending_compactions stat did not tick down after compaction command");
2755
2756    return SUCCESS;
2757}
2758
2759static enum test_result vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
2760                                             const char* value = NULL) {
2761    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed to set vbucket state.");
2762
2763    vbucketDelete(h, h1, 2, value);
2764    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2765          "Expected failure deleting non-existent bucket.");
2766
2767    check(set_vbucket_state(h, h1, 1, vbucket_state_dead), "Failed set set vbucket 1 state.");
2768
2769    vbucketDelete(h, h1, 1, value);
2770    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
2771          "Expected failure deleting non-existent bucket.");
2772
2773    check(verify_vbucket_missing(h, h1, 1),
2774          "vbucket 0 was not missing after deleting it.");
2775
2776    return SUCCESS;
2777}
2778
2779static enum test_result test_vbucket_destroy_stats(ENGINE_HANDLE *h,
2780                                                   ENGINE_HANDLE_V1 *h1) {
2781
2782    int mem_used = get_int_stat(h, h1, "mem_used");
2783    int cacheSize = get_int_stat(h, h1, "ep_total_cache_size");
2784    int overhead = get_int_stat(h, h1, "ep_overhead");
2785    int nonResident = get_int_stat(h, h1, "ep_num_non_resident");
2786
2787    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed to set vbucket state.");
2788
2789    std::vector<std::string> keys;
2790    for (int j = 0; j < 2000; ++j) {
2791        std::stringstream ss;
2792        ss << "key" << j;
2793        std::string key(ss.str());
2794        keys.push_back(key);
2795    }
2796
2797    int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
2798    std::vector<std::string>::iterator it;
2799    for (it = keys.begin(); it != keys.end(); ++it) {
2800        item *i;
2801        check(store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str(), &i, 0, 1)
2802              == ENGINE_SUCCESS, "Failed to store a value");
2803        h1->release(h, NULL, i);
2804    }
2805    wait_for_flusher_to_settle(h, h1);
2806    testHarness.time_travel(65);
2807    wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
2808
2809    check(set_vbucket_state(h, h1, 1, vbucket_state_dead), "Failed set set vbucket 1 state.");
2810
2811    int vbucketDel = get_int_stat(h, h1, "ep_vbucket_del");
2812    vbucketDelete(h, h1, 1);
2813    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
2814          "Expected failure deleting non-existent bucket.");
2815
2816    check(verify_vbucket_missing(h, h1, 1),
2817          "vbucket 1 was not missing after deleting it.");
2818
2819    wait_for_stat_change(h, h1, "ep_vbucket_del", vbucketDel);
2820
2821    wait_for_stat_to_be(h, h1, "mem_used", mem_used);
2822    wait_for_stat_to_be(h, h1, "ep_total_cache_size", cacheSize);
2823    wait_for_stat_to_be(h, h1, "ep_overhead", overhead);
2824    wait_for_stat_to_be(h, h1, "ep_num_non_resident", nonResident);
2825
2826    return SUCCESS;
2827}
2828
2829static enum test_result vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
2830                                                const char* value = NULL) {
2831    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed to set vbucket state.");
2832
2833    // Store a value so the restart will try to resurrect it.
2834    item *i = NULL;
2835    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i, 0, 1)
2836          == ENGINE_SUCCESS, "Failed to set a value");
2837    check_key_value(h, h1, "key", "somevalue", 9, 1);
2838    h1->release(h, NULL, i);
2839
2840    // Reload to get a flush forced.
2841    testHarness.reload_engine(&h, &h1,
2842                              testHarness.engine_path,
2843                              testHarness.get_current_testcase()->cfg,
2844                              true, false);
2845    wait_for_warmup_complete(h, h1);
2846
2847    check(verify_vbucket_state(h, h1, 1, vbucket_state_active),
2848          "Bucket state was what it was initially, after restart.");
2849    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed to set vbucket state.");
2850    check_key_value(h, h1, "key", "somevalue", 9, 1);
2851
2852    check(set_vbucket_state(h, h1, 1, vbucket_state_dead), "Failed set set vbucket 1 state.");
2853
2854    vbucketDelete(h, h1, 1, value);
2855    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
2856          "Expected failure deleting non-existent bucket.");
2857
2858    check(verify_vbucket_missing(h, h1, 1),
2859          "vbucket 1 was not missing after deleting it.");
2860
2861    testHarness.reload_engine(&h, &h1,
2862                              testHarness.engine_path,
2863                              testHarness.get_current_testcase()->cfg,
2864                              true, false);
2865    wait_for_warmup_complete(h, h1);
2866
2867    if (verify_vbucket_state(h, h1, 1, vbucket_state_pending, true)) {
2868        std::cerr << "Bucket came up in pending state after delete." << std::endl;
2869        abort();
2870    }
2871
2872    check(verify_vbucket_missing(h, h1, 1),
2873          "vbucket 1 was not missing after restart.");
2874
2875    return SUCCESS;
2876}
2877
2878static enum test_result test_async_vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2879    return vbucket_destroy(h, h1);
2880}
2881
2882static enum test_result test_sync_vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2883    return vbucket_destroy(h, h1, "async=0");
2884}
2885
2886static enum test_result test_async_vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2887    return vbucket_destroy_restart(h, h1);
2888}
2889
2890static enum test_result test_sync_vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2891    return vbucket_destroy_restart(h, h1, "async=0");
2892}
2893
2894static enum test_result test_vb_set_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2895    return test_pending_vb_mutation(h, h1, OPERATION_SET);
2896}
2897
2898static enum test_result test_vb_add_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2899    return test_pending_vb_mutation(h, h1, OPERATION_ADD);
2900}
2901
2902static enum test_result test_vb_cas_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2903    return test_pending_vb_mutation(h, h1, OPERATION_CAS);
2904}
2905
2906static enum test_result test_vb_append_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2907    return test_pending_vb_mutation(h, h1, OPERATION_APPEND);
2908}
2909
2910static enum test_result test_vb_prepend_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2911    return test_pending_vb_mutation(h, h1, OPERATION_PREPEND);
2912}
2913
2914static enum test_result test_vb_set_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2915    return test_replica_vb_mutation(h, h1, OPERATION_SET);
2916}
2917
2918static enum test_result test_vb_replace_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2919    return test_replica_vb_mutation(h, h1, OPERATION_REPLACE);
2920}
2921
2922static enum test_result test_vb_replace_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2923    return test_pending_vb_mutation(h, h1, OPERATION_REPLACE);
2924}
2925
2926static enum test_result test_vb_add_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2927    return test_replica_vb_mutation(h, h1, OPERATION_ADD);
2928}
2929
2930static enum test_result test_vb_cas_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2931    return test_replica_vb_mutation(h, h1, OPERATION_CAS);
2932}
2933
2934static enum test_result test_vb_append_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2935    return test_replica_vb_mutation(h, h1, OPERATION_APPEND);
2936}
2937
2938static enum test_result test_vb_prepend_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2939    return test_replica_vb_mutation(h, h1, OPERATION_PREPEND);
2940}
2941
2942static enum test_result test_stats_seqno(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2943    check(set_vbucket_state(h, h1, 1, vbucket_state_active),
2944          "Failed to set vbucket state.");
2945
2946    int num_keys = 100;
2947    for (int ii = 0; ii < num_keys; ++ii) {
2948        std::stringstream ss;
2949        ss << "key" << ii;
2950        check(store(h, h1, NULL, OPERATION_SET, ss.str().c_str(),
2951                    "value", NULL, 0, 0) == ENGINE_SUCCESS,
2952              "Failed to store an item.");
2953    }
2954
2955    check(get_int_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno") == 100,
2956          "Invalid seqno");
2957    check(get_int_stat(h, h1, "vb_1:high_seqno", "vbucket-seqno") == 0,
2958          "Invalid seqno");
2959    check(get_int_stat(h, h1, "vb_1:high_seqno", "vbucket-seqno 1") == 0,
2960          "Invalid seqno");
2961
2962    uint64_t vb_uuid = get_ull_stat(h, h1, "vb_1:0:id", "failovers");
2963    check(get_ull_stat(h, h1, "vb_1:uuid", "vbucket-seqno 1") == vb_uuid,
2964          "Invalid uuid");
2965    check(vals.size() == 4, "Expected four stats");
2966
2967    // Check invalid vbucket
2968    check(h1->get_stats(h, NULL, "vbucket-seqno 2", 15, add_stats)
2969          == ENGINE_NOT_MY_VBUCKET, "Expected not my vbucket");
2970
2971    // Check bad vbucket parameter (not numeric)
2972    check(h1->get_stats(h, NULL, "vbucket-seqno tt2", 17, add_stats)
2973          == ENGINE_EINVAL, "Expected invalid");
2974
2975    // Check extra spaces at the end
2976    check(h1->get_stats(h, NULL, "vbucket-seqno    ", 17, add_stats)
2977          == ENGINE_EINVAL, "Expected invalid");
2978
2979    return SUCCESS;
2980}
2981
2982static enum test_result test_stats_diskinfo(ENGINE_HANDLE *h,
2983                                            ENGINE_HANDLE_V1 *h1) {
2984    check(set_vbucket_state(h, h1, 1, vbucket_state_active),
2985          "Failed to set vbucket state.");
2986
2987    int num_keys = 100;
2988    for (int ii = 0; ii < num_keys; ++ii) {
2989        std::stringstream ss;
2990        ss << "