xref: /3.0.3-GA/ep-engine/tests/ep_testsuite.cc (revision 5174ad24)
1/* -*- MODE: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/*
3 *     Copyright 2010 Couchbase, Inc
4 *
5 *   Licensed under the Apache License, Version 2.0 (the "License");
6 *   you may not use this file except in compliance with the License.
7 *   You may obtain a copy of the License at
8 *
9 *       http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *   Unless required by applicable law or agreed to in writing, software
12 *   distributed under the License is distributed on an "AS IS" BASIS,
13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *   See the License for the specific language governing permissions and
15 *   limitations under the License.
16 */
17
18// Usage: (to repeatedly run just a single test case)
19// make engine_tests IS_LOOP=-L EP_TEST_NUM=3
20
21#include "config.h"
22
23#include <stdio.h>
24#include <stdlib.h>
25#include <string.h>
26#include <sys/stat.h>
27#ifdef _MSC_VER
28#include <direct.h>
29#define mkdir(a, b) _mkdir(a)
30#else
31#include <sys/wait.h>
32#endif
33
34#include <cstdlib>
35#include <iostream>
36#include <map>
37#include <set>
38#include <sstream>
39#include <string>
40#include <vector>
41
42#include <platform/dirutils.h>
43
44#include "atomic.h"
45#include "ep-engine/command_ids.h"
46#include "ep_test_apis.h"
47#include "ep_testsuite.h"
48#include "locks.h"
49#include "mock/mccouch.h"
50#include "mock/mock_upr.h"
51#include "mutex.h"
52
53#include <snappy-c.h>
54
55#ifdef linux
56/* /usr/include/netinet/in.h defines macros from ntohs() to _bswap_nn to
57 * optimize the conversion functions, but the prototypes generate warnings
58 * from gcc. The conversion methods isn't the bottleneck for my app, so
59 * just remove the warnings by undef'ing the optimization ..
60 */
61#undef ntohs
62#undef ntohl
63#undef htons
64#undef htonl
65#endif
66
67// ptr_fun don't like the extern "C" thing for unlock cookie.. cast it
68// away ;)
69typedef void (*UNLOCK_COOKIE_T)(const void *cookie);
70
71extern "C" bool abort_msg(const char *expr, const char *msg, int line);
72
73
74template <typename T>
75static void checkeqfn(T exp, T got, const char *msg, const char *file, const int linenum) {
76    if (exp != got) {
77        std::stringstream ss;
78        ss << "Expected `" << exp << "', got `" << got << "' - " << msg;
79        abort_msg(ss.str().c_str(), file, linenum);
80    }
81}
82
83#define checkeq(a, b, c) checkeqfn(a, b, c, __FILE__, __LINE__)
84
85extern "C" {
86
87#define check(expr, msg) \
88    static_cast<void>((expr) ? 0 : abort_msg(#expr, msg, __LINE__))
89
90#define WHITESPACE_DB "whitespace sucks.db"
91#define MULTI_DISPATCHER_CONFIG \
92    "ht_size=129;ht_locks=3;chk_remover_stime=1;chk_period=60"
93
94struct test_harness testHarness;
95
96class ThreadData {
97public:
98    ThreadData(ENGINE_HANDLE *eh, ENGINE_HANDLE_V1 *ehv1,
99               int e=0) : h(eh), h1(ehv1), extra(e) {}
100    ENGINE_HANDLE    *h;
101    ENGINE_HANDLE_V1 *h1;
102    int               extra;
103};
104
105bool abort_msg(const char *expr, const char *msg, int line) {
106    fprintf(stderr, "%s:%d Test failed: `%s' (%s)\n",
107            __FILE__, line, msg, expr);
108    abort();
109    // UNREACHABLE
110    return false;
111}
112
113static enum test_result rmdb(void)
114{
115    const char *files[] = { WHITESPACE_DB,
116                            "/tmp/test",
117                            "/tmp/mutation.log",
118                            NULL };
119    int ii = 0;
120    while (files[ii] != NULL) {
121        CouchbaseDirectoryUtilities::rmrf(files[ii]);
122        if (access(files[ii], F_OK) != -1) {
123            std::cerr << "Failed to remove: " << files[ii] << " " << std::endl;
124            return FAIL;
125        }
126        ++ii;
127    }
128
129    return SUCCESS;
130}
131
132static enum test_result skipped_test_function(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
133    (void) h;
134    (void) h1;
135    return SKIPPED;
136}
137
138static bool teardown(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
139    (void)h; (void)h1;
140    vals.clear();
141    return true;
142}
143
144static const void* createTapConn(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
145                                 const char *name) {
146    const void *cookie = testHarness.create_cookie();
147    testHarness.lock_cookie(cookie);
148    TAP_ITERATOR iter = h1->get_tap_iterator(h, cookie, name,
149                                             strlen(name),
150                                             TAP_CONNECT_FLAG_DUMP, NULL,
151                                             0);
152    check(iter != NULL, "Failed to create a tap iterator");
153    return cookie;
154}
155
156static void check_key_value(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
157                            const char* key, const char* val, size_t vlen,
158                            uint16_t vbucket = 0) {
159    item_info info;
160    check(get_item_info(h, h1, &info, key, vbucket), "checking key and value");
161    check(info.nvalue == 1, "info.nvalue != 1");
162    check(vlen == info.value[0].iov_len, "Value length mismatch");
163    check(memcmp(info.value[0].iov_base, val, vlen) == 0, "Data mismatch");
164}
165
166static bool test_setup(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
167    wait_for_warmup_complete(h, h1);
168    wait_for_stat_change(h, h1, "ep_vb_snapshot_total", 0);
169
170    // warmup is complete, notify ep engine that it must now enable
171    // data traffic
172    protocol_binary_request_header *pkt = createPacket(PROTOCOL_BINARY_CMD_ENABLE_TRAFFIC);
173    check(h1->unknown_command(h, NULL, pkt, add_response) == ENGINE_SUCCESS,
174          "Failed to enable data traffic");
175    free(pkt);
176
177    return true;
178}
179
180static enum test_result test_getl(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
181    const char *key = "k1";
182    uint16_t vbucketId = 0;
183    uint32_t expiration = 25;
184
185    getl(h, h1, key, vbucketId, expiration);
186    check(last_status == PROTOCOL_BINARY_RESPONSE_KEY_ENOENT,
187          "expected the key to be missing...");
188    if (last_body != NULL && (strcmp(last_body, "NOT_FOUND") != 0)) {
189        fprintf(stderr, "Should have returned NOT_FOUND. Getl Failed");
190        abort();
191    }
192
193    item *i = NULL;
194    check(store(h, h1, NULL, OPERATION_SET, key, "lockdata", &i, 0, vbucketId)
195          == ENGINE_SUCCESS, "Failed to store an item.");
196    h1->release(h, NULL, i);
197
198    /* retry getl, should succeed */
199    getl(h, h1, key, vbucketId, expiration);
200    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
201          "Expected to be able to getl on first try");
202    check(strcmp("lockdata", last_body) == 0, "Body was malformed.");
203
204    /* wait 16 seconds */
205    testHarness.time_travel(16);
206
207    /* lock's taken so this should fail */
208    getl(h, h1, key, vbucketId, expiration);
209    check(last_status == PROTOCOL_BINARY_RESPONSE_ETMPFAIL,
210          "Expected to fail getl on second try");
211
212    if (last_body != NULL && (strcmp(last_body, "LOCK_ERROR") != 0)) {
213        fprintf(stderr, "Should have returned LOCK_ERROR. Getl Failed");
214        abort();
215    }
216
217    check(store(h, h1, NULL, OPERATION_SET, key, "lockdata2", &i, 0, vbucketId)
218          != ENGINE_SUCCESS, "Should have failed to store an item.");
219    h1->release(h, NULL, i);
220
221    /* wait another 10 seconds */
222    testHarness.time_travel(10);
223
224    /* retry set, should succeed */
225    check(store(h, h1, NULL, OPERATION_SET, key, "lockdata", &i, 0, vbucketId)
226          == ENGINE_SUCCESS, "Failed to store an item.");
227    h1->release(h, NULL, i);
228
229    /* acquire lock, should succeed */
230    getl(h, h1, key, vbucketId, expiration);
231    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
232          "Aquire lock should have succeeded");
233
234    /* try an incr operation followed by a delete, both of which should fail */
235    uint64_t cas = 0;
236    uint64_t result = 0;
237
238    check(h1->arithmetic(h, NULL, key, 2, true, false, 1, 1, 0,
239                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
240                         0)  == ENGINE_TMPFAIL, "Incr failed");
241
242
243    check(del(h, h1, key, 0, 0) == ENGINE_TMPFAIL, "Delete failed");
244
245
246    /* bug MB 2699 append after getl should fail with ENGINE_TMPFAIL */
247
248    testHarness.time_travel(26);
249
250    char binaryData1[] = "abcdefg\0gfedcba";
251    char binaryData2[] = "abzdefg\0gfedcba";
252
253    check(storeCasVb11(h, h1, NULL, OPERATION_SET, key,
254                       binaryData1, sizeof(binaryData1) - 1, 82758, &i, 0, 0)
255          == ENGINE_SUCCESS,
256          "Failed set.");
257    h1->release(h, NULL, i);
258
259    /* acquire lock, should succeed */
260    getl(h, h1, key, vbucketId, expiration);
261    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
262          "Aquire lock should have succeeded");
263
264    /* append should fail */
265    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, key,
266                       binaryData2, sizeof(binaryData2) - 1, 82758, &i, 0, 0)
267          == ENGINE_TMPFAIL,
268          "Append should fail.");
269    h1->release(h, NULL, i);
270
271    /* bug MB 3252 & MB 3354.
272     * 1. Set a key with an expiry value.
273     * 2. Take a lock on the item before it expires
274     * 3. Wait for the item to expire
275     * 4. Perform a CAS operation, should fail
276     * 5. Perform a set operation, should succeed
277     */
278    const char *ekey = "test_expiry";
279    const char *edata = "some test data here.";
280
281    item *it = NULL;
282
283    check(h1->allocate(h, NULL, &it, ekey, strlen(ekey), strlen(edata), 0, 2,
284          PROTOCOL_BINARY_RAW_BYTES) == ENGINE_SUCCESS, "Allocation Failed");
285
286    item_info info;
287    info.nvalue = 1;
288    if (!h1->get_item_info(h, NULL, it, &info)) {
289        abort();
290    }
291    memcpy(info.value[0].iov_base, edata, strlen(edata));
292
293    check(h1->store(h, NULL, it, &cas, OPERATION_SET, 0) ==
294        ENGINE_SUCCESS, "Failed to Store item");
295    check_key_value(h, h1, ekey, edata, strlen(edata));
296    h1->release(h, NULL, it);
297
298    testHarness.time_travel(3);
299    cas = last_cas;
300
301    /* cas should fail */
302    check(storeCasVb11(h, h1, NULL, OPERATION_CAS, ekey,
303                       binaryData1, sizeof(binaryData1) - 1, 82758, &i, cas, 0)
304          != ENGINE_SUCCESS,
305          "CAS succeeded.");
306    h1->release(h, NULL, i);
307
308    /* but a simple store should succeed */
309    check(store(h, h1, NULL, OPERATION_SET, ekey, edata, &i, 0, vbucketId)
310          == ENGINE_SUCCESS, "Failed to store an item.");
311    h1->release(h, NULL, i);
312    return SUCCESS;
313}
314
315static enum test_result test_unl(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
316
317    const char *key = "k2";
318    uint16_t vbucketId = 0;
319
320    unl(h, h1, key, vbucketId);
321    check(last_status != PROTOCOL_BINARY_RESPONSE_SUCCESS,
322          "expected the key to be missing...");
323
324    item *i = NULL;
325    check(store(h, h1, NULL, OPERATION_SET, key, "lockdata", &i, 0, vbucketId)
326          == ENGINE_SUCCESS, "Failed to store an item.");
327    h1->release(h, NULL, i);
328
329    /* getl, should succeed */
330    getl(h, h1, key, vbucketId, 0);
331    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
332          "Expected to be able to getl on first try");
333
334    /* save the returned cas value for later */
335    uint64_t cas = last_cas;
336
337    /* lock's taken unlocking with a random cas value should fail */
338    unl(h, h1, key, vbucketId);
339    check(last_status == PROTOCOL_BINARY_RESPONSE_ETMPFAIL,
340          "Expected to fail getl on second try");
341
342    if (last_body != NULL && (strcmp(last_body, "UNLOCK_ERROR") != 0)) {
343        fprintf(stderr, "Should have returned UNLOCK_ERROR. Unl Failed");
344        abort();
345    }
346
347    unl(h, h1, key, vbucketId, cas);
348    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
349          "Expected to succed unl with correct cas");
350
351    /* acquire lock, should succeed */
352    getl(h, h1, key, vbucketId, 0);
353
354    /* wait 16 seconds */
355    testHarness.time_travel(16);
356
357    /* lock has expired, unl should fail */
358    unl(h, h1, key, vbucketId, last_cas);
359    check(last_status == PROTOCOL_BINARY_RESPONSE_ETMPFAIL,
360          "Expected to fail unl on lock timeout");
361
362    return SUCCESS;
363}
364
365static enum test_result test_wrong_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
366                                               ENGINE_STORE_OPERATION op) {
367    item *i = NULL;
368    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
369    uint64_t cas = 11;
370    if (op == OPERATION_ADD) {
371        // Add operation with cas != 0 doesn't make sense
372        cas = 0;
373    }
374    check(store(h, h1, NULL, op,
375                "key", "somevalue", &i, cas, 1) == ENGINE_NOT_MY_VBUCKET,
376        "Expected not_my_vbucket");
377    h1->release(h, NULL, i);
378    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
379    return SUCCESS;
380}
381
382static enum test_result test_pending_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
383                                                 ENGINE_STORE_OPERATION op) {
384    const void *cookie = testHarness.create_cookie();
385    testHarness.set_ewouldblock_handling(cookie, false);
386    item *i = NULL;
387    check(set_vbucket_state(h, h1, 1, vbucket_state_pending), "Failed to set vbucket state.");
388    check(verify_vbucket_state(h, h1, 1, vbucket_state_pending), "Bucket state was not set to pending.");
389    uint64_t cas = 11;
390    if (op == OPERATION_ADD) {
391        // Add operation with cas != 0 doesn't make sense..
392        cas = 0;
393    }
394    check(store(h, h1, cookie, op,
395                "key", "somevalue", &i, cas, 1) == ENGINE_EWOULDBLOCK,
396        "Expected woodblock");
397    h1->release(h, NULL, i);
398    testHarness.destroy_cookie(cookie);
399    return SUCCESS;
400}
401
402static enum test_result test_replica_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
403                                                 ENGINE_STORE_OPERATION op) {
404    item *i = NULL;
405    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set vbucket state.");
406    check(verify_vbucket_state(h, h1, 1, vbucket_state_replica), "Bucket state was not set to replica.");
407    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
408
409    uint64_t cas = 11;
410    if (op == OPERATION_ADD) {
411        // performing add with a CAS != 0 doesn't make sense...
412        cas = 0;
413    }
414    check(store(h, h1, NULL, op,
415                "key", "somevalue", &i, cas, 1) == ENGINE_NOT_MY_VBUCKET,
416        "Expected not my vbucket");
417    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
418    h1->release(h, NULL, i);
419    return SUCCESS;
420}
421
422//
423// ----------------------------------------------------------------------
424// The actual tests are below.
425// ----------------------------------------------------------------------
426//
427
428static enum test_result test_get_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
429    check(verify_key(h, h1, "k") == ENGINE_KEY_ENOENT, "Expected miss.");
430    return SUCCESS;
431}
432
433static enum test_result test_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
434    item *i = NULL;
435    check(ENGINE_SUCCESS ==
436          store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
437          "Error setting.");
438    h1->release(h, NULL, i);
439    return SUCCESS;
440}
441
442struct handle_pair {
443    ENGINE_HANDLE *h;
444    ENGINE_HANDLE_V1 *h1;
445};
446
447extern "C" {
448    static void conc_del_set_thread(void *arg) {
449        struct handle_pair *hp = static_cast<handle_pair *>(arg);
450        item *it = NULL;
451
452        for (int i = 0; i < 5000; ++i) {
453            store(hp->h, hp->h1, NULL, OPERATION_ADD,
454                  "key", "somevalue", &it);
455            hp->h1->release(hp->h, NULL, it);
456            usleep(10);
457            checkeq(ENGINE_SUCCESS,
458                    store(hp->h, hp->h1, NULL, OPERATION_SET,
459                          "key", "somevalue", &it),
460                    "Error setting.");
461            hp->h1->release(hp->h, NULL, it);
462            usleep(10);
463            // Ignoring the result here -- we're racing.
464            del(hp->h, hp->h1, "key", 0, 0);
465            usleep(10);
466        }
467    }
468}
469
470static enum test_result test_conc_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
471
472    const int n_threads = 8;
473    cb_thread_t threads[n_threads];
474    struct handle_pair hp = {h, h1};
475
476    wait_for_persisted_value(h, h1, "key", "value1");
477
478    for (int i = 0; i < n_threads; i++) {
479        int r = cb_create_thread(&threads[i], conc_del_set_thread, &hp, 0);
480        cb_assert(r == 0);
481    }
482
483    for (int i = 0; i < n_threads; i++) {
484        int r = cb_join_thread(threads[i]);
485        cb_assert(r == 0);
486    }
487
488    wait_for_flusher_to_settle(h, h1);
489
490    testHarness.reload_engine(&h, &h1,
491                              testHarness.engine_path,
492                              testHarness.get_current_testcase()->cfg,
493                              true, false);
494    wait_for_warmup_complete(h, h1);
495
496    cb_assert(0 == get_int_stat(h, h1, "ep_warmed_dups"));
497
498    return SUCCESS;
499}
500
501static enum test_result test_set_get_hit(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
502    item *i = NULL;
503    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
504          "store failure");
505    check_key_value(h, h1, "key", "somevalue", 9);
506    h1->release(h, NULL, i);
507    return SUCCESS;
508}
509
510static enum test_result test_set_get_hit_bin(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
511    char binaryData[] = "abcdefg\0gfedcba";
512    cb_assert(sizeof(binaryData) != strlen(binaryData));
513
514    item *i = NULL;
515    check(ENGINE_SUCCESS ==
516          storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
517                       binaryData, sizeof(binaryData), 82758, &i, 0, 0),
518          "Failed to set.");
519    h1->release(h, NULL, i);
520    check_key_value(h, h1, "key", binaryData, sizeof(binaryData));
521    return SUCCESS;
522}
523
524static enum test_result test_set_with_cas_non_existent(ENGINE_HANDLE *h,
525                                                       ENGINE_HANDLE_V1 *h1) {
526    const char *key = "test_expiry_flush";
527    item *i = NULL;
528
529    check(h1->allocate(h, NULL, &i, key, strlen(key), 10, 0, 0,
530          PROTOCOL_BINARY_RAW_BYTES) == ENGINE_SUCCESS, "Allocation failed.");
531
532    Item *it = reinterpret_cast<Item*>(i);
533    it->setCas(1234);
534
535    uint64_t cas = 0;
536    check(h1->store(h, NULL, i, &cas, OPERATION_SET, 0) == ENGINE_KEY_ENOENT,
537          "Expected not found");
538    h1->release(h, NULL, i);
539
540    return SUCCESS;
541}
542
543static enum test_result test_set_change_flags(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
544    item *i = NULL;
545    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
546          "Failed to set.");
547    h1->release(h, NULL, i);
548
549    item_info info;
550    uint32_t flags = 828258;
551    check(get_item_info(h, h1, &info, "key"), "Failed to get value.");
552    cb_assert(info.flags != flags);
553
554    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
555                       "newvalue", strlen("newvalue"), flags, &i, 0, 0) == ENGINE_SUCCESS,
556          "Failed to set again.");
557    h1->release(h, NULL, i);
558
559    check(get_item_info(h, h1, &info, "key"), "Failed to get value.");
560
561    return info.flags == flags ? SUCCESS : FAIL;
562}
563
564static enum test_result test_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
565    item *i = NULL;
566    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
567          "Failed to do initial set.");
568    h1->release(h, NULL, i);
569    check(store(h, h1, NULL, OPERATION_CAS, "key", "failcas", &i) != ENGINE_SUCCESS,
570          "Failed to fail initial CAS.");
571    h1->release(h, NULL, i);
572    check_key_value(h, h1, "key", "somevalue", 9);
573
574    check(h1->get(h, NULL, &i, "key", 3, 0) == ENGINE_SUCCESS,
575          "Failed to get value.");
576
577    item_info info;
578    info.nvalue = 1;
579    check(h1->get_item_info(h, NULL, i, &info), "Failed to get item info.");
580    h1->release(h, NULL, i);
581
582    check(store(h, h1, NULL, OPERATION_CAS, "key", "winCas", &i,
583                info.cas) == ENGINE_SUCCESS,
584          "Failed to store CAS");
585    h1->release(h, NULL, i);
586    check_key_value(h, h1, "key", "winCas", 6);
587
588    uint64_t cval = 99999;
589    check(store(h, h1, NULL, OPERATION_CAS, "non-existing", "winCas", &i,
590                cval) == ENGINE_KEY_ENOENT,
591          "CAS for non-existing key returned the wrong error code");
592    h1->release(h, NULL, i);
593    return SUCCESS;
594}
595
596static enum test_result test_add(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
597    item *i = NULL;
598    check(store(h, h1, NULL, OPERATION_ADD,"key", "somevalue", &i) == ENGINE_SUCCESS,
599          "Failed to add value.");
600    h1->release(h, NULL, i);
601    check(store(h, h1, NULL, OPERATION_ADD,"key", "somevalue", &i) == ENGINE_NOT_STORED,
602          "Failed to fail to re-add value.");
603    h1->release(h, NULL, i);
604
605    // This aborts on failure.
606    check_key_value(h, h1, "key", "somevalue", 9);
607
608    // Expiration above was an hour, so let's go to The Future
609    testHarness.time_travel(3800);
610
611    check(store(h, h1, NULL, OPERATION_ADD,"key", "newvalue", &i) == ENGINE_SUCCESS,
612          "Failed to add value again.");
613    h1->release(h, NULL, i);
614    check_key_value(h, h1, "key", "newvalue", 8);
615    return SUCCESS;
616}
617
618static enum test_result test_add_add_with_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
619    item *i = NULL;
620    check(store(h, h1, NULL, OPERATION_ADD, "key",
621                "somevalue", &i) == ENGINE_SUCCESS,
622          "Failed set.");
623    check_key_value(h, h1, "key", "somevalue", 9);
624    item_info info;
625    info.nvalue = 1;
626    info.nvalue = 1;
627    check(h1->get_item_info(h, NULL, i, &info) == true,
628          "Should be able to get info");
629
630    item *i2 = NULL;
631    ENGINE_ERROR_CODE ret;
632    check((ret = store(h, h1, NULL, OPERATION_ADD, "key",
633                       "somevalue", &i2, info.cas)) == ENGINE_KEY_EEXISTS,
634          "Should not be able to add the key two times");
635
636    h1->release(h, NULL, i);
637    h1->release(h, NULL, i2);
638    return SUCCESS;
639}
640
641static enum test_result test_replace(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
642    item *i = NULL;
643    check(store(h, h1, NULL, OPERATION_REPLACE,"key", "somevalue", &i) != ENGINE_SUCCESS,
644          "Failed to fail to replace non-existing value.");
645    h1->release(h, NULL, i);
646    check(store(h, h1, NULL, OPERATION_SET,"key", "somevalue", &i) == ENGINE_SUCCESS,
647          "Failed to set value.");
648    h1->release(h, NULL, i);
649    check(store(h, h1, NULL, OPERATION_REPLACE,"key", "somevalue", &i) == ENGINE_SUCCESS,
650          "Failed to replace existing value.");
651    h1->release(h, NULL, i);
652    check_key_value(h, h1, "key", "somevalue", 9);
653    return SUCCESS;
654}
655
656static enum test_result test_incr_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
657    uint64_t cas = 0, result = 0;
658    h1->arithmetic(h, NULL, "key", 3, true, false, 1, 0, 0,
659                   &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
660                   0);
661    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected to not find key");
662    return SUCCESS;
663}
664
665static enum test_result test_incr_default(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
666    uint64_t cas = 0, result = 0;
667    check(h1->arithmetic(h, NULL, "key", 3, true, true, 1, 1, 0,
668                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
669                         0) == ENGINE_SUCCESS,
670          "Failed first arith");
671    check(result == 1, "Failed result verification.");
672
673    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
674                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
675                         0) == ENGINE_SUCCESS,
676          "Failed second arith.");
677    check(result == 2, "Failed second result verification.");
678
679    check(h1->arithmetic(h, NULL, "key", 3, true, true, 1, 1, 0,
680                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
681                         0) == ENGINE_SUCCESS,
682          "Failed third arith.");
683    check(result == 3, "Failed third result verification.");
684
685    check_key_value(h, h1, "key", "3", 1);
686    return SUCCESS;
687}
688
689static enum test_result test_append(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
690    item *i = NULL;
691
692    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
693                       "\r\n", 2, 82758, &i, 0, 0)
694          == ENGINE_SUCCESS,
695          "Failed set.");
696    h1->release(h, NULL, i);
697
698    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key",
699                       "foo\r\n", 5, 82758, &i, 0, 0)
700          == ENGINE_SUCCESS,
701          "Failed append.");
702    h1->release(h, NULL, i);
703
704    check_key_value(h, h1, "key", "\r\nfoo\r\n", 7);
705
706    char binaryData1[] = "abcdefg\0gfedcba\r\n";
707    char binaryData2[] = "abzdefg\0gfedcba\r\n";
708    size_t dataSize = 20*1024*1024;
709    char *bigBinaryData3 = new char[dataSize];
710    memset(bigBinaryData3, '\0', dataSize);
711
712    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
713                       binaryData1, sizeof(binaryData1) - 1, 82758, &i, 0, 0)
714          == ENGINE_SUCCESS,
715          "Failed set.");
716    h1->release(h, NULL, i);
717
718    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key",
719                       bigBinaryData3, dataSize, 82758, &i, 0, 0)
720          == ENGINE_E2BIG,
721          "Expected append failure.");
722    h1->release(h, NULL, i);
723    delete[] bigBinaryData3;
724
725    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key",
726                       binaryData2, sizeof(binaryData2) - 1, 82758, &i, 0, 0)
727          == ENGINE_SUCCESS,
728          "Failed append.");
729    h1->release(h, NULL, i);
730
731    std::string expected;
732    expected.append(binaryData1, sizeof(binaryData1) - 1);
733    expected.append(binaryData2, sizeof(binaryData2) - 1);
734
735    check_key_value(h, h1, "key", expected.data(), expected.length());
736    return SUCCESS;
737}
738
739static enum test_result test_prepend(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
740    item *i = NULL;
741
742    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
743                       "\r\n", 2, 82758, &i, 0, 0)
744          == ENGINE_SUCCESS,
745          "Failed set.");
746    h1->release(h, NULL, i);
747
748    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key",
749                       "foo\r\n", 5, 82758, &i, 0, 0)
750          == ENGINE_SUCCESS,
751          "Failed append.");
752    h1->release(h, NULL, i);
753
754    check_key_value(h, h1, "key", "foo\r\n\r\n", 7);
755
756    char binaryData1[] = "abcdefg\0gfedcba\r\n";
757    char binaryData2[] = "abzdefg\0gfedcba\r\n";
758    size_t dataSize = 20*1024*1024;
759    char *bigBinaryData3 = new char[dataSize];
760    memset(bigBinaryData3, '\0', dataSize);
761
762    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
763                       binaryData1, sizeof(binaryData1) - 1, 82758, &i, 0, 0)
764          == ENGINE_SUCCESS,
765          "Failed set.");
766    h1->release(h, NULL, i);
767
768    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key",
769                       bigBinaryData3, dataSize, 82758, &i, 0, 0)
770          == ENGINE_E2BIG,
771          "Expected prepend failure.");
772    h1->release(h, NULL, i);
773    delete[] bigBinaryData3;
774
775    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key",
776                       binaryData2, sizeof(binaryData2) - 1, 82758, &i, 0, 0)
777          == ENGINE_SUCCESS,
778          "Failed append.");
779    h1->release(h, NULL, i);
780
781    std::string expected;
782    expected.append(binaryData2, sizeof(binaryData2) - 1);
783    expected.append(binaryData1, sizeof(binaryData1) - 1);
784
785    check_key_value(h, h1, "key", expected.data(), expected.length());
786    return SUCCESS;
787}
788
789static enum test_result test_append_compressed(ENGINE_HANDLE *h,
790                                               ENGINE_HANDLE_V1 *h1) {
791
792    item *i = NULL;
793
794    size_t len = snappy_max_compressed_length(2);
795    char *newBuf = (char *) malloc (len);
796    snappy_compress("\r\n", 2, newBuf, &len);
797    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key1",
798                       (const char *)newBuf, len, 82758, &i, 0, 0, 3600,
799                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
800          == ENGINE_SUCCESS, "Failed set.");
801    h1->release(h, NULL, i);
802    free (newBuf);
803
804    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key1",
805                       "foo\r\n", 5, 82758, &i, 0, 0)
806          == ENGINE_SUCCESS,
807          "Failed append uncompressed to compressed.");
808    h1->release(h, NULL, i);
809
810    size_t len1 = snappy_max_compressed_length(7);
811    char *newBuf1 = (char *) malloc (len1);
812    snappy_compress("\r\nfoo\r\n", 7, newBuf1, &len1);
813
814    item_info info;
815    check(get_item_info(h, h1, &info, "key1", 0), "checking key and value");
816    check(info.nvalue == 1, "info.nvalue != 1");
817    check(len1 == info.value[0].iov_len, "Value length mismatch");
818    check(memcmp(info.value[0].iov_base, newBuf1, len1) == 0, "Data mismatch");
819    check(info.datatype == 0x02, "Datatype mismatch");
820    free (newBuf1);
821
822    len = snappy_max_compressed_length(3);
823    newBuf = (char *) malloc (len);
824    snappy_compress("bar", 3, newBuf, &len);
825    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key1",
826                       (const char*)newBuf, len, 82758, &i, 0, 0, 3600,
827                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
828            == ENGINE_SUCCESS,
829            "Failed append compressed to compressed.");
830    h1->release(h, NULL, i);
831    free (newBuf);
832
833    len1 = snappy_max_compressed_length(10);
834    newBuf1 = (char *) malloc (len1);
835    snappy_compress("\r\nfoo\r\nbar", 10, newBuf1, &len1);
836
837    check(get_item_info(h, h1, &info, "key1", 0), "checking key and value");
838    check(info.nvalue == 1, "info.nvalue != 1");
839    check(len1 == info.value[0].iov_len, "Value length mismatch");
840    check(memcmp(info.value[0].iov_base, newBuf1, len1) == 0, "Data mismatch");
841    check(info.datatype == 0x02, "Datatype mismatch");
842    free (newBuf1);
843
844    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key2",
845                       "foo", 3, 82758, &i, 0, 0, 3600,
846                       PROTOCOL_BINARY_RAW_BYTES)
847          == ENGINE_SUCCESS, "Failed set.");
848    h1->release(h, NULL, i);
849
850    len = snappy_max_compressed_length(3);
851    newBuf = (char *) malloc (len);
852    snappy_compress("bar", 3, newBuf, &len);
853    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key2",
854                       newBuf, len, 82758, &i, 0, 0, 3600,
855                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
856            == ENGINE_SUCCESS,
857            "Failed append compressed to uncompressed.");
858    h1->release(h, NULL, i);
859    free (newBuf);
860
861    check(get_item_info(h, h1, &info, "key2", 0), "checking key and value");
862    check(info.nvalue == 1, "info.nvalue != 1");
863    check(info.value[0].iov_len == 6, "Value length mismatch");
864    check(memcmp(info.value[0].iov_base, "foobar", 6) == 0, "Data mismatch");
865    check(info.datatype == 0x00, "Datatype mismatch");
866
867    return SUCCESS;
868}
869
870static enum test_result test_prepend_compressed(ENGINE_HANDLE *h,
871                                               ENGINE_HANDLE_V1 *h1) {
872
873    item *i = NULL;
874
875    size_t len = snappy_max_compressed_length(2);
876    char *newBuf = (char *) malloc (len);
877    snappy_compress("\r\n", 2, newBuf, &len);
878    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key1",
879                       (const char *)newBuf, len, 82758, &i, 0, 0, 3600,
880                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
881          == ENGINE_SUCCESS, "Failed set.");
882    h1->release(h, NULL, i);
883    free (newBuf);
884
885    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key1",
886                       "foo\r\n", 5, 82758, &i, 0, 0)
887          == ENGINE_SUCCESS,
888          "Failed prepend uncompressed to compressed.");
889    h1->release(h, NULL, i);
890
891    size_t len1 = snappy_max_compressed_length(7);
892    char *newBuf1 = (char *) malloc (len1);
893    snappy_compress("foo\r\n\r\n", 7, newBuf1, &len1);
894
895    item_info info;
896    check(get_item_info(h, h1, &info, "key1", 0), "checking key and value");
897    check(info.nvalue == 1, "info.nvalue != 1");
898    check(len1 == info.value[0].iov_len, "Value length mismatch");
899    check(memcmp(info.value[0].iov_base, newBuf1, len1) == 0, "Data mismatch");
900    check(info.datatype == 0x02, "Datatype mismatch");
901    free (newBuf1);
902
903    len = snappy_max_compressed_length(3);
904    newBuf = (char *) malloc (len);
905    snappy_compress("bar", 3, newBuf, &len);
906    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key1",
907                       (const char*)newBuf, len, 82758, &i, 0, 0, 3600,
908                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
909            == ENGINE_SUCCESS,
910            "Failed prepend compressed to compressed.");
911    h1->release(h, NULL, i);
912    free (newBuf);
913
914    len1 = snappy_max_compressed_length(10);
915    newBuf1 = (char *) malloc (len1);
916    snappy_compress("barfoo\r\n\r\n", 10, newBuf1, &len1);
917
918    check(get_item_info(h, h1, &info, "key1", 0), "checking key and value");
919    check(info.nvalue == 1, "info.nvalue != 1");
920    check(len1 == info.value[0].iov_len, "Value length mismatch");
921    check(memcmp(info.value[0].iov_base, newBuf1, len1) == 0, "Data mismatch");
922    check(info.datatype == 0x02, "Datatype mismatch");
923    free (newBuf1);
924
925    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key2",
926                       "foo", 3, 82758, &i, 0, 0, 3600,
927                       PROTOCOL_BINARY_RAW_BYTES)
928          == ENGINE_SUCCESS, "Failed set.");
929    h1->release(h, NULL, i);
930
931    len = snappy_max_compressed_length(3);
932    newBuf = (char *) malloc (len);
933    snappy_compress("bar", 3, newBuf, &len);
934    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key2",
935                       newBuf, len, 82758, &i, 0, 0, 3600,
936                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
937            == ENGINE_SUCCESS,
938            "Failed prepend compressed to uncompressed.");
939    h1->release(h, NULL, i);
940    free (newBuf);
941
942    check(get_item_info(h, h1, &info, "key2", 0), "checking key and value");
943    check(info.nvalue == 1, "info.nvalue != 1");
944    check(info.value[0].iov_len == 6, "Value length mismatch");
945    check(memcmp(info.value[0].iov_base, "barfoo", 6) == 0, "Data mismatch");
946    check(info.datatype == 0x00, "Datatype mismatch");
947
948    return SUCCESS;
949}
950
951static enum test_result test_incr(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
952    uint64_t cas = 0, result = 0;
953    item *i = NULL;
954    check(store(h, h1, NULL, OPERATION_ADD,"key", "1", &i) == ENGINE_SUCCESS,
955          "Failed to add value.");
956    h1->release(h, NULL, i);
957
958    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
959                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
960                         0) == ENGINE_SUCCESS,
961          "Failed to incr value.");
962
963    check_key_value(h, h1, "key", "2", 1);
964    return SUCCESS;
965}
966
967static enum test_result test_bug2799(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
968    uint64_t cas = 0, result = 0;
969    item *i = NULL;
970    check(store(h, h1, NULL, OPERATION_ADD, "key", "1", &i) == ENGINE_SUCCESS,
971          "Failed to add value.");
972    h1->release(h, NULL, i);
973
974    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
975                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
976                         0) == ENGINE_SUCCESS,
977          "Failed to incr value.");
978
979    check_key_value(h, h1, "key", "2", 1);
980
981    testHarness.time_travel(3617);
982
983    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
984    return SUCCESS;
985}
986
987static enum test_result test_flush(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
988    item *i = NULL;
989    // First try to delete something we know to not be there.
990    check(del(h, h1, "key", 0, 0) == ENGINE_KEY_ENOENT, "Failed to fail initial delete.");
991    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
992          "Failed set.");
993    h1->release(h, NULL, i);
994    check_key_value(h, h1, "key", "somevalue", 9);
995    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS,
996          "Failed to flush");
997    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
998
999    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1000          "Failed post-flush set.");
1001    h1->release(h, NULL, i);
1002    check_key_value(h, h1, "key", "somevalue", 9);
1003
1004    return SUCCESS;
1005}
1006
1007static enum test_result test_flush_disabled(ENGINE_HANDLE *h,
1008                                            ENGINE_HANDLE_V1 *h1) {
1009    item *i = NULL;
1010    // start an engine with disabled flush, the flush() should be noop and
1011    // we expect to see the key after flush()
1012
1013    // store a key and check its existence
1014    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1015          "Failed set.");
1016    h1->release(h, NULL, i);
1017    check_key_value(h, h1, "key", "somevalue", 9);
1018    // expect error msg engine does not support operation
1019    check(h1->flush(h, NULL, 0) == ENGINE_ENOTSUP, "Flush should be disabled");
1020    //check the key
1021    check(ENGINE_SUCCESS == verify_key(h, h1, "key"), "Expected key");
1022
1023    // restart engine with flush enabled and redo the test, we expect flush to succeed
1024    std::string param = "flushall_enabled=false";
1025    std::string config = testHarness.get_current_testcase()->cfg;
1026    size_t found = config.find(param);
1027    if(found != config.npos) {
1028        config.replace(found, param.size(), "flushall_enabled=true");
1029    }
1030    testHarness.reload_engine(&h, &h1,
1031                              testHarness.engine_path,
1032                              config.c_str(),
1033                              true, false);
1034    wait_for_warmup_complete(h, h1);
1035
1036    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS, "Flush should be enabled");
1037
1038    //expect missing key
1039    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1040
1041    return SUCCESS;
1042}
1043
1044static enum test_result test_flush_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1045    item *i = NULL;
1046    int mem_used = get_int_stat(h, h1, "mem_used");
1047    int overhead = get_int_stat(h, h1, "ep_overhead");
1048    int cacheSize = get_int_stat(h, h1, "ep_total_cache_size");
1049    int nonResident = get_int_stat(h, h1, "ep_num_non_resident");
1050
1051    int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
1052    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1053          "Failed set.");
1054    h1->release(h, NULL, i);
1055    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i) == ENGINE_SUCCESS,
1056          "Failed set.");
1057    h1->release(h, NULL, i);
1058    testHarness.time_travel(65);
1059    wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
1060
1061    check(ENGINE_SUCCESS == verify_key(h, h1, "key"), "Expected key");
1062    check(ENGINE_SUCCESS == verify_key(h, h1, "key2"), "Expected key2");
1063
1064    check_key_value(h, h1, "key", "somevalue", 9);
1065    check_key_value(h, h1, "key2", "somevalue", 9);
1066
1067    int mem_used2 = get_int_stat(h, h1, "mem_used");
1068    int overhead2 = get_int_stat(h, h1, "ep_overhead");
1069    int cacheSize2 = get_int_stat(h, h1, "ep_total_cache_size");
1070
1071    cb_assert(mem_used2 > mem_used);
1072    // "mem_used2 - overhead2" (i.e., ep_kv_size) should be greater than the hashtable cache size
1073    // due to the checkpoint overhead
1074    cb_assert(mem_used2 - overhead2 > cacheSize2);
1075
1076    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS, "Failed to flush");
1077    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1078    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key2"), "Expected missing key");
1079
1080    wait_for_flusher_to_settle(h, h1);
1081
1082    mem_used2 = get_int_stat(h, h1, "mem_used");
1083    overhead2 = get_int_stat(h, h1, "ep_overhead");
1084    cacheSize2 = get_int_stat(h, h1, "ep_total_cache_size");
1085    int nonResident2 = get_int_stat(h, h1, "ep_num_non_resident");
1086
1087    cb_assert(mem_used2 == mem_used);
1088    cb_assert(overhead2 == overhead);
1089    cb_assert(nonResident2 == nonResident);
1090    cb_assert(cacheSize2 == cacheSize);
1091
1092    return SUCCESS;
1093}
1094
1095static enum test_result test_flush_multiv(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1096    item *i = NULL;
1097    check(set_vbucket_state(h, h1, 2, vbucket_state_active), "Failed to set vbucket state.");
1098    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1099          "Failed set.");
1100    h1->release(h, NULL, i);
1101    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i,
1102                0, 2) == ENGINE_SUCCESS,
1103          "Failed set in vb2.");
1104    h1->release(h, NULL, i);
1105
1106    check(ENGINE_SUCCESS == verify_key(h, h1, "key"), "Expected key");
1107    check(ENGINE_SUCCESS == verify_key(h, h1, "key2", 2), "Expected key2");
1108
1109    check_key_value(h, h1, "key", "somevalue", 9);
1110    check_key_value(h, h1, "key2", "somevalue", 9, 2);
1111
1112    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS, "Failed to flush");
1113
1114    vals.clear();
1115    check(h1->get_stats(h, NULL, NULL, 0, add_stats) == ENGINE_SUCCESS,
1116          "Failed to get stats.");
1117    check(vals.find("ep_flush_all") != vals.end(), "Failed to get the status of flush_all");
1118
1119    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1120    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key2", 2), "Expected missing key");
1121
1122    return SUCCESS;
1123}
1124
1125static int checkCurrItemsAfterShutdown(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1126                                       int numItems2Load, bool shutdownForce) {
1127    std::vector<std::string> keys;
1128    for (int index = 0; index < numItems2Load; ++index) {
1129        std::stringstream s;
1130        s << "keys_2_load-" << index;
1131        std::string key(s.str());
1132        keys.push_back(key);
1133    }
1134
1135    check(get_int_stat(h, h1, "ep_total_persisted") == 0,
1136          "Expected ep_total_persisted equals 0");
1137    check(get_int_stat(h, h1, "curr_items") == 0,
1138          "Expected curr_items equals 0");
1139
1140    // stop flusher before loading new items
1141    protocol_binary_request_header *pkt = createPacket(PROTOCOL_BINARY_CMD_STOP_PERSISTENCE);
1142    check(h1->unknown_command(h, NULL, pkt, add_response) == ENGINE_SUCCESS,
1143          "CMD_STOP_PERSISTENCE failed!");
1144    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
1145          "Failed to stop persistence!");
1146    free(pkt);
1147
1148    std::vector<std::string>::iterator itr;
1149    for (itr = keys.begin(); itr != keys.end(); ++itr) {
1150        item *i;
1151        check(store(h, h1, NULL, OPERATION_SET, itr->c_str(), "oracle", &i, 0, 0)
1152              == ENGINE_SUCCESS, "Failed to store a value");
1153        h1->release(h, NULL, i);
1154    }
1155
1156    check(get_int_stat(h, h1, "ep_total_persisted") == 0,
1157          "Incorrect ep_total_persisted, expected 0");
1158    std::stringstream ss;
1159    ss << "Incorrect curr_items, expected " << numItems2Load;
1160    std::string errmsg(ss.str());
1161    check(get_int_stat(h, h1, "curr_items") == numItems2Load,
1162          errmsg.c_str());
1163
1164    // resume flusher before shutdown + warmup
1165    pkt = createPacket(PROTOCOL_BINARY_CMD_START_PERSISTENCE);
1166    check(h1->unknown_command(h, NULL, pkt, add_response) == ENGINE_SUCCESS,
1167          "CMD_START_PERSISTENCE failed!");
1168    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
1169          "Failed to start persistence!");
1170    free(pkt);
1171
1172    // shutdown engine force and restart
1173    testHarness.reload_engine(&h, &h1,
1174                              testHarness.engine_path,
1175                              testHarness.get_current_testcase()->cfg,
1176                              true, shutdownForce);
1177    wait_for_warmup_complete(h, h1);
1178    return get_int_stat(h, h1, "curr_items");
1179}
1180
1181static enum test_result test_flush_shutdown_force(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1182    int numItems2load = 3000;
1183    bool shutdownForce = true;
1184    int currItems = checkCurrItemsAfterShutdown(h, h1, numItems2load, shutdownForce);
1185    check (currItems <= numItems2load,
1186           "Number of curr items should be <= 3000, unless previous "
1187           "shutdown force had to wait for the flusher");
1188    return SUCCESS;
1189}
1190
1191static enum test_result test_flush_shutdown_noforce(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1192    int numItems2load = 3000;
1193    bool shutdownForce = false;
1194    int currItems = checkCurrItemsAfterShutdown(h, h1, numItems2load, shutdownForce);
1195    check (currItems == numItems2load,
1196           "Number of curr items should be equal to 3000, unless previous "
1197           "shutdown did not wait for the flusher");
1198    return SUCCESS;
1199}
1200
1201static enum test_result test_flush_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1202    item *i = NULL;
1203    // First try to delete something we know to not be there.
1204    check(del(h, h1, "key", 0, 0) == ENGINE_KEY_ENOENT, "Failed to fail initial delete.");
1205    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1206          "Failed set.");
1207    h1->release(h, NULL, i);
1208    check_key_value(h, h1, "key", "somevalue", 9);
1209
1210    // Restart once to ensure written to disk.
1211    testHarness.reload_engine(&h, &h1,
1212                              testHarness.engine_path,
1213                              testHarness.get_current_testcase()->cfg,
1214                              true, false);
1215    wait_for_warmup_complete(h, h1);
1216
1217    // Read value from disk.
1218    check_key_value(h, h1, "key", "somevalue", 9);
1219
1220    // Flush
1221    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS,
1222          "Failed to flush");
1223
1224    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i) == ENGINE_SUCCESS,
1225          "Failed post-flush set.");
1226    h1->release(h, NULL, i);
1227    check_key_value(h, h1, "key2", "somevalue", 9);
1228
1229    // Restart again, ensure written to disk.
1230    testHarness.reload_engine(&h, &h1,
1231                              testHarness.engine_path,
1232                              testHarness.get_current_testcase()->cfg,
1233                              true, false);
1234    wait_for_warmup_complete(h, h1);
1235
1236    check(store(h, h1, NULL, OPERATION_SET, "key3", "somevalue", &i) == ENGINE_SUCCESS,
1237          "Failed post-flush, post-restart set.");
1238    h1->release(h, NULL, i);
1239    check_key_value(h, h1, "key3", "somevalue", 9);
1240
1241    // Read value again, should not be there.
1242    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1243    return SUCCESS;
1244}
1245
1246static enum test_result test_flush_multiv_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1247    item *i = NULL;
1248    check(set_vbucket_state(h, h1, 2, vbucket_state_active), "Failed to set vbucket state.");
1249    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1250          "Failed set.");
1251    h1->release(h, NULL, i);
1252    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i,
1253                0, 2) == ENGINE_SUCCESS,
1254          "Failed set in vb2.");
1255    h1->release(h, NULL, i);
1256
1257    // Restart once to ensure written to disk.
1258    testHarness.reload_engine(&h, &h1,
1259                              testHarness.engine_path,
1260                              testHarness.get_current_testcase()->cfg,
1261                              true, false);
1262    wait_for_warmup_complete(h, h1);
1263
1264    // Read value from disk.
1265    check_key_value(h, h1, "key", "somevalue", 9);
1266
1267    // Flush
1268    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS,
1269          "Failed to flush");
1270
1271    // Restart again, ensure written to disk.
1272    testHarness.reload_engine(&h, &h1,
1273                              testHarness.engine_path,
1274                              testHarness.get_current_testcase()->cfg,
1275                              true, false);
1276    wait_for_warmup_complete(h, h1);
1277
1278    // Read value again, should not be there.
1279    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1280    check(verify_vbucket_missing(h, h1, 2), "Bucket 2 came back.");
1281    return SUCCESS;
1282}
1283
1284static enum test_result test_delete(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1285    item *i = NULL;
1286    // First try to delete something we know to not be there.
1287    check(del(h, h1, "key", 0, 0) == ENGINE_KEY_ENOENT, "Failed to fail initial delete.");
1288    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1289          "Failed set.");
1290    Item *it = reinterpret_cast<Item*>(i);
1291    uint64_t orig_cas = it->getCas();
1292    h1->release(h, NULL, i);
1293    check_key_value(h, h1, "key", "somevalue", 9);
1294
1295    uint64_t cas = 0;
1296    check(h1->remove(h, NULL, "key", 3, &cas, 0) == ENGINE_SUCCESS,
1297          "Failed remove with value.");
1298    check(orig_cas + 1 == cas, "Cas mismatch on delete");
1299    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1300
1301    // Can I time travel to an expired object and delete it?
1302    checkeq(ENGINE_SUCCESS, store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1303            "Failed set.");
1304    h1->release(h, NULL, i);
1305    testHarness.time_travel(3617);
1306    checkeq(ENGINE_KEY_ENOENT, del(h, h1, "key", 0, 0),
1307            "Did not get ENOENT removing an expired object.");
1308    checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"), "Expected missing key");
1309
1310    return SUCCESS;
1311}
1312
1313static enum test_result test_set_delete(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1314    item *i = NULL;
1315    checkeq(ENGINE_SUCCESS, store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1316            "Failed set.");
1317    h1->release(h, NULL, i);
1318    check_key_value(h, h1, "key", "somevalue", 9);
1319    checkeq(ENGINE_SUCCESS, del(h, h1, "key", 0, 0),
1320            "Failed remove with value.");
1321    checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"), "Expected missing key");
1322    wait_for_flusher_to_settle(h, h1);
1323    wait_for_stat_to_be(h, h1, "curr_items", 0);
1324    return SUCCESS;
1325}
1326
1327static enum test_result test_set_delete_invalid_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1328    item *i = NULL;
1329    check(store(h, h1, NULL, OPERATION_SET, "key",
1330                "somevalue", &i) == ENGINE_SUCCESS,
1331          "Failed set.");
1332    check_key_value(h, h1, "key", "somevalue", 9);
1333    item_info info;
1334    info.nvalue = 1;
1335    info.nvalue = 1;
1336    check(h1->get_item_info(h, NULL, i, &info) == true,
1337          "Should be able to get info");
1338    h1->release(h, NULL, i);
1339
1340    check(del(h, h1, "key", info.cas + 1, 0) == ENGINE_KEY_EEXISTS,
1341          "Didn't expect to be able to remove the item with wrong cas");
1342    return SUCCESS;
1343}
1344
1345static enum test_result test_bug2509(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1346    for (int j = 0; j < 10000; ++j) {
1347        item *itm = NULL;
1348        checkeq(ENGINE_SUCCESS,
1349                store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &itm),
1350                "Failed set.");
1351        h1->release(h, NULL, itm);
1352        usleep(10);
1353        checkeq(ENGINE_SUCCESS, del(h, h1, "key", 0, 0), "Failed remove with value.");
1354        usleep(10);
1355    }
1356
1357    // Restart again, to verify we don't have any duplicates.
1358    testHarness.reload_engine(&h, &h1,
1359                              testHarness.engine_path,
1360                              testHarness.get_current_testcase()->cfg,
1361                              true, false);
1362    wait_for_warmup_complete(h, h1);
1363
1364    return get_int_stat(h, h1, "ep_warmup_dups") == 0 ? SUCCESS : FAIL;
1365}
1366
1367static enum test_result test_bug7023(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1368    std::vector<std::string> keys;
1369    // Make a vbucket mess.
1370    for (int j = 0; j < 10000; ++j) {
1371        std::stringstream ss;
1372        ss << "key" << j;
1373        std::string key(ss.str());
1374        keys.push_back(key);
1375    }
1376
1377    std::vector<std::string>::iterator it;
1378    for (int j = 0; j < 5; ++j) {
1379        check(set_vbucket_state(h, h1, 0, vbucket_state_dead), "Failed set set vbucket 0 dead.");
1380        vbucketDelete(h, h1, 0);
1381        check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
1382              "Expected vbucket deletion to work.");
1383        check(set_vbucket_state(h, h1, 0, vbucket_state_active), "Failed set set vbucket 0 active.");
1384        for (it = keys.begin(); it != keys.end(); ++it) {
1385            item *i;
1386            check(store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str(), &i)
1387                  == ENGINE_SUCCESS, "Failed to store a value");
1388        }
1389    }
1390    wait_for_flusher_to_settle(h, h1);
1391
1392    // Restart again, to verify no data loss.
1393    testHarness.reload_engine(&h, &h1,
1394                              testHarness.engine_path,
1395                              testHarness.get_current_testcase()->cfg,
1396                              true, false);
1397    wait_for_warmup_complete(h, h1);
1398    return get_int_stat(h, h1, "ep_warmup_value_count", "warmup") == 10000 ? SUCCESS : FAIL;
1399}
1400
1401static enum test_result test_delete_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1402    wait_for_persisted_value(h, h1, "key", "value1");
1403
1404    check(del(h, h1, "key", 0, 0) == ENGINE_SUCCESS, "Failed remove with value.");
1405
1406    wait_for_persisted_value(h, h1, "key", "value2");
1407
1408    testHarness.reload_engine(&h, &h1,
1409                              testHarness.engine_path,
1410                              testHarness.get_current_testcase()->cfg,
1411                              true, false);
1412    wait_for_warmup_complete(h, h1);
1413
1414    check_key_value(h, h1, "key", "value2", 6);
1415    check(del(h, h1, "key", 0, 0) == ENGINE_SUCCESS, "Failed remove with value.");
1416    wait_for_flusher_to_settle(h, h1);
1417
1418    testHarness.reload_engine(&h, &h1,
1419                              testHarness.engine_path,
1420                              testHarness.get_current_testcase()->cfg,
1421                              true, false);
1422    wait_for_warmup_complete(h, h1);
1423
1424    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1425
1426    return SUCCESS;
1427}
1428
1429static enum test_result test_get_delete_missing_file(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1430    const char *key = "key";
1431    wait_for_persisted_value(h, h1, key, "value2delete");
1432
1433    // whack the db file and directory where the key is stored
1434    rmdb();
1435
1436    item *i = NULL;
1437    int errorCode = h1->get(h, NULL, &i, key, strlen(key), 0);
1438    h1->release(h, NULL, i);
1439
1440    // ep engine must be unaware of well-being of the db file as long as
1441    // the item is still in the memory
1442    check(errorCode == ENGINE_SUCCESS, "Expected success for get");
1443
1444    i = NULL;
1445    evict_key(h, h1, key);
1446    errorCode = h1->get(h, NULL, &i, key, strlen(key), 0);
1447    h1->release(h, NULL, i);
1448
1449    // ep engine must be now aware of the ill-fated db file where
1450    // the item is supposedly stored
1451    check(errorCode == ENGINE_TMPFAIL, "Expected tmp fail for get");
1452
1453    return SUCCESS;
1454}
1455
1456
1457static enum test_result test_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1458    item *i = NULL;
1459    static const char val[] = "somevalue";
1460    ENGINE_ERROR_CODE ret;
1461    check((ret = store(h, h1, NULL, OPERATION_SET, "key", val, &i)) == ENGINE_SUCCESS,
1462          "Failed set.");
1463    h1->release(h, NULL, i);
1464
1465    testHarness.reload_engine(&h, &h1,
1466                              testHarness.engine_path,
1467                              testHarness.get_current_testcase()->cfg,
1468                              true, false);
1469    wait_for_warmup_complete(h, h1);
1470    check_key_value(h, h1, "key", val, strlen(val));
1471    return SUCCESS;
1472}
1473
1474static enum test_result test_restart_session_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1475    createTapConn(h, h1, "tap_client_thread");
1476
1477    testHarness.reload_engine(&h, &h1,
1478                              testHarness.engine_path,
1479                              testHarness.get_current_testcase()->cfg,
1480                              true, false);
1481    wait_for_warmup_complete(h, h1);
1482    createTapConn(h, h1, "tap_client_thread");
1483
1484    check(h1->get_stats(h, NULL, "tap", 3, add_stats) == ENGINE_SUCCESS,
1485          "Failed to get stats.");
1486    std::string val = vals["eq_tapq:tap_client_thread:backfill_completed"];
1487    check(strcmp(val.c_str(), "true") == 0, "Don't expect the backfill upon restart");
1488    return SUCCESS;
1489}
1490
1491static enum test_result test_specialKeys(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1492    item *i = NULL;
1493    ENGINE_ERROR_CODE ret;
1494
1495    // Simplified Chinese "Couchbase"
1496    static const char key0[] = "沙发数据库";
1497    static const char val0[] = "some Chinese value";
1498    check((ret = store(h, h1, NULL, OPERATION_SET, key0, val0, &i)) == ENGINE_SUCCESS,
1499          "Failed set Chinese key");
1500    check_key_value(h, h1, key0, val0, strlen(val0));
1501    h1->release(h, NULL, i);
1502    // Traditional Chinese "Couchbase"
1503    static const char key1[] = "沙發數據庫";
1504    static const char val1[] = "some Traditional Chinese value";
1505    check((ret = store(h, h1, NULL, OPERATION_SET, key1, val1, &i)) == ENGINE_SUCCESS,
1506          "Failed set Traditional Chinese key");
1507    h1->release(h, NULL, i);
1508    // Korean "couch potato"
1509    static const char key2[] = "쇼파감자";
1510    static const char val2[] = "some Korean value";
1511    check((ret = store(h, h1, NULL, OPERATION_SET, key2, val2, &i)) == ENGINE_SUCCESS,
1512          "Failed set Korean key");
1513    h1->release(h, NULL, i);
1514    // Russian "couch potato"
1515    static const char key3[] = "лодырь, лентяй";
1516    static const char val3[] = "some Russian value";
1517    check((ret = store(h, h1, NULL, OPERATION_SET, key3, val3, &i)) == ENGINE_SUCCESS,
1518          "Failed set Russian key");
1519    h1->release(h, NULL, i);
1520    // Japanese "couch potato"
1521    static const char key4[] = "カウチポテト";
1522    static const char val4[] = "some Japanese value";
1523    check((ret = store(h, h1, NULL, OPERATION_SET, key4, val4, &i)) == ENGINE_SUCCESS,
1524          "Failed set Japanese key");
1525    h1->release(h, NULL, i);
1526    // Indian char key, and no idea what it is
1527    static const char key5[] = "हरियानवी";
1528    static const char val5[] = "some Indian value";
1529    check((ret = store(h, h1, NULL, OPERATION_SET, key5, val5, &i)) == ENGINE_SUCCESS,
1530          "Failed set Indian key");
1531    h1->release(h, NULL, i);
1532    // Portuguese translation "couch potato"
1533    static const char key6[] = "sedentário";
1534    static const char val6[] = "some Portuguese value";
1535    check((ret = store(h, h1, NULL, OPERATION_SET, key6, val6, &i)) == ENGINE_SUCCESS,
1536          "Failed set Portuguese key");
1537    h1->release(h, NULL, i);
1538    // Arabic translation "couch potato"
1539    static const char key7[] = "الحافلةالبطاطة";
1540    static const char val7[] = "some Arabic value";
1541    check((ret = store(h, h1, NULL, OPERATION_SET, key7, val7, &i)) == ENGINE_SUCCESS,
1542          "Failed set Arabic key");
1543    h1->release(h, NULL, i);
1544
1545    testHarness.reload_engine(&h, &h1,
1546                              testHarness.engine_path,
1547                              testHarness.get_current_testcase()->cfg,
1548                              true, false);
1549    wait_for_warmup_complete(h, h1);
1550    check_key_value(h, h1, key0, val0, strlen(val0));
1551    check_key_value(h, h1, key1, val1, strlen(val1));
1552    check_key_value(h, h1, key2, val2, strlen(val2));
1553    check_key_value(h, h1, key3, val3, strlen(val3));
1554    check_key_value(h, h1, key4, val4, strlen(val4));
1555    check_key_value(h, h1, key5, val5, strlen(val5));
1556    check_key_value(h, h1, key6, val6, strlen(val6));
1557    check_key_value(h, h1, key7, val7, strlen(val7));
1558    return SUCCESS;
1559}
1560
1561static enum test_result test_binKeys(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1562    item *i = NULL;
1563    ENGINE_ERROR_CODE ret;
1564
1565    // binary key with char values beyond 0x7F
1566    static const char key0[] = "\xe0\xed\xf1\x6f\x7f\xf8\xfa";
1567    static const char val0[] = "some value val8";
1568    check((ret = store(h, h1, NULL, OPERATION_SET, key0, val0, &i)) == ENGINE_SUCCESS,
1569          "Failed set binary key0");
1570    check_key_value(h, h1, key0, val0, strlen(val0));
1571    h1->release(h, NULL, i);
1572    // binary keys with char values beyond 0x7F
1573    static const char key1[] = "\xf1\xfd\xfe\xff\xf0\xf8\xef";
1574    static const char val1[] = "some value val9";
1575    check((ret = store(h, h1, NULL, OPERATION_SET, key1, val1, &i)) == ENGINE_SUCCESS,
1576          "Failed set binary key1");
1577    check_key_value(h, h1, key1, val1, strlen(val1));
1578    h1->release(h, NULL, i);
1579    // binary keys with special utf-8 BOM (Byte Order Mark) values 0xBB 0xBF 0xEF
1580    static const char key2[] = "\xff\xfe\xbb\xbf\xef";
1581    static const char val2[] = "some utf-8 bom value";
1582    check((ret = store(h, h1, NULL, OPERATION_SET, key2, val2, &i)) == ENGINE_SUCCESS,
1583          "Failed set binary utf-8 bom key");
1584    check_key_value(h, h1, key2, val2, strlen(val2));
1585    h1->release(h, NULL, i);
1586    // binary keys with special utf-16BE BOM values "U+FEFF"
1587    static const char key3[] = "U+\xfe\xff\xefU+\xff\xfe";
1588    static const char val3[] = "some utf-16 bom value";
1589    check((ret = store(h, h1, NULL, OPERATION_SET, key3, val3, &i)) == ENGINE_SUCCESS,
1590          "Failed set binary utf-16 bom key");
1591    check_key_value(h, h1, key3, val3, strlen(val3));
1592    h1->release(h, NULL, i);
1593
1594    testHarness.reload_engine(&h, &h1,
1595                              testHarness.engine_path,
1596                              testHarness.get_current_testcase()->cfg,
1597                              true, false);
1598    wait_for_warmup_complete(h, h1);
1599    check_key_value(h, h1, key0, val0, strlen(val0));
1600    check_key_value(h, h1, key1, val1, strlen(val1));
1601    check_key_value(h, h1, key2, val2, strlen(val2));
1602    check_key_value(h, h1, key3, val3, strlen(val3));
1603    return SUCCESS;
1604}
1605
1606static enum test_result test_restart_bin_val(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1607
1608
1609
1610    char binaryData[] = "abcdefg\0gfedcba";
1611    cb_assert(sizeof(binaryData) != strlen(binaryData));
1612
1613    item *i = NULL;
1614    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
1615                       binaryData, sizeof(binaryData), 82758, &i, 0, 0)
1616          == ENGINE_SUCCESS,
1617          "Failed set.");
1618    h1->release(h, NULL, i);
1619
1620    testHarness.reload_engine(&h, &h1,
1621                              testHarness.engine_path,
1622                              testHarness.get_current_testcase()->cfg,
1623                              true, false);
1624    wait_for_warmup_complete(h, h1);
1625
1626    check_key_value(h, h1, "key", binaryData, sizeof(binaryData));
1627    return SUCCESS;
1628}
1629
1630static enum test_result test_wrong_vb_get(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1631    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1632    check(ENGINE_NOT_MY_VBUCKET == verify_key(h, h1, "key", 1),
1633          "Expected wrong bucket.");
1634    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1635    return SUCCESS;
1636}
1637
1638static enum test_result test_vb_get_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1639    check(set_vbucket_state(h, h1, 1, vbucket_state_pending), "Failed to set vbucket state.");
1640    const void *cookie = testHarness.create_cookie();
1641    testHarness.set_ewouldblock_handling(cookie, false);
1642
1643    item *i = NULL;
1644    check(ENGINE_EWOULDBLOCK == h1->get(h, cookie, &i, "key", strlen("key"), 1),
1645          "Expected woodblock.");
1646    h1->release(h, NULL, i);
1647
1648    testHarness.destroy_cookie(cookie);
1649    return SUCCESS;
1650}
1651
1652static enum test_result test_vb_get_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1653    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set vbucket state.");
1654    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1655    check(ENGINE_NOT_MY_VBUCKET == verify_key(h, h1, "key", 1),
1656          "Expected not my bucket.");
1657    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1658    return SUCCESS;
1659}
1660
1661static enum test_result test_wrong_vb_incr(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1662    uint64_t cas, result;
1663    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1664    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
1665                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1666                         1) == ENGINE_NOT_MY_VBUCKET,
1667          "Expected not my vbucket.");
1668    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1669    return SUCCESS;
1670}
1671
1672static enum test_result test_vb_incr_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1673    const void *cookie = testHarness.create_cookie();
1674    testHarness.set_ewouldblock_handling(cookie, false);
1675    uint64_t cas, result;
1676    check(set_vbucket_state(h, h1, 1, vbucket_state_pending), "Failed to set vbucket state.");
1677    check(h1->arithmetic(h, cookie, "key", 3, true, false, 1, 1, 0,
1678                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1679                         1) == ENGINE_EWOULDBLOCK,
1680          "Expected woodblock.");
1681    testHarness.destroy_cookie(cookie);
1682    return SUCCESS;
1683}
1684
1685static enum test_result test_vb_incr_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1686    uint64_t cas, result;
1687    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set vbucket state.");
1688    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1689    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
1690                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1691                         1) == ENGINE_NOT_MY_VBUCKET,
1692          "Expected not my bucket.");
1693    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1694    return SUCCESS;
1695}
1696
1697static enum test_result test_wrong_vb_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1698    return test_wrong_vb_mutation(h, h1, OPERATION_SET);
1699}
1700
1701static enum test_result test_wrong_vb_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1702    return test_wrong_vb_mutation(h, h1, OPERATION_CAS);
1703}
1704
1705static enum test_result test_wrong_vb_add(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1706    return test_wrong_vb_mutation(h, h1, OPERATION_ADD);
1707}
1708
1709static enum test_result test_wrong_vb_replace(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1710    return test_wrong_vb_mutation(h, h1, OPERATION_REPLACE);
1711}
1712
1713static enum test_result test_wrong_vb_append(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1714    return test_wrong_vb_mutation(h, h1, OPERATION_APPEND);
1715}
1716
1717static enum test_result test_wrong_vb_prepend(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1718    return test_wrong_vb_mutation(h, h1, OPERATION_PREPEND);
1719}
1720
1721static enum test_result test_wrong_vb_del(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1722    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1723    check(ENGINE_NOT_MY_VBUCKET == del(h, h1, "key", 0, 1), "Expected wrong bucket.");
1724    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1725    return SUCCESS;
1726}
1727
1728static enum test_result test_expiry(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1729    const char *key = "test_expiry";
1730    const char *data = "some test data here.";
1731
1732    item *it = NULL;
1733
1734    ENGINE_ERROR_CODE rv;
1735    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 2,
1736                      PROTOCOL_BINARY_RAW_BYTES);
1737    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1738
1739    item_info info;
1740    info.nvalue = 1;
1741    if (!h1->get_item_info(h, NULL, it, &info)) {
1742        abort();
1743    }
1744    memcpy(info.value[0].iov_base, data, strlen(data));
1745
1746    uint64_t cas = 0;
1747    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
1748    check(rv == ENGINE_SUCCESS, "Set failed.");
1749    check_key_value(h, h1, key, data, strlen(data));
1750    h1->release(h, NULL, it);
1751
1752    testHarness.time_travel(5);
1753    check(h1->get(h, NULL, &it, key, strlen(key), 0) == ENGINE_KEY_ENOENT,
1754          "Item didn't expire");
1755
1756    int expired_access = get_int_stat(h, h1, "ep_expired_access");
1757    int expired_pager = get_int_stat(h, h1, "ep_expired_pager");
1758    int active_expired = get_int_stat(h, h1, "vb_active_expired");
1759    check(expired_pager == 0, "Expected zero expired item by pager");
1760    check(expired_access == 1, "Expected an expired item on access");
1761    check(active_expired == 1, "Expected an expired active item");
1762    checkeq(ENGINE_SUCCESS, store(h, h1, NULL, OPERATION_SET, key, data, &it),
1763            "Failed set.");
1764    h1->release(h, NULL, it);
1765
1766    std::stringstream ss;
1767    ss << "curr_items stat should be still 1 after ";
1768    ss << "overwriting the key that was expired, but not purged yet";
1769    checkeq(1, get_int_stat(h, h1, "curr_items"), ss.str().c_str());
1770
1771    return SUCCESS;
1772}
1773
1774static enum test_result test_expiry_loader(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1775    const char *key = "test_expiry_loader";
1776    const char *data = "some test data here.";
1777
1778    item *it = NULL;
1779
1780    ENGINE_ERROR_CODE rv;
1781    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 2,
1782                      PROTOCOL_BINARY_RAW_BYTES);
1783    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1784
1785    item_info info;
1786    info.nvalue = 1;
1787    if (!h1->get_item_info(h, NULL, it, &info)) {
1788        abort();
1789    }
1790    memcpy(info.value[0].iov_base, data, strlen(data));
1791
1792    uint64_t cas = 0;
1793    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
1794    check(rv == ENGINE_SUCCESS, "Set failed.");
1795    check_key_value(h, h1, key, data, strlen(data));
1796    h1->release(h, NULL, it);
1797
1798    testHarness.time_travel(3);
1799
1800    check(h1->get(h, NULL, &it, key, strlen(key), 0) == ENGINE_KEY_ENOENT,
1801          "Item didn't expire");
1802
1803    // Restart the engine to ensure the above expired item is not loaded
1804    testHarness.reload_engine(&h, &h1,
1805                              testHarness.engine_path,
1806                              testHarness.get_current_testcase()->cfg,
1807                              true, false);
1808    wait_for_warmup_complete(h, h1);
1809    cb_assert(0 == get_int_stat(h, h1, "ep_warmup_value_count", "warmup"));
1810
1811    return SUCCESS;
1812}
1813
1814static enum test_result test_bug3454(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1815    const char *key = "test_expiry_duplicate_warmup";
1816    const char *data = "some test data here.";
1817
1818    item *it = NULL;
1819
1820    ENGINE_ERROR_CODE rv;
1821    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 5,
1822                      PROTOCOL_BINARY_RAW_BYTES);
1823    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1824
1825    item_info info;
1826    info.nvalue = 1;
1827    if (!h1->get_item_info(h, NULL, it, &info)) {
1828        abort();
1829    }
1830    memcpy(info.value[0].iov_base, data, strlen(data));
1831
1832    uint64_t cas = 0;
1833    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
1834    check(rv == ENGINE_SUCCESS, "Set failed.");
1835    check_key_value(h, h1, key, data, strlen(data));
1836    h1->release(h, NULL, it);
1837    wait_for_flusher_to_settle(h, h1);
1838
1839    // Advance the ep_engine time by 10 sec for the above item to be expired.
1840    testHarness.time_travel(10);
1841    check(h1->get(h, NULL, &it, key, strlen(key), 0) == ENGINE_KEY_ENOENT,
1842          "Item didn't expire");
1843
1844    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 0,
1845                      PROTOCOL_BINARY_RAW_BYTES);
1846    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1847
1848    info.nvalue = 1;
1849    if (!h1->get_item_info(h, NULL, it, &info)) {
1850        abort();
1851    }
1852    memcpy(info.value[0].iov_base, data, strlen(data));
1853
1854    cas = 0;
1855    // Add a new item with the same key.
1856    rv = h1->store(h, NULL, it, &cas, OPERATION_ADD, 0);
1857    check(rv == ENGINE_SUCCESS, "Add failed.");
1858    check_key_value(h, h1, key, data, strlen(data));
1859    h1->release(h, NULL, it);
1860
1861    check(h1->get(h, NULL, &it, key, strlen(key), 0) == ENGINE_SUCCESS,
1862          "Item shouldn't expire");
1863    h1->release(h, NULL, it);
1864
1865    // Restart the engine to ensure the above unexpired new item is loaded
1866    testHarness.reload_engine(&h, &h1,
1867                              testHarness.engine_path,
1868                              testHarness.get_current_testcase()->cfg,
1869                              true, false);
1870    wait_for_warmup_complete(h, h1);
1871    cb_assert(1 == get_int_stat(h, h1, "ep_warmup_value_count", "warmup"));
1872    cb_assert(0 == get_int_stat(h, h1, "ep_warmup_dups", "warmup"));
1873
1874    return SUCCESS;
1875}
1876
1877static enum test_result test_bug3522(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1878    const char *key = "test_expiry_no_items_warmup";
1879    const char *data = "some test data here.";
1880
1881    item *it = NULL;
1882
1883    ENGINE_ERROR_CODE rv;
1884    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 0,
1885                      PROTOCOL_BINARY_RAW_BYTES);
1886    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1887
1888    item_info info;
1889    info.nvalue = 1;
1890    if (!h1->get_item_info(h, NULL, it, &info)) {
1891        abort();
1892    }
1893    memcpy(info.value[0].iov_base, data, strlen(data));
1894
1895    uint64_t cas = 0;
1896    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
1897    check(rv == ENGINE_SUCCESS, "Set failed.");
1898    check_key_value(h, h1, key, data, strlen(data));
1899    h1->release(h, NULL, it);
1900    wait_for_flusher_to_settle(h, h1);
1901
1902    // Add a new item with the same key and 2 sec of expiration.
1903    const char *new_data = "new data here.";
1904    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(new_data), 0, 2,
1905                      PROTOCOL_BINARY_RAW_BYTES);
1906    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1907
1908    info.nvalue = 1;
1909    if (!h1->get_item_info(h, NULL, it, &info)) {
1910        abort();
1911    }
1912    memcpy(info.value[0].iov_base, new_data, strlen(new_data));
1913
1914    int pager_runs = get_int_stat(h, h1, "ep_num_expiry_pager_runs");
1915    cas = 0;
1916    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
1917    check(rv == ENGINE_SUCCESS, "Set failed.");
1918    check_key_value(h, h1, key, new_data, strlen(new_data));
1919    h1->release(h, NULL, it);
1920    testHarness.time_travel(3);
1921    wait_for_stat_change(h, h1, "ep_num_expiry_pager_runs", pager_runs);
1922    wait_for_flusher_to_settle(h, h1);
1923
1924    // Restart the engine.
1925    testHarness.reload_engine(&h, &h1,
1926                              testHarness.engine_path,
1927                              testHarness.get_current_testcase()->cfg,
1928                              true, false);
1929    wait_for_warmup_complete(h, h1);
1930    // TODO: modify this for a better test case
1931    cb_assert(0 == get_int_stat(h, h1, "ep_warmup_dups", "warmup"));
1932
1933    return SUCCESS;
1934}
1935
1936static enum test_result test_get_replica_active_state(ENGINE_HANDLE *h,
1937                                                      ENGINE_HANDLE_V1 *h1) {
1938    protocol_binary_request_header *pkt;
1939    pkt = prepare_get_replica(h, h1, vbucket_state_active);
1940    check(h1->unknown_command(h, NULL, pkt, add_response) ==
1941          ENGINE_SUCCESS, "Get Replica Failed");
1942    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
1943          "Expected PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET response.");
1944
1945    return SUCCESS;
1946}
1947
1948static enum test_result test_get_replica_pending_state(ENGINE_HANDLE *h,
1949                                                       ENGINE_HANDLE_V1 *h1) {
1950    protocol_binary_request_header *pkt;
1951
1952    const void *cookie = testHarness.create_cookie();
1953    testHarness.set_ewouldblock_handling(cookie, false);
1954    pkt = prepare_get_replica(h, h1, vbucket_state_pending);
1955    check(h1->unknown_command(h, cookie, pkt, add_response) ==
1956          ENGINE_EWOULDBLOCK, "Should have returned error for pending state");
1957    testHarness.destroy_cookie(cookie);
1958    return SUCCESS;
1959}
1960
1961static enum test_result test_get_replica_dead_state(ENGINE_HANDLE *h,
1962                                                    ENGINE_HANDLE_V1 *h1) {
1963    protocol_binary_request_header *pkt;
1964    pkt = prepare_get_replica(h, h1, vbucket_state_dead);
1965    check(h1->unknown_command(h, NULL, pkt, add_response) ==
1966          ENGINE_SUCCESS, "Get Replica Failed");
1967    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
1968          "Expected PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET response.");
1969
1970    return SUCCESS;
1971}
1972
1973static enum test_result test_get_replica(ENGINE_HANDLE *h,
1974                                         ENGINE_HANDLE_V1 *h1) {
1975    protocol_binary_request_header *pkt;
1976    pkt = prepare_get_replica(h, h1, vbucket_state_replica);
1977    check(h1->unknown_command(h, NULL, pkt, add_response) == ENGINE_SUCCESS,
1978                              "Get Replica Failed");
1979    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
1980          "Expected PROTOCOL_BINARY_RESPONSE_SUCCESS response.");
1981    check(strcmp("replicadata", last_body) == 0,
1982                 "Should have returned identical value");
1983
1984    return SUCCESS;
1985}
1986
1987static enum test_result test_get_replica_non_resident(ENGINE_HANDLE *h,
1988                                                      ENGINE_HANDLE_V1 *h1) {
1989
1990    item *i = NULL;
1991    check(store(h, h1, NULL, OPERATION_SET, "key", "value", &i, 0, 0)
1992          == ENGINE_SUCCESS, "Store Failed");
1993    h1->release(h, NULL, i);
1994    wait_for_flusher_to_settle(h, h1);
1995    wait_for_stat_to_be(h, h1, "ep_total_persisted", 1);
1996
1997    evict_key(h, h1, "key", 0, "Ejected.");
1998    check(set_vbucket_state(h, h1, 0, vbucket_state_replica),
1999          "Failed to set vbucket to replica");
2000
2001    get_replica(h, h1, "key", 0);
2002    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "Expected success");
2003
2004    return SUCCESS;
2005}
2006
2007static enum test_result test_get_replica_invalid_key(ENGINE_HANDLE *h,
2008                                                     ENGINE_HANDLE_V1 *h1) {
2009    protocol_binary_request_header *pkt;
2010    bool makeinvalidkey = true;
2011    pkt = prepare_get_replica(h, h1, vbucket_state_replica, makeinvalidkey);
2012    check(h1->unknown_command(h, NULL, pkt, add_response) ==
2013          ENGINE_SUCCESS, "Get Replica Failed");
2014    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2015          "Expected PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET response.");
2016    return SUCCESS;
2017}
2018
2019static enum test_result test_vb_del_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2020    const void *cookie = testHarness.create_cookie();
2021    testHarness.set_ewouldblock_handling(cookie, false);
2022    check(set_vbucket_state(h, h1, 1, vbucket_state_pending), "Failed to set vbucket state.");
2023    check(ENGINE_EWOULDBLOCK == del(h, h1, "key", 0, 1, cookie),
2024          "Expected woodblock.");
2025    testHarness.destroy_cookie(cookie);
2026    return SUCCESS;
2027}
2028
2029static enum test_result test_vb_del_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2030    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set vbucket state.");
2031    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
2032    check(ENGINE_NOT_MY_VBUCKET == del(h, h1, "key", 0, 1),
2033          "Expected not my vbucket.");
2034    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
2035    return SUCCESS;
2036}
2037
2038static enum test_result test_touch(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2039    // key is a mandatory field!
2040    touch(h, h1, NULL, 0, (time(NULL) + 10));
2041    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2042
2043    // extlen is a mandatory field!
2044    protocol_binary_request_header *request;
2045    request = createPacket(PROTOCOL_BINARY_CMD_TOUCH, 0, 0, NULL, 0, "akey", 4);
2046    check(h1->unknown_command(h, NULL, request, add_response) == ENGINE_SUCCESS,
2047          "Failed to call touch");
2048    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2049    free(request);
2050
2051    // Try to touch an unknown item...
2052    touch(h, h1, "mykey", 0, (time(NULL) + 10));
2053    check(last_status == PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, "Testing unknown key");
2054
2055    // illegal vbucket
2056    touch(h, h1, "mykey", 5, (time(NULL) + 10));
2057    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET, "Testing illegal vbucket");
2058
2059    // Store the item!
2060    item *itm = NULL;
2061    check(store(h, h1, NULL, OPERATION_SET, "mykey", "somevalue", &itm) == ENGINE_SUCCESS,
2062          "Failed set.");
2063    h1->release(h, NULL, itm);
2064
2065    check_key_value(h, h1, "mykey", "somevalue", strlen("somevalue"));
2066
2067    touch(h, h1, "mykey", 0, (time(NULL) + 10));
2068    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch mykey");
2069
2070    // time-travel 9 secs..
2071    testHarness.time_travel(9);
2072
2073    // The item should still exist
2074    check_key_value(h, h1, "mykey", "somevalue", 9);
2075
2076    // time-travel 2 secs..
2077    testHarness.time_travel(2);
2078
2079    // The item should have expired now...
2080    check(h1->get(h, NULL, &itm, "mykey", 5, 0) == ENGINE_KEY_ENOENT, "Item should be gone");
2081    return SUCCESS;
2082}
2083
2084static enum test_result test_touch_mb7342(ENGINE_HANDLE *h,
2085                                          ENGINE_HANDLE_V1 *h1) {
2086    const char *key = "MB-7342";
2087    // Store the item!
2088    item *itm = NULL;
2089    check(store(h, h1, NULL, OPERATION_SET, key, "v", &itm) == ENGINE_SUCCESS,
2090          "Failed set.");
2091    h1->release(h, NULL, itm);
2092
2093    touch(h, h1, key, 0, 0);
2094    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch key");
2095
2096    check_key_value(h, h1, key, "v", 1);
2097
2098    // Travel a loong time to see if the object is still there (the default
2099    // store sets an exp time of 3600
2100    testHarness.time_travel(3700);
2101
2102    check_key_value(h, h1, key, "v", 1);
2103
2104    return SUCCESS;
2105}
2106
2107static enum test_result test_touch_mb10277(ENGINE_HANDLE *h,
2108                                            ENGINE_HANDLE_V1 *h1) {
2109    const char *key = "MB-10277";
2110    // Store the item!
2111    item *itm = NULL;
2112    check(store(h, h1, NULL, OPERATION_SET, key, "v", &itm) == ENGINE_SUCCESS,
2113          "Failed set.");
2114    h1->release(h, NULL, itm);
2115    wait_for_flusher_to_settle(h, h1);
2116    evict_key(h, h1, key, 0, "Ejected.");
2117
2118    touch(h, h1, key, 0, 3600); // A new expiration time remains in the same.
2119    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch key");
2120
2121    return SUCCESS;
2122}
2123
2124static enum test_result test_gat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2125    // key is a mandatory field!
2126    gat(h, h1, NULL, 0, 10);
2127    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2128
2129    // extlen is a mandatory field!
2130    protocol_binary_request_header *request;
2131    request = createPacket(PROTOCOL_BINARY_CMD_GAT, 0, 0, NULL, 0, "akey", 4);
2132    check(h1->unknown_command(h, NULL, request, add_response) == ENGINE_SUCCESS,
2133          "Failed to call gat");
2134    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2135    free(request);
2136
2137    // Try to gat an unknown item...
2138    gat(h, h1, "mykey", 0, 10);
2139    check(last_status == PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, "Testing unknown key");
2140
2141    // illegal vbucket
2142    gat(h, h1, "mykey", 5, 10);
2143    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET, "Testing illegal vbucket");
2144
2145    // Store the item!
2146    item *itm = NULL;
2147    check(store(h, h1, NULL, OPERATION_SET, "mykey", "somevalue", &itm) == ENGINE_SUCCESS,
2148          "Failed set.");
2149    h1->release(h, NULL, itm);
2150
2151    check_key_value(h, h1, "mykey", "somevalue", strlen("somevalue"));
2152
2153    gat(h, h1, "mykey", 0, 10);
2154    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "gat mykey");
2155    check(memcmp(last_body, "somevalue", sizeof("somevalue")) == 0,
2156          "Invalid data returned");
2157
2158    // time-travel 9 secs..
2159    testHarness.time_travel(9);
2160
2161    // The item should still exist
2162    check_key_value(h, h1, "mykey", "somevalue", 9);
2163
2164    // time-travel 2 secs..
2165    testHarness.time_travel(2);
2166
2167    // The item should have expired now...
2168    check(h1->get(h, NULL, &itm, "mykey", 5, 0) == ENGINE_KEY_ENOENT, "Item should be gone");
2169    return SUCCESS;
2170}
2171
2172static enum test_result test_gatq(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2173    // key is a mandatory field!
2174    gat(h, h1, NULL, 0, 10, true);
2175    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2176
2177    // extlen is a mandatory field!
2178    protocol_binary_request_header *request;
2179    request = createPacket(PROTOCOL_BINARY_CMD_GATQ, 0, 0, NULL, 0, "akey", 4);
2180    check(h1->unknown_command(h, NULL, request, add_response) == ENGINE_SUCCESS,
2181          "Failed to call gatq");
2182    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2183    free(request);
2184
2185    // Try to gatq an unknown item...
2186    last_status = static_cast<protocol_binary_response_status>(0xffff);
2187    gat(h, h1, "mykey", 0, 10, true);
2188
2189    // We should not have sent any response!
2190    check(last_status == (protocol_binary_response_status)0xffff, "Testing unknown key");
2191
2192    // illegal vbucket
2193    gat(h, h1, "mykey", 5, 10, true);
2194    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2195          "Testing illegal vbucket");
2196
2197    // Store the item!
2198    item *itm = NULL;
2199    check(store(h, h1, NULL, OPERATION_SET, "mykey", "somevalue", &itm) == ENGINE_SUCCESS,
2200          "Failed set.");
2201    h1->release(h, NULL, itm);
2202
2203    check_key_value(h, h1, "mykey", "somevalue", strlen("somevalue"));
2204
2205    gat(h, h1, "mykey", 0, 10, true);
2206    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "gat mykey");
2207    check(memcmp(last_body, "somevalue", sizeof("somevalue")) == 0,
2208          "Invalid data returned");
2209
2210    // time-travel 9 secs..
2211    testHarness.time_travel(9);
2212
2213    // The item should still exist
2214    check_key_value(h, h1, "mykey", "somevalue", 9);
2215
2216    // time-travel 2 secs..
2217    testHarness.time_travel(2);
2218
2219    // The item should have expired now...
2220    check(h1->get(h, NULL, &itm, "mykey", 5, 0) == ENGINE_KEY_ENOENT, "Item should be gone");
2221    return SUCCESS;
2222}
2223
2224static enum test_result test_mb5215(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2225    item *itm = NULL;
2226    check(store(h, h1, NULL, OPERATION_SET, "coolkey", "cooler", &itm)
2227          == ENGINE_SUCCESS, "Failed set.");
2228    h1->release(h, NULL, itm);
2229
2230    check_key_value(h, h1, "coolkey", "cooler", strlen("cooler"));
2231
2232    // set new exptime to 111
2233    int expTime = time(NULL) + 111;
2234
2235    touch(h, h1, "coolkey", 0, expTime);
2236    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch coolkey");
2237
2238    //reload engine
2239    testHarness.reload_engine(&h, &h1,
2240                              testHarness.engine_path,
2241                              testHarness.get_current_testcase()->cfg,
2242                              true, false);
2243    wait_for_warmup_complete(h, h1);
2244
2245    //verify persisted expiration time
2246    const char *statkey = "key coolkey 0";
2247    int newExpTime;
2248    check(h1->get(h, NULL, &itm, "coolkey", 7, 0) == ENGINE_SUCCESS,
2249          "Missing key");
2250    h1->release(h, NULL, itm);
2251    newExpTime = get_int_stat(h, h1, "key_exptime", statkey);
2252    check(newExpTime == expTime, "Failed to persist new exptime");
2253
2254    // evict key, touch expiration time, and verify
2255    evict_key(h, h1, "coolkey", 0, "Ejected.");
2256
2257    expTime = time(NULL) + 222;
2258    touch(h, h1, "coolkey", 0, expTime);
2259    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch coolkey");
2260
2261    testHarness.reload_engine(&h, &h1,
2262                              testHarness.engine_path,
2263                              testHarness.get_current_testcase()->cfg,
2264                              true, false);
2265    wait_for_warmup_complete(h, h1);
2266
2267    check(h1->get(h, NULL, &itm, "coolkey", 7, 0) == ENGINE_SUCCESS,
2268          "Missing key");
2269    h1->release(h, NULL, itm);
2270    newExpTime = get_int_stat(h, h1, "key_exptime", statkey);
2271    check(newExpTime == expTime, "Failed to persist new exptime");
2272
2273    return SUCCESS;
2274}
2275
2276static enum test_result test_alloc_limit(ENGINE_HANDLE *h,
2277                                         ENGINE_HANDLE_V1 *h1) {
2278    item *it = NULL;
2279    ENGINE_ERROR_CODE rv;
2280
2281    rv = h1->allocate(h, NULL, &it, "key", 3, 20 * 1024 * 1024, 0, 0,
2282                      PROTOCOL_BINARY_RAW_BYTES);
2283    check(rv == ENGINE_SUCCESS, "Allocated 20MB item");
2284    h1->release(h, NULL, it);
2285
2286    rv = h1->allocate(h, NULL, &it, "key", 3, (20 * 1024 * 1024) + 1, 0, 0,
2287                      PROTOCOL_BINARY_RAW_BYTES);
2288    check(rv == ENGINE_E2BIG, "Object too big");
2289
2290    return SUCCESS;
2291}
2292
2293static enum test_result test_whitespace_db(ENGINE_HANDLE *h,
2294                                           ENGINE_HANDLE_V1 *h1) {
2295    vals.clear();
2296    check(h1->get_stats(h, NULL, NULL, 0, add_stats) == ENGINE_SUCCESS,
2297          "Failed to get stats.");
2298    if (vals["ep_dbname"] != std::string(WHITESPACE_DB)) {
2299        std::cerr << "Expected dbname = ``" << WHITESPACE_DB << "''"
2300                  << ", got ``" << vals["ep_dbname"] << "''" << std::endl;
2301        return FAIL;
2302    }
2303
2304    check(access(WHITESPACE_DB, F_OK) != -1, "I expected the whitespace db to exist");
2305    return SUCCESS;
2306}
2307
2308static enum test_result test_memory_limit(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2309    set_param(h, h1, protocol_binary_engine_param_flush, "mutation_mem_threshold", "95");
2310    int used = get_int_stat(h, h1, "mem_used");
2311    double mem_threshold =
2312        static_cast<double>(get_int_stat(h, h1, "ep_mutation_mem_threshold")) / 100;
2313    int max = static_cast<int>(get_int_stat(h, h1, "ep_max_size") * mem_threshold);
2314    check(get_int_stat(h, h1, "ep_oom_errors") == 0 &&
2315          get_int_stat(h, h1, "ep_tmp_oom_errors") == 0, "Expected no OOM errors.");
2316    cb_assert(used < max);
2317
2318    char data[8192];
2319    memset(data, 'x', sizeof(data));
2320    size_t vlen = max - used - 192;
2321    data[vlen] = 0x00;
2322
2323    item *i = NULL;
2324    // So if we add an item,
2325    check(store(h, h1, NULL, OPERATION_SET, "key", data, &i) == ENGINE_SUCCESS,
2326          "store failure");
2327    check_key_value(h, h1, "key", data, vlen);
2328    h1->release(h, NULL, i);
2329
2330    // There should be no room for another.
2331    ENGINE_ERROR_CODE second = store(h, h1, NULL, OPERATION_SET, "key2", data, &i);
2332    check(second == ENGINE_ENOMEM || second == ENGINE_TMPFAIL,
2333          "should have failed second set");
2334    h1->release(h, NULL, i);
2335    check(get_int_stat(h, h1, "ep_oom_errors") == 1 ||
2336          get_int_stat(h, h1, "ep_tmp_oom_errors") == 1, "Expected an OOM error.");
2337
2338    ENGINE_ERROR_CODE overwrite = store(h, h1, NULL, OPERATION_SET, "key", data, &i);
2339    check(overwrite == ENGINE_ENOMEM || overwrite == ENGINE_TMPFAIL,
2340          "should have failed second override");
2341    h1->release(h, NULL, i);
2342    check(get_int_stat(h, h1, "ep_oom_errors") == 2 ||
2343          get_int_stat(h, h1, "ep_tmp_oom_errors") == 2, "Expected another OOM error.");
2344    check_key_value(h, h1, "key", data, vlen);
2345    check(ENGINE_SUCCESS != verify_key(h, h1, "key2"), "Expected a failure in GET");
2346    int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
2347    // Until we remove that item
2348    check(del(h, h1, "key", 0, 0) == ENGINE_SUCCESS, "Failed remove with value.");
2349    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
2350    testHarness.time_travel(65);
2351    wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
2352
2353    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue2", &i) == ENGINE_SUCCESS,
2354          "should have succeded on the last set");
2355    check_key_value(h, h1, "key2", "somevalue2", 10);
2356    h1->release(h, NULL, i);
2357    return SUCCESS;
2358}
2359
2360static enum test_result test_vbucket_get_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2361    return verify_vbucket_missing(h, h1, 1) ? SUCCESS : FAIL;
2362}
2363
2364static enum test_result test_vbucket_get(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2365    return verify_vbucket_state(h, h1, 0, vbucket_state_active) ? SUCCESS : FAIL;
2366}
2367
2368static enum test_result test_vbucket_create(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2369    if (!verify_vbucket_missing(h, h1, 1)) {
2370        fprintf(stderr, "vbucket wasn't missing.\n");
2371        return FAIL;
2372    }
2373
2374    if (!set_vbucket_state(h, h1, 1, vbucket_state_active)) {
2375        fprintf(stderr, "set state failed.\n");
2376        return FAIL;
2377    }
2378
2379    return verify_vbucket_state(h, h1, 1, vbucket_state_active) ? SUCCESS : FAIL;
2380}
2381
2382static enum test_result test_vbucket_compact(ENGINE_HANDLE *h,
2383                                             ENGINE_HANDLE_V1 *h1) {
2384    const char *key = "Carss";
2385    const char *value = "pollute";
2386    if (!verify_vbucket_missing(h, h1, 0)) {
2387        fprintf(stderr, "vbucket wasn't missing.\n");
2388        return FAIL;
2389    }
2390
2391    if (!set_vbucket_state(h, h1, 0, vbucket_state_active)) {
2392        fprintf(stderr, "set state failed.\n");
2393        return FAIL;
2394    }
2395
2396    check(verify_vbucket_state(h, h1, 0, vbucket_state_active),
2397            "VBucket state not active");
2398
2399    // Set two keys - one to be expired and other to remain...
2400    item *itm = NULL;
2401    check(store(h, h1, NULL, OPERATION_SET, key, value, &itm)
2402          == ENGINE_SUCCESS, "Failed set.");
2403    h1->release(h, NULL, itm);
2404
2405    check_key_value(h, h1, key, value, strlen(value));
2406
2407    // Set a non-expiring key...
2408    check(store(h, h1, NULL, OPERATION_SET, "trees", "cleanse", &itm)
2409          == ENGINE_SUCCESS, "Failed set.");
2410    h1->release(h, NULL, itm);
2411
2412    check_key_value(h, h1, "trees", "cleanse", strlen("cleanse"));
2413
2414    touch(h, h1, key, 0, 11);
2415    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch Carss");
2416
2417    testHarness.time_travel(12);
2418    wait_for_flusher_to_settle(h, h1);
2419
2420    check(get_int_stat(h, h1, "vb_0:purge_seqno", "vbucket-seqno") == 0,
2421            "purge_seqno not found to be zero before compaction");
2422
2423    // Compaction on VBucket
2424    compact_db(h, h1, 0, 2, 3, 1);
2425
2426    check(get_int_stat(h, h1, "ep_pending_compactions") == 0,
2427    "ep_pending_compactions stat did not tick down after compaction command");
2428
2429    // the key tree and its value should be intact...
2430    check(verify_key(h, h1, "trees") == ENGINE_SUCCESS,
2431          "key trees should be found.");
2432    // the key Carrs should have disappeared...
2433    int val = verify_key(h, h1, "Carss");
2434    check(val == ENGINE_KEY_ENOENT, "Key Carss has not expired.");
2435
2436    check(get_int_stat(h, h1, "vb_0:purge_seqno", "vbucket-seqno") == 3,
2437        "purge_seqno didn't match expected value");
2438
2439    return SUCCESS;
2440}
2441
2442static enum test_result vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
2443                                             const char* value = NULL) {
2444    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed to set vbucket state.");
2445
2446    vbucketDelete(h, h1, 2, value);
2447    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2448          "Expected failure deleting non-existent bucket.");
2449
2450    check(set_vbucket_state(h, h1, 1, vbucket_state_dead), "Failed set set vbucket 1 state.");
2451
2452    vbucketDelete(h, h1, 1, value);
2453    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
2454          "Expected failure deleting non-existent bucket.");
2455
2456    check(verify_vbucket_missing(h, h1, 1),
2457          "vbucket 0 was not missing after deleting it.");
2458
2459    return SUCCESS;
2460}
2461
2462static enum test_result test_vbucket_destroy_stats(ENGINE_HANDLE *h,
2463                                                   ENGINE_HANDLE_V1 *h1) {
2464
2465    int mem_used = get_int_stat(h, h1, "mem_used");
2466    int cacheSize = get_int_stat(h, h1, "ep_total_cache_size");
2467    int overhead = get_int_stat(h, h1, "ep_overhead");
2468    int nonResident = get_int_stat(h, h1, "ep_num_non_resident");
2469
2470    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed to set vbucket state.");
2471
2472    std::vector<std::string> keys;
2473    for (int j = 0; j < 2000; ++j) {
2474        std::stringstream ss;
2475        ss << "key" << j;
2476        std::string key(ss.str());
2477        keys.push_back(key);
2478    }
2479
2480    int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
2481    std::vector<std::string>::iterator it;
2482    for (it = keys.begin(); it != keys.end(); ++it) {
2483        item *i;
2484        check(store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str(), &i, 0, 1)
2485              == ENGINE_SUCCESS, "Failed to store a value");
2486        h1->release(h, NULL, i);
2487    }
2488    wait_for_flusher_to_settle(h, h1);
2489    testHarness.time_travel(65);
2490    wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
2491
2492    check(set_vbucket_state(h, h1, 1, vbucket_state_dead), "Failed set set vbucket 1 state.");
2493
2494    int vbucketDel = get_int_stat(h, h1, "ep_vbucket_del");
2495    vbucketDelete(h, h1, 1);
2496    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
2497          "Expected failure deleting non-existent bucket.");
2498
2499    check(verify_vbucket_missing(h, h1, 1),
2500          "vbucket 1 was not missing after deleting it.");
2501
2502    wait_for_stat_change(h, h1, "ep_vbucket_del", vbucketDel);
2503
2504    wait_for_stat_to_be(h, h1, "mem_used", mem_used);
2505    wait_for_stat_to_be(h, h1, "ep_total_cache_size", cacheSize);
2506    wait_for_stat_to_be(h, h1, "ep_overhead", overhead);
2507    wait_for_stat_to_be(h, h1, "ep_num_non_resident", nonResident);
2508
2509    return SUCCESS;
2510}
2511
2512static enum test_result vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
2513                                                const char* value = NULL) {
2514    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed to set vbucket state.");
2515
2516    // Store a value so the restart will try to resurrect it.
2517    item *i = NULL;
2518    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i, 0, 1)
2519          == ENGINE_SUCCESS, "Failed to set a value");
2520    check_key_value(h, h1, "key", "somevalue", 9, 1);
2521    h1->release(h, NULL, i);
2522
2523    // Reload to get a flush forced.
2524    testHarness.reload_engine(&h, &h1,
2525                              testHarness.engine_path,
2526                              testHarness.get_current_testcase()->cfg,
2527                              true, false);
2528    wait_for_warmup_complete(h, h1);
2529
2530    check(verify_vbucket_state(h, h1, 1, vbucket_state_active),
2531          "Bucket state was what it was initially, after restart.");
2532    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed to set vbucket state.");
2533    check_key_value(h, h1, "key", "somevalue", 9, 1);
2534
2535    check(set_vbucket_state(h, h1, 1, vbucket_state_dead), "Failed set set vbucket 1 state.");
2536
2537    vbucketDelete(h, h1, 1, value);
2538    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
2539          "Expected failure deleting non-existent bucket.");
2540
2541    check(verify_vbucket_missing(h, h1, 1),
2542          "vbucket 1 was not missing after deleting it.");
2543
2544    testHarness.reload_engine(&h, &h1,
2545                              testHarness.engine_path,
2546                              testHarness.get_current_testcase()->cfg,
2547                              true, false);
2548    wait_for_warmup_complete(h, h1);
2549
2550    if (verify_vbucket_state(h, h1, 1, vbucket_state_pending, true)) {
2551        std::cerr << "Bucket came up in pending state after delete." << std::endl;
2552        abort();
2553    }
2554
2555    check(verify_vbucket_missing(h, h1, 1),
2556          "vbucket 1 was not missing after restart.");
2557
2558    return SUCCESS;
2559}
2560
2561static enum test_result test_async_vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2562    return vbucket_destroy(h, h1);
2563}
2564
2565static enum test_result test_sync_vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2566    return vbucket_destroy(h, h1, "async=0");
2567}
2568
2569static enum test_result test_async_vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2570    return vbucket_destroy_restart(h, h1);
2571}
2572
2573static enum test_result test_sync_vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2574    return vbucket_destroy_restart(h, h1, "async=0");
2575}
2576
2577static enum test_result test_vb_set_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2578    return test_pending_vb_mutation(h, h1, OPERATION_SET);
2579}
2580
2581static enum test_result test_vb_add_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2582    return test_pending_vb_mutation(h, h1, OPERATION_ADD);
2583}
2584
2585static enum test_result test_vb_cas_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2586    return test_pending_vb_mutation(h, h1, OPERATION_CAS);
2587}
2588
2589static enum test_result test_vb_append_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2590    return test_pending_vb_mutation(h, h1, OPERATION_APPEND);
2591}
2592
2593static enum test_result test_vb_prepend_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2594    return test_pending_vb_mutation(h, h1, OPERATION_PREPEND);
2595}
2596
2597static enum test_result test_vb_set_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2598    return test_replica_vb_mutation(h, h1, OPERATION_SET);
2599}
2600
2601static enum test_result test_vb_replace_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2602    return test_replica_vb_mutation(h, h1, OPERATION_REPLACE);
2603}
2604
2605static enum test_result test_vb_replace_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2606    return test_pending_vb_mutation(h, h1, OPERATION_REPLACE);
2607}
2608
2609static enum test_result test_vb_add_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2610    return test_replica_vb_mutation(h, h1, OPERATION_ADD);
2611}
2612
2613static enum test_result test_vb_cas_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2614    return test_replica_vb_mutation(h, h1, OPERATION_CAS);
2615}
2616
2617static enum test_result test_vb_append_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2618    return test_replica_vb_mutation(h, h1, OPERATION_APPEND);
2619}
2620
2621static enum test_result test_vb_prepend_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2622    return test_replica_vb_mutation(h, h1, OPERATION_PREPEND);
2623}
2624
2625static enum test_result test_stats_seqno(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2626    check(set_vbucket_state(h, h1, 1, vbucket_state_active),
2627          "Failed to set vbucket state.");
2628
2629    int num_keys = 100;
2630    for (int ii = 0; ii < num_keys; ++ii) {
2631        std::stringstream ss;
2632        ss << "key" << ii;
2633        check(store(h, h1, NULL, OPERATION_SET, ss.str().c_str(),
2634                    "value", NULL, 0, 0) == ENGINE_SUCCESS,
2635              "Failed to store an item.");
2636    }
2637
2638    check(get_int_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno") == 100,
2639          "Invalid seqno");
2640    check(get_int_stat(h, h1, "vb_1:high_seqno", "vbucket-seqno") == 0,
2641          "Invalid seqno");
2642    check(get_int_stat(h, h1, "vb_1:high_seqno", "vbucket-seqno 1") == 0,
2643          "Invalid seqno");
2644
2645    uint64_t vb_uuid = get_ull_stat(h, h1, "failovers:vb_1:0:id", "failovers");
2646    check(get_ull_stat(h, h1, "vb_1:uuid", "vbucket-seqno 1") == vb_uuid,
2647          "Invalid uuid");
2648    check(vals.size() == 3, "Expected three stats");
2649
2650    // Check invalid vbucket
2651    check(h1->get_stats(h, NULL, "vbucket-seqno 2", 15, add_stats)
2652          == ENGINE_NOT_MY_VBUCKET, "Expected not my vbucket");
2653
2654    // Check bad vbucket parameter (not numeric)
2655    check(h1->get_stats(h, NULL, "vbucket-seqno tt2", 17, add_stats)
2656          == ENGINE_EINVAL, "Expected invalid");
2657
2658    // Check extra spaces at the end
2659    check(h1->get_stats(h, NULL, "vbucket-seqno    ", 17, add_stats)
2660          == ENGINE_EINVAL, "Expected invalid");
2661
2662    return SUCCESS;
2663}
2664
2665static enum test_result test_stats_diskinfo(ENGINE_HANDLE *h,
2666                                            ENGINE_HANDLE_V1 *h1) {
2667    check(set_vbucket_state(h, h1, 1, vbucket_state_active),
2668          "Failed to set vbucket state.");
2669
2670    int num_keys = 100;
2671    for (int ii = 0; ii < num_keys; ++ii) {
2672        std::stringstream ss;
2673        ss << "key" << ii;
2674        check(store(h, h1, NULL, OPERATION_SET, ss.str().c_str(),
2675                    "value", NULL, 0, 1) == ENGINE_SUCCESS,
2676              "Failed to store an item.");
2677    }
2678    wait_for_flusher_to_settle(h, h1);
2679
2680    size_t file_size = get_int_stat(h, h1, "ep_db_file_size", "diskinfo");
2681    size_t data_size = get_int_stat(h, h1, "ep_db_data_size", "diskinfo");
2682    check(file_size > 0, "DB file size should be greater than 0");
2683    check(data_size > 0, "DB data size should be greater than 0");
2684    check(file_size >= data_size, "DB file size should be >= DB data size");
2685    check(get_int_stat(h, h1, "vb_1:data_size", "diskinfo detail") > 0,
2686          "VB 1 data size should be greater than 0");
2687
2688    check(h1->get_stats(h, NULL, "diskinfo ", 9, add_stats)
2689          == ENGINE_EINVAL, "Expected invalid");
2690
2691    check(h1->get_stats(h, NULL, "diskinfo detai", 14, add_stats)
2692          == ENGINE_EINVAL, "Expected invalid");
2693
2694    check(h1->get_stats(h, NULL, "diskinfo detaillll", 18, add_stats)
2695          == ENGINE_EINVAL, "Expected invalid");
2696
2697    return SUCCESS;
2698}
2699
2700static void notifier_request(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
2701                             const void* cookie, uint32_t opaque,
2702                             uint16_t vbucket, uint64_t start,
2703                             bool shouldSucceed) {
2704
2705    uint32_t flags = 0;
2706    uint64_t rollback = 0;
2707    uint64_t vb_uuid = get_ull_stat(h, h1, "failovers:vb_0:0:id", "failovers");
2708    uint64_t snap_start_seqno = get_ull_stat(h, h1, "failovers:vb_0:0:seq", "failovers");
2709    uint64_t snap_end_seqno = snap_start_seqno;
2710    ENGINE_ERROR_CODE err = h1->upr.stream_req(h, cookie, flags, opaque,
2711                                               vbucket, start, 0, vb_uuid,
2712                                               snap_start_seqno, snap_end_seqno,
2713                                               &rollback,
2714                                               mock_upr_add_failover_log);
2715    check(err == ENGINE_SUCCESS, "Failed to initiate stream request");
2716
2717    std::string type = get_str_stat(h, h1, "eq_uprq:unittest:type", "upr");
2718    check(type.compare("notifier") == 0, "Consumer not found");
2719
2720    check((uint32_t)get_int_stat(h, h1, "eq_uprq:unittest:stream_0_flags", "upr")
2721          == flags, "Flags didn't match");
2722    check((uint32_t)get_int_stat(h, h1, "eq_uprq:unittest:stream_0_opaque", "upr")
2723          == opaque, "Opaque didn't match");
2724    check((uint64_t)get_ull_stat(h, h1, "eq_uprq:unittest:stream_0_start_seqno", "upr")
2725          == start, "Start Seqno Didn't match");
2726    check((uint64_t)get_ull_stat(h, h1, "eq_uprq:unittest:stream_0_end_seqno", "upr")
2727          == 0, "End Seqno didn't match");
2728    check((uint64_t)get_ull_stat(h, h1, "eq_uprq:unittest:stream_0_vb_uuid", "upr")
2729          == vb_uuid, "VBucket UUID didn't match");
2730    check((uint64_t)get_ull_stat(h, h1, "eq_uprq:unittest:stream_0_snap_start_seqno", "upr")
2731          == snap_start_seqno, "snap start seqno didn't match");
2732}
2733
2734static enum test_result test_upr_notifier(ENGINE_HANDLE *h,
2735                                          ENGINE_HANDLE_V1 *h1) {
2736
2737    int num_items = 10;
2738    for (int j = 0; j < num_items; ++j) {
2739        item *i = NULL;
2740        std::stringstream ss;
2741        ss << "key" << j;
2742        check(store(h, h1, NULL, OPERATION_SET, ss.str().c_str(), "data", &i)
2743              == ENGINE_SUCCESS, "Failed to store a value");
2744        h1->release(h, NULL, i);
2745    }
2746
2747    const void *cookie = testHarness.create_cookie();
2748    uint32_t opaque = 0;
2749    uint32_t flags = UPR_OPEN_NOTIFIER;
2750    const char *name = "unittest";
2751    uint16_t nname = strlen(name);
2752
2753    check(h1->upr.open(h, cookie, opaque, 0, flags, (void*)name, nname)
2754          == ENGINE_SUCCESS,
2755          "Failed upr notifier open connection.");
2756
2757    // Get notification for an old item
2758    notifier_request(h, h1, cookie, ++opaque, 0, 0, true);
2759    upr_step(h, h1, cookie);
2760    check(upr_last_op == PROTOCOL_BINARY_CMD_UPR_STREAM_END,
2761          "Expected stream end");
2762
2763    // Get notification when we're slightly behind
2764    notifier_request(h, h1, cookie, ++opaque, 0, 9, true);
2765    upr_step(h, h1, cookie);
2766    check(upr_last_op == PROTOCOL_BINARY_CMD_UPR_STREAM_END,
2767          "Expected stream end");
2768
2769    // Wait for notification of a future item
2770    notifier_request(h, h1, cookie, ++opaque, 0, 20, true);
2771    upr_step(h, h1, cookie);
2772
2773    for (int j = 0; j < 5; ++j) {
2774        item *i = NULL;
2775        std::stringstream ss;
2776        ss << "key" << j;
2777        check(store(h, h1, NULL, OPERATION_SET, ss.str().c_str(), "data", &i)
2778              == ENGINE_SUCCESS, "Failed to store a value");
2779        h1->release(h, NULL, i);
2780    }
2781
2782    // Shouldn't get a stream end yet
2783    upr_step(h, h1, cookie);
2784    check(upr_last_op != PROTOCOL_BINARY_CMD_UPR_STREAM_END,
2785          "Wasn't expecting a stream end");
2786
2787    for (int j = 0; j < 6; ++j) {
2788        item *i = NULL;
2789        std::stringstream ss;
2790        ss << "key" << j;
2791        check(store(h, h1, NULL, OPERATION_SET, ss.str().c_str(), "data", &i)
2792              == ENGINE_SUCCESS, "Failed to store a value");
2793        h1->release(h, NULL, i);
2794    }
2795
2796    // Should get a stream end
2797    upr_step(h, h1, cookie);
2798    check(upr_last_op == PROTOCOL_BINARY_CMD_UPR_STREAM_END,
2799          "Expected stream end");
2800
2801    testHarness.destroy_cookie(cookie);
2802
2803    return SUCCESS;
2804}
2805
2806static enum test_result test_upr_consumer_open(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2807    const void *cookie1 = testHarness.create_cookie();
2808    uint32_t opaque = 0;
2809    uint32_t seqno = 0;
2810    uint32_t flags = 0;
2811    const char *name = "unittest";
2812    uint16_t nname = strlen(name);
2813
2814    check(h1->upr.open(h, cookie1, opaque, seqno, flags, (void*)name, nname)
2815          == ENGINE_SUCCESS,
2816          "Failed upr consumer open connection.");
2817
2818    std::string type = get_str_stat(h, h1, "eq_uprq:unittest:type", "upr");
2819    int created = get_int_stat(h, h1, "eq_uprq:unittest:created", "upr");
2820    check(type.compare("consumer") == 0, "Consumer not found");
2821    testHarness.destroy_cookie(cookie1);
2822
2823    testHarness.time_travel(600);
2824
2825    const void *cookie2 = testHarness.create_cookie();
2826    check(h1->upr.open(h, cookie2, opaque, seqno, flags, (void*)name, nname)
2827          == ENGINE_SUCCESS,
2828          "Failed upr consumer open connection.");
2829
2830    type = get_str_stat(h, h1, "eq_uprq:unittest:type", "upr");
2831    check(type.compare("consumer") == 0, "Consumer not found");
2832    check(get_int_stat(h, h1, "eq_uprq:unittest:created", "upr") > created,
2833          "New upr stream is not newer");
2834    testHarness.destroy_cookie(cookie2);
2835
2836    return SUCCESS;
2837}
2838
2839static enum test_result test_upr_producer_open(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2840    const void *cookie1 = testHarness.create_cookie();
2841    uint32_t opaque = 0;
2842    uint32_t seqno = 0;
2843    uint32_t flags = UPR_OPEN_PRODUCER;
2844    const char *name  = "unittest";
2845    uint16_t nname = strlen(name);
2846
2847    check(h1->upr.open(h, cookie1, opaque, seqno, flags, (void*)name, nname)
2848          == ENGINE_SUCCESS,
2849          "Failed upr producer open connection.");
2850
2851    std::string type = get_str_stat(h, h1, "eq_uprq:unittest:type", "upr");
2852    int created = get_int_stat(h, h1, "eq_uprq:unittest:created", "upr");
2853    check(type.compare("producer") == 0, "Producer not found");
2854    testHarness.destroy_cookie(cookie1);
2855
2856    testHarness.time_travel(600);
2857
2858    const void *cookie2 = testHarness.create_cookie();
2859    check(h1->upr.open(h, cookie2, opaque, seqno, flags, (void*)name, nname)
2860          == ENGINE_SUCCESS,
2861          "Failed upr producer open connection.");
2862
2863    type = get_str_stat(h, h1, "eq_uprq:unittest:type", "upr");
2864    check(type.compare("producer") == 0, "Producer not found");
2865    check(get_int_stat(h, h1, "eq_uprq:unittest:created", "upr") > created,
2866          "New upr stream is not newer");
2867    testHarness.destroy_cookie(cookie2);
2868
2869    return SUCCESS;
2870}
2871
2872static void upr_stream(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *name,
2873                       const void *cookie, uint16_t vbucket, uint32_t flags,
2874                       uint64_t start, uint64_t end, uint64_t vb_uuid,
2875                       uint64_t snap_start_seqno, uint64_t snap_end_seqno,
2876                       int exp_mutations, int exp_deletions, int exp_markers,
2877                       int extra_takeover_ops, int exp_nru_value,
2878                       bool exp_disk_snapshot = false) {
2879    uint32_t opaque = 1;
2880    uint16_t nname = strlen(name);
2881
2882    check(h1->upr.open(h, cookie, ++opaque, 0, UPR_OPEN_PRODUCER, (void*)name,
2883                       nname) == ENGINE_SUCCESS,
2884          "Failed upr producer open connection.");
2885
2886    check(h1->upr.control(h, cookie, ++opaque, "connection_buffer_size", 22,
2887                          "1024", 4) == ENGINE_SUCCESS,
2888          "Failed to establish connection buffer");
2889
2890    uint64_t rollback = 0;
2891    check(h1->upr.stream_req(h, cookie, flags, opaque, vbucket, start, end,
2892                             vb_uuid, snap_start_seqno, snap_end_seqno, &rollback,
2893                             mock_upr_add_failover_log)
2894                == ENGINE_SUCCESS,
2895          "Failed to initiate stream request");
2896
2897    end = (flags == UPR_ADD_STREAM_FLAG_TAKEOVER) ? -1 : end;
2898
2899    char stats_flags[50];
2900    snprintf(stats_flags, sizeof(stats_flags),"eq_uprq:%s:stream_0_flags", name);
2901    check((uint32_t)get_int_stat(h, h1, stats_flags, "upr")
2902          == flags, "Flags didn't match");
2903
2904    char stats_opaque[50];
2905    snprintf(stats_opaque, sizeof(stats_opaque),"eq_uprq:%s:stream_0_opaque", name);
2906    check((uint32_t)get_int_stat(h, h1, stats_opaque, "upr")
2907          == opaque, "Opaque didn't match");
2908
2909    char stats_start_seqno[50];
2910    snprintf(stats_start_seqno, sizeof(stats_start_seqno),"eq_uprq:%s:stream_0_start_seqno", name);
2911    check((uint64_t)get_ull_stat(h, h1, stats_start_seqno, "upr")
2912          == start, "Start Seqno Didn't match");
2913
2914    char stats_end_seqno[50];
2915    snprintf(stats_end_seqno, sizeof(stats_end_seqno),"eq_uprq:%s:stream_0_end_seqno", name);
2916    check((uint64_t)get_ull_stat(h, h1, stats_end_seqno, "upr")
2917          == end, "End Seqno didn't match");
2918
2919    char stats_vb_uuid[50];
2920    snprintf(stats_vb_uuid, sizeof(stats_vb_uuid),"eq_uprq:%s:stream_0_vb_uuid", name);
2921    check((uint64_t)get_ull_stat(h, h1, stats_vb_uuid, "upr")
2922          == vb_uuid, "VBucket UUID didn't match");
2923
2924    char stats_snap_seqno[50];
2925    snprintf(stats_snap_seqno, sizeof(stats_snap_seqno),"eq_uprq:%s:stream_0_snap_start_seqno", name);
2926    check((uint64_t)get_ull_stat(h, h1, stats_snap_seqno, "upr")
2927          == snap_start_seqno, "snap start seqno didn't match");
2928
2929    struct upr_message_producers* producers = get_upr_producers();
2930
2931    if ((flags & UPR_ADD_STREAM_FLAG_TAKEOVER) == 0 &&
2932        (flags & UPR_ADD_STREAM_FLAG_DISKONLY) == 0) {
2933        int est = exp_deletions + exp_mutations;
2934        char stats_takeover[50];
2935        snprintf(stats_takeover, sizeof(stats_takeover), "upr-vbtakeover 0 %s", name);
2936        wait_for_stat_to_be(h, h1, "estimate", est, stats_takeover);
2937    }
2938
2939    bool done = false;
2940    int num_mutations = 0;
2941    int num_deletions = 0;
2942    int num_snapshot_marker = 0;
2943    int num_set_vbucket_pending = 0;
2944    int num_set_vbucket_active = 0;
2945
2946    uint64_t last_by_seqno = 0;
2947    uint32_t bytes_read = 0;
2948    do {
2949        if (bytes_read > 512) {
2950            h1->upr.buffer_acknowledgement(h, cookie, ++opaque, 0, bytes_read);
2951            bytes_read = 0;
2952        }
2953        ENGINE_ERROR_CODE err = h1->upr.step(h, cookie, producers);
2954        if (err == ENGINE_DISCONNECT) {
2955            done = true;
2956        } else {
2957            switch (upr_last_op) {
2958                case PROTOCOL_BINARY_CMD_UPR_MUTATION:
2959                    check(last_by_seqno < upr_last_byseqno, "Expected bigger seqno");
2960                    check(upr_last_nru == exp_nru_value, "Expected different NRU value");
2961                    last_by_seqno =