xref: /3.0.3-GA/ep-engine/tests/ep_testsuite.cc (revision 46df358f)
1/* -*- MODE: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/*
3 *     Copyright 2010 Couchbase, Inc
4 *
5 *   Licensed under the Apache License, Version 2.0 (the "License");
6 *   you may not use this file except in compliance with the License.
7 *   You may obtain a copy of the License at
8 *
9 *       http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *   Unless required by applicable law or agreed to in writing, software
12 *   distributed under the License is distributed on an "AS IS" BASIS,
13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *   See the License for the specific language governing permissions and
15 *   limitations under the License.
16 */
17
18// Usage: (to repeatedly run just a single test case)
19// make engine_tests IS_LOOP=-L EP_TEST_NUM=3
20
21#include "config.h"
22
23#include <stdio.h>
24#include <stdlib.h>
25#include <string.h>
26#include <sys/stat.h>
27#ifdef _MSC_VER
28#include <direct.h>
29#define mkdir(a, b) _mkdir(a)
30#else
31#include <sys/wait.h>
32#endif
33
34#include <cstdlib>
35#include <iostream>
36#include <map>
37#include <set>
38#include <sstream>
39#include <string>
40#include <vector>
41
42#include <platform/dirutils.h>
43
44#include "atomic.h"
45#include "ep-engine/command_ids.h"
46#include "ep_test_apis.h"
47#include "ep_testsuite.h"
48#include "locks.h"
49#include "mock/mccouch.h"
50#include "mock/mock_upr.h"
51#include "mutex.h"
52
53#include <snappy-c.h>
54#include <JSON_checker.h>
55
56#ifdef linux
57/* /usr/include/netinet/in.h defines macros from ntohs() to _bswap_nn to
58 * optimize the conversion functions, but the prototypes generate warnings
59 * from gcc. The conversion methods isn't the bottleneck for my app, so
60 * just remove the warnings by undef'ing the optimization ..
61 */
62#undef ntohs
63#undef ntohl
64#undef htons
65#undef htonl
66#endif
67
68// ptr_fun don't like the extern "C" thing for unlock cookie.. cast it
69// away ;)
70typedef void (*UNLOCK_COOKIE_T)(const void *cookie);
71
72extern "C" bool abort_msg(const char *expr, const char *msg, int line);
73
74
75template <typename T>
76static void checkeqfn(T exp, T got, const char *msg, const char *file, const int linenum) {
77    if (exp != got) {
78        std::stringstream ss;
79        ss << "Expected `" << exp << "', got `" << got << "' - " << msg;
80        abort_msg(ss.str().c_str(), file, linenum);
81    }
82}
83
84#define checkeq(a, b, c) checkeqfn(a, b, c, __FILE__, __LINE__)
85
86extern "C" {
87
88#define check(expr, msg) \
89    static_cast<void>((expr) ? 0 : abort_msg(#expr, msg, __LINE__))
90
91#define WHITESPACE_DB "whitespace sucks.db"
92#define MULTI_DISPATCHER_CONFIG \
93    "ht_size=129;ht_locks=3;chk_remover_stime=1;chk_period=60"
94
95struct test_harness testHarness;
96
97class ThreadData {
98public:
99    ThreadData(ENGINE_HANDLE *eh, ENGINE_HANDLE_V1 *ehv1,
100               int e=0) : h(eh), h1(ehv1), extra(e) {}
101    ENGINE_HANDLE    *h;
102    ENGINE_HANDLE_V1 *h1;
103    int               extra;
104};
105
106bool abort_msg(const char *expr, const char *msg, int line) {
107    fprintf(stderr, "%s:%d Test failed: `%s' (%s)\n",
108            __FILE__, line, msg, expr);
109    abort();
110    // UNREACHABLE
111    return false;
112}
113
114static const char *dbname_env;
115static enum test_result rmdb(void)
116{
117    const char *files[] = { WHITESPACE_DB,
118                            "/tmp/test",
119                            "/tmp/mutation.log",
120                            dbname_env,
121                            NULL };
122    int ii = 0;
123    while (files[ii] != NULL) {
124        CouchbaseDirectoryUtilities::rmrf(files[ii]);
125        if (access(files[ii], F_OK) != -1) {
126            std::cerr << "Failed to remove: " << files[ii] << " " << std::endl;
127            return FAIL;
128        }
129        ++ii;
130    }
131
132    return SUCCESS;
133}
134
135static enum test_result skipped_test_function(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
136    (void) h;
137    (void) h1;
138    return SKIPPED;
139}
140
141static bool teardown(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
142    (void)h; (void)h1;
143    vals.clear();
144    return true;
145}
146
147static const void* createTapConn(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
148                                 const char *name) {
149    const void *cookie = testHarness.create_cookie();
150    testHarness.lock_cookie(cookie);
151    TAP_ITERATOR iter = h1->get_tap_iterator(h, cookie, name,
152                                             strlen(name),
153                                             TAP_CONNECT_FLAG_DUMP, NULL,
154                                             0);
155    check(iter != NULL, "Failed to create a tap iterator");
156    return cookie;
157}
158
159static void check_key_value(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
160                            const char* key, const char* val, size_t vlen,
161                            uint16_t vbucket = 0) {
162    item_info info;
163    check(get_item_info(h, h1, &info, key, vbucket), "checking key and value");
164    check(info.nvalue == 1, "info.nvalue != 1");
165    check(vlen == info.value[0].iov_len, "Value length mismatch");
166    check(memcmp(info.value[0].iov_base, val, vlen) == 0, "Data mismatch");
167}
168
169static bool test_setup(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
170    wait_for_warmup_complete(h, h1);
171
172    check(h1->get_stats(h, NULL, "prev-vbucket", 12, add_stats) == ENGINE_SUCCESS,
173          "Failed to get the previous state of vbuckets");
174    if (vals.find("vb_0") == vals.end()) {
175        check(set_vbucket_state(h, h1, 0, vbucket_state_active),
176              "Failed to set VB0 state.");
177    }
178
179    wait_for_stat_change(h, h1, "ep_vb_snapshot_total", 0);
180
181    // warmup is complete, notify ep engine that it must now enable
182    // data traffic
183    protocol_binary_request_header *pkt = createPacket(PROTOCOL_BINARY_CMD_ENABLE_TRAFFIC);
184    check(h1->unknown_command(h, NULL, pkt, add_response) == ENGINE_SUCCESS,
185          "Failed to enable data traffic");
186    free(pkt);
187
188    return true;
189}
190
191static enum test_result test_getl(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
192    const char *key = "k1";
193    uint16_t vbucketId = 0;
194    uint32_t expiration = 25;
195
196    getl(h, h1, key, vbucketId, expiration);
197    check(last_status == PROTOCOL_BINARY_RESPONSE_KEY_ENOENT,
198          "expected the key to be missing...");
199    if (last_body != NULL && (strcmp(last_body, "NOT_FOUND") != 0)) {
200        fprintf(stderr, "Should have returned NOT_FOUND. Getl Failed");
201        abort();
202    }
203
204    item *i = NULL;
205    check(store(h, h1, NULL, OPERATION_SET, key, "{\"lock\":\"data\"}",
206                &i, 0, vbucketId, 3600, PROTOCOL_BINARY_DATATYPE_JSON)
207          == ENGINE_SUCCESS, "Failed to store an item.");
208    h1->release(h, NULL, i);
209
210    /* retry getl, should succeed */
211    getl(h, h1, key, vbucketId, expiration);
212    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
213          "Expected to be able to getl on first try");
214    check(strcmp("{\"lock\":\"data\"}", last_body) == 0, "Body was malformed.");
215    check(last_datatype == PROTOCOL_BINARY_DATATYPE_JSON,
216            "Expected datatype to be JSON");
217
218    /* wait 16 seconds */
219    testHarness.time_travel(16);
220
221    /* lock's taken so this should fail */
222    getl(h, h1, key, vbucketId, expiration);
223    check(last_status == PROTOCOL_BINARY_RESPONSE_ETMPFAIL,
224          "Expected to fail getl on second try");
225
226    if (last_body != NULL && (strcmp(last_body, "LOCK_ERROR") != 0)) {
227        fprintf(stderr, "Should have returned LOCK_ERROR. Getl Failed");
228        abort();
229    }
230
231    check(store(h, h1, NULL, OPERATION_SET, key, "lockdata2", &i, 0, vbucketId)
232          != ENGINE_SUCCESS, "Should have failed to store an item.");
233    h1->release(h, NULL, i);
234
235    /* wait another 10 seconds */
236    testHarness.time_travel(10);
237
238    /* retry set, should succeed */
239    check(store(h, h1, NULL, OPERATION_SET, key, "lockdata", &i, 0, vbucketId)
240          == ENGINE_SUCCESS, "Failed to store an item.");
241    h1->release(h, NULL, i);
242
243    /* point to wrong vbucket, to test NOT_MY_VB response */
244    getl(h, h1, key, 10, expiration);
245    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
246          "Should have received not my vbucket response");
247
248    /* acquire lock, should succeed */
249    getl(h, h1, key, vbucketId, expiration);
250    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
251          "Aquire lock should have succeeded");
252    check(last_datatype == PROTOCOL_BINARY_RAW_BYTES,
253            "Expected datatype to be RAW BYTES");
254
255    /* try an incr operation followed by a delete, both of which should fail */
256    uint64_t cas = 0;
257    uint64_t result = 0;
258
259    check(h1->arithmetic(h, NULL, key, 2, true, false, 1, 1, 0,
260                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
261                         0)  == ENGINE_TMPFAIL, "Incr failed");
262
263
264    check(del(h, h1, key, 0, 0) == ENGINE_TMPFAIL, "Delete failed");
265
266
267    /* bug MB 2699 append after getl should fail with ENGINE_TMPFAIL */
268
269    testHarness.time_travel(26);
270
271    char binaryData1[] = "abcdefg\0gfedcba";
272    char binaryData2[] = "abzdefg\0gfedcba";
273
274    check(storeCasVb11(h, h1, NULL, OPERATION_SET, key,
275                       binaryData1, sizeof(binaryData1) - 1, 82758, &i, 0, 0)
276          == ENGINE_SUCCESS,
277          "Failed set.");
278    h1->release(h, NULL, i);
279
280    /* acquire lock, should succeed */
281    getl(h, h1, key, vbucketId, expiration);
282    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
283          "Aquire lock should have succeeded");
284
285    /* append should fail */
286    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, key,
287                       binaryData2, sizeof(binaryData2) - 1, 82758, &i, 0, 0)
288          == ENGINE_TMPFAIL,
289          "Append should fail.");
290    h1->release(h, NULL, i);
291
292    /* bug MB 3252 & MB 3354.
293     * 1. Set a key with an expiry value.
294     * 2. Take a lock on the item before it expires
295     * 3. Wait for the item to expire
296     * 4. Perform a CAS operation, should fail
297     * 5. Perform a set operation, should succeed
298     */
299    const char *ekey = "test_expiry";
300    const char *edata = "some test data here.";
301
302    item *it = NULL;
303
304    check(h1->allocate(h, NULL, &it, ekey, strlen(ekey), strlen(edata), 0, 2,
305          PROTOCOL_BINARY_RAW_BYTES) == ENGINE_SUCCESS, "Allocation Failed");
306
307    item_info info;
308    info.nvalue = 1;
309    if (!h1->get_item_info(h, NULL, it, &info)) {
310        abort();
311    }
312    memcpy(info.value[0].iov_base, edata, strlen(edata));
313
314    check(h1->store(h, NULL, it, &cas, OPERATION_SET, 0) ==
315        ENGINE_SUCCESS, "Failed to Store item");
316    check_key_value(h, h1, ekey, edata, strlen(edata));
317    h1->release(h, NULL, it);
318
319    testHarness.time_travel(3);
320    cas = last_cas;
321
322    /* cas should fail */
323    check(storeCasVb11(h, h1, NULL, OPERATION_CAS, ekey,
324                       binaryData1, sizeof(binaryData1) - 1, 82758, &i, cas, 0)
325          != ENGINE_SUCCESS,
326          "CAS succeeded.");
327    h1->release(h, NULL, i);
328
329    /* but a simple store should succeed */
330    check(store(h, h1, NULL, OPERATION_SET, ekey, edata, &i, 0, vbucketId)
331          == ENGINE_SUCCESS, "Failed to store an item.");
332    h1->release(h, NULL, i);
333    return SUCCESS;
334}
335
336static enum test_result test_unl(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
337
338    const char *key = "k2";
339    uint16_t vbucketId = 0;
340
341    unl(h, h1, key, vbucketId);
342    check(last_status != PROTOCOL_BINARY_RESPONSE_SUCCESS,
343          "expected the key to be missing...");
344
345    item *i = NULL;
346    check(store(h, h1, NULL, OPERATION_SET, key, "lockdata", &i, 0, vbucketId)
347          == ENGINE_SUCCESS, "Failed to store an item.");
348    h1->release(h, NULL, i);
349
350    /* getl, should succeed */
351    getl(h, h1, key, vbucketId, 0);
352    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
353          "Expected to be able to getl on first try");
354
355    /* save the returned cas value for later */
356    uint64_t cas = last_cas;
357
358    /* lock's taken unlocking with a random cas value should fail */
359    unl(h, h1, key, vbucketId);
360    check(last_status == PROTOCOL_BINARY_RESPONSE_ETMPFAIL,
361          "Expected to fail getl on second try");
362
363    if (last_body != NULL && (strcmp(last_body, "UNLOCK_ERROR") != 0)) {
364        fprintf(stderr, "Should have returned UNLOCK_ERROR. Unl Failed");
365        abort();
366    }
367
368    unl(h, h1, key, vbucketId, cas);
369    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
370          "Expected to succed unl with correct cas");
371
372    /* acquire lock, should succeed */
373    getl(h, h1, key, vbucketId, 0);
374
375    /* wait 16 seconds */
376    testHarness.time_travel(16);
377
378    /* lock has expired, unl should fail */
379    unl(h, h1, key, vbucketId, last_cas);
380    check(last_status == PROTOCOL_BINARY_RESPONSE_ETMPFAIL,
381          "Expected to fail unl on lock timeout");
382
383    return SUCCESS;
384}
385
386static enum test_result test_wrong_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
387                                               ENGINE_STORE_OPERATION op) {
388    item *i = NULL;
389    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
390    uint64_t cas = 11;
391    if (op == OPERATION_ADD) {
392        // Add operation with cas != 0 doesn't make sense
393        cas = 0;
394    }
395    check(store(h, h1, NULL, op,
396                "key", "somevalue", &i, cas, 1) == ENGINE_NOT_MY_VBUCKET,
397        "Expected not_my_vbucket");
398    h1->release(h, NULL, i);
399    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
400    return SUCCESS;
401}
402
403static enum test_result test_pending_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
404                                                 ENGINE_STORE_OPERATION op) {
405    const void *cookie = testHarness.create_cookie();
406    testHarness.set_ewouldblock_handling(cookie, false);
407    item *i = NULL;
408    check(set_vbucket_state(h, h1, 1, vbucket_state_pending), "Failed to set vbucket state.");
409    check(verify_vbucket_state(h, h1, 1, vbucket_state_pending), "Bucket state was not set to pending.");
410    uint64_t cas = 11;
411    if (op == OPERATION_ADD) {
412        // Add operation with cas != 0 doesn't make sense..
413        cas = 0;
414    }
415    check(store(h, h1, cookie, op,
416                "key", "somevalue", &i, cas, 1) == ENGINE_EWOULDBLOCK,
417        "Expected woodblock");
418    h1->release(h, NULL, i);
419    testHarness.destroy_cookie(cookie);
420    return SUCCESS;
421}
422
423static enum test_result test_replica_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
424                                                 ENGINE_STORE_OPERATION op) {
425    item *i = NULL;
426    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set vbucket state.");
427    check(verify_vbucket_state(h, h1, 1, vbucket_state_replica), "Bucket state was not set to replica.");
428    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
429
430    uint64_t cas = 11;
431    if (op == OPERATION_ADD) {
432        // performing add with a CAS != 0 doesn't make sense...
433        cas = 0;
434    }
435    check(store(h, h1, NULL, op,
436                "key", "somevalue", &i, cas, 1) == ENGINE_NOT_MY_VBUCKET,
437        "Expected not my vbucket");
438    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
439    h1->release(h, NULL, i);
440    return SUCCESS;
441}
442
443//
444// ----------------------------------------------------------------------
445// The actual tests are below.
446// ----------------------------------------------------------------------
447//
448
449static enum test_result test_get_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
450    check(verify_key(h, h1, "k") == ENGINE_KEY_ENOENT, "Expected miss.");
451    return SUCCESS;
452}
453
454static enum test_result test_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
455    item *i = NULL;
456    check(ENGINE_SUCCESS ==
457          store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
458          "Error setting.");
459    h1->release(h, NULL, i);
460    return SUCCESS;
461}
462
463struct handle_pair {
464    ENGINE_HANDLE *h;
465    ENGINE_HANDLE_V1 *h1;
466};
467
468extern "C" {
469    static void conc_del_set_thread(void *arg) {
470        struct handle_pair *hp = static_cast<handle_pair *>(arg);
471        item *it = NULL;
472
473        for (int i = 0; i < 5000; ++i) {
474            store(hp->h, hp->h1, NULL, OPERATION_ADD,
475                  "key", "somevalue", &it);
476            hp->h1->release(hp->h, NULL, it);
477            usleep(10);
478            checkeq(ENGINE_SUCCESS,
479                    store(hp->h, hp->h1, NULL, OPERATION_SET,
480                          "key", "somevalue", &it),
481                    "Error setting.");
482            hp->h1->release(hp->h, NULL, it);
483            usleep(10);
484            // Ignoring the result here -- we're racing.
485            del(hp->h, hp->h1, "key", 0, 0);
486            usleep(10);
487        }
488    }
489}
490
491static enum test_result test_conc_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
492
493    const int n_threads = 8;
494    cb_thread_t threads[n_threads];
495    struct handle_pair hp = {h, h1};
496
497    wait_for_persisted_value(h, h1, "key", "value1");
498
499    for (int i = 0; i < n_threads; i++) {
500        int r = cb_create_thread(&threads[i], conc_del_set_thread, &hp, 0);
501        cb_assert(r == 0);
502    }
503
504    for (int i = 0; i < n_threads; i++) {
505        int r = cb_join_thread(threads[i]);
506        cb_assert(r == 0);
507    }
508
509    wait_for_flusher_to_settle(h, h1);
510
511    testHarness.reload_engine(&h, &h1,
512                              testHarness.engine_path,
513                              testHarness.get_current_testcase()->cfg,
514                              true, false);
515    wait_for_warmup_complete(h, h1);
516
517    cb_assert(0 == get_int_stat(h, h1, "ep_warmed_dups"));
518
519    return SUCCESS;
520}
521
522static enum test_result test_set_get_hit(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
523    item *i = NULL;
524    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
525          "store failure");
526    check_key_value(h, h1, "key", "somevalue", 9);
527    h1->release(h, NULL, i);
528    return SUCCESS;
529}
530
531static enum test_result test_set_get_hit_bin(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
532    char binaryData[] = "abcdefg\0gfedcba";
533    cb_assert(sizeof(binaryData) != strlen(binaryData));
534
535    item *i = NULL;
536    check(ENGINE_SUCCESS ==
537          storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
538                       binaryData, sizeof(binaryData), 82758, &i, 0, 0),
539          "Failed to set.");
540    h1->release(h, NULL, i);
541    check_key_value(h, h1, "key", binaryData, sizeof(binaryData));
542    return SUCCESS;
543}
544
545static enum test_result test_set_with_cas_non_existent(ENGINE_HANDLE *h,
546                                                       ENGINE_HANDLE_V1 *h1) {
547    const char *key = "test_expiry_flush";
548    item *i = NULL;
549
550    check(h1->allocate(h, NULL, &i, key, strlen(key), 10, 0, 0,
551          PROTOCOL_BINARY_RAW_BYTES) == ENGINE_SUCCESS, "Allocation failed.");
552
553    Item *it = reinterpret_cast<Item*>(i);
554    it->setCas(1234);
555
556    uint64_t cas = 0;
557    check(h1->store(h, NULL, i, &cas, OPERATION_SET, 0) == ENGINE_KEY_ENOENT,
558          "Expected not found");
559    h1->release(h, NULL, i);
560
561    return SUCCESS;
562}
563
564static enum test_result test_set_change_flags(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
565    item *i = NULL;
566    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
567          "Failed to set.");
568    h1->release(h, NULL, i);
569
570    item_info info;
571    uint32_t flags = 828258;
572    check(get_item_info(h, h1, &info, "key"), "Failed to get value.");
573    cb_assert(info.flags != flags);
574
575    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
576                       "newvalue", strlen("newvalue"), flags, &i, 0, 0) == ENGINE_SUCCESS,
577          "Failed to set again.");
578    h1->release(h, NULL, i);
579
580    check(get_item_info(h, h1, &info, "key"), "Failed to get value.");
581
582    return info.flags == flags ? SUCCESS : FAIL;
583}
584
585static enum test_result test_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
586    item *i = NULL;
587    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
588          "Failed to do initial set.");
589    h1->release(h, NULL, i);
590    check(store(h, h1, NULL, OPERATION_CAS, "key", "failcas", &i) != ENGINE_SUCCESS,
591          "Failed to fail initial CAS.");
592    h1->release(h, NULL, i);
593    check_key_value(h, h1, "key", "somevalue", 9);
594
595    check(h1->get(h, NULL, &i, "key", 3, 0) == ENGINE_SUCCESS,
596          "Failed to get value.");
597
598    item_info info;
599    info.nvalue = 1;
600    check(h1->get_item_info(h, NULL, i, &info), "Failed to get item info.");
601    h1->release(h, NULL, i);
602
603    check(store(h, h1, NULL, OPERATION_CAS, "key", "winCas", &i,
604                info.cas) == ENGINE_SUCCESS,
605          "Failed to store CAS");
606    h1->release(h, NULL, i);
607    check_key_value(h, h1, "key", "winCas", 6);
608
609    uint64_t cval = 99999;
610    check(store(h, h1, NULL, OPERATION_CAS, "non-existing", "winCas", &i,
611                cval) == ENGINE_KEY_ENOENT,
612          "CAS for non-existing key returned the wrong error code");
613    h1->release(h, NULL, i);
614    return SUCCESS;
615}
616
617static enum test_result test_add(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
618    item *i = NULL;
619    check(store(h, h1, NULL, OPERATION_ADD,"key", "somevalue", &i) == ENGINE_SUCCESS,
620          "Failed to add value.");
621    h1->release(h, NULL, i);
622    check(store(h, h1, NULL, OPERATION_ADD,"key", "somevalue", &i) == ENGINE_NOT_STORED,
623          "Failed to fail to re-add value.");
624    h1->release(h, NULL, i);
625
626    // This aborts on failure.
627    check_key_value(h, h1, "key", "somevalue", 9);
628
629    // Expiration above was an hour, so let's go to The Future
630    testHarness.time_travel(3800);
631
632    check(store(h, h1, NULL, OPERATION_ADD,"key", "newvalue", &i) == ENGINE_SUCCESS,
633          "Failed to add value again.");
634    h1->release(h, NULL, i);
635    check_key_value(h, h1, "key", "newvalue", 8);
636    return SUCCESS;
637}
638
639static enum test_result test_add_add_with_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
640    item *i = NULL;
641    check(store(h, h1, NULL, OPERATION_ADD, "key",
642                "somevalue", &i) == ENGINE_SUCCESS,
643          "Failed set.");
644    check_key_value(h, h1, "key", "somevalue", 9);
645    item_info info;
646    info.nvalue = 1;
647    info.nvalue = 1;
648    check(h1->get_item_info(h, NULL, i, &info) == true,
649          "Should be able to get info");
650
651    item *i2 = NULL;
652    ENGINE_ERROR_CODE ret;
653    check((ret = store(h, h1, NULL, OPERATION_ADD, "key",
654                       "somevalue", &i2, info.cas)) == ENGINE_KEY_EEXISTS,
655          "Should not be able to add the key two times");
656
657    h1->release(h, NULL, i);
658    h1->release(h, NULL, i2);
659    return SUCCESS;
660}
661
662static enum test_result test_replace(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
663    item *i = NULL;
664    check(store(h, h1, NULL, OPERATION_REPLACE,"key", "somevalue", &i) != ENGINE_SUCCESS,
665          "Failed to fail to replace non-existing value.");
666    h1->release(h, NULL, i);
667    check(store(h, h1, NULL, OPERATION_SET,"key", "somevalue", &i) == ENGINE_SUCCESS,
668          "Failed to set value.");
669    h1->release(h, NULL, i);
670    check(store(h, h1, NULL, OPERATION_REPLACE,"key", "somevalue", &i) == ENGINE_SUCCESS,
671          "Failed to replace existing value.");
672    h1->release(h, NULL, i);
673    check_key_value(h, h1, "key", "somevalue", 9);
674    return SUCCESS;
675}
676
677static enum test_result test_replace_with_eviction(ENGINE_HANDLE *h,
678                                                   ENGINE_HANDLE_V1 *h1) {
679    item *i = NULL;
680    check(store(h, h1, NULL, OPERATION_SET,"key", "somevalue", &i) == ENGINE_SUCCESS,
681          "Failed to set value.");
682    h1->release(h, NULL, i);
683    wait_for_flusher_to_settle(h, h1);
684    evict_key(h, h1, "key");
685    int numBgFetched = get_int_stat(h, h1, "ep_bg_fetched");
686
687    check(store(h, h1, NULL, OPERATION_REPLACE,"key", "somevalue1", &i) == ENGINE_SUCCESS,
688          "Failed to replace existing value.");
689
690    check(h1->get_stats(h, NULL, NULL, 0, add_stats) == ENGINE_SUCCESS,
691          "Failed to get stats.");
692    std::string eviction_policy = vals.find("ep_item_eviction_policy")->second;
693    if (eviction_policy == "full_eviction") {
694        numBgFetched++;
695    }
696
697    check(get_int_stat(h, h1, "ep_bg_fetched") == numBgFetched,
698          "Bg fetched value didn't match");
699
700    h1->release(h, NULL, i);
701    check_key_value(h, h1, "key", "somevalue1", 10);
702    return SUCCESS;
703}
704
705static enum test_result test_incr_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
706    uint64_t cas = 0, result = 0;
707    h1->arithmetic(h, NULL, "key", 3, true, false, 1, 0, 0,
708                   &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
709                   0);
710    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected to not find key");
711    return SUCCESS;
712}
713
714static enum test_result test_incr_default(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
715    uint64_t cas = 0, result = 0;
716    check(h1->arithmetic(h, NULL, "key", 3, true, true, 1, 1, 0,
717                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
718                         0) == ENGINE_SUCCESS,
719          "Failed first arith");
720    check(result == 1, "Failed result verification.");
721
722    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
723                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
724                         0) == ENGINE_SUCCESS,
725          "Failed second arith.");
726    check(result == 2, "Failed second result verification.");
727
728    check(h1->arithmetic(h, NULL, "key", 3, true, true, 1, 1, 0,
729                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
730                         0) == ENGINE_SUCCESS,
731          "Failed third arith.");
732    check(result == 3, "Failed third result verification.");
733
734    check_key_value(h, h1, "key", "3", 1);
735    return SUCCESS;
736}
737
738static enum test_result test_append(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
739    item *i = NULL;
740
741    // MB-11332: append on non-existing key should return NOT_STORED
742    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key",
743                       "foo\r\n", 5, 82758, &i, 0, 0)
744          == ENGINE_NOT_STORED,
745          "MB-11332: Failed append.");
746
747    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
748                       "\r\n", 2, 82758, &i, 0, 0)
749          == ENGINE_SUCCESS,
750          "Failed set.");
751    h1->release(h, NULL, i);
752
753    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key",
754                       "foo\r\n", 5, 82758, &i, 0, 0)
755          == ENGINE_SUCCESS,
756          "Failed append.");
757    h1->release(h, NULL, i);
758
759    check_key_value(h, h1, "key", "\r\nfoo\r\n", 7);
760
761    char binaryData1[] = "abcdefg\0gfedcba\r\n";
762    char binaryData2[] = "abzdefg\0gfedcba\r\n";
763    size_t dataSize = 20*1024*1024;
764    char *bigBinaryData3 = new char[dataSize];
765    memset(bigBinaryData3, '\0', dataSize);
766
767    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
768                       binaryData1, sizeof(binaryData1) - 1, 82758, &i, 0, 0)
769          == ENGINE_SUCCESS,
770          "Failed set.");
771    h1->release(h, NULL, i);
772
773    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key",
774                       bigBinaryData3, dataSize, 82758, &i, 0, 0)
775          == ENGINE_E2BIG,
776          "Expected append failure.");
777    h1->release(h, NULL, i);
778    delete[] bigBinaryData3;
779
780    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key",
781                       binaryData2, sizeof(binaryData2) - 1, 82758, &i, 0, 0)
782          == ENGINE_SUCCESS,
783          "Failed append.");
784    h1->release(h, NULL, i);
785
786    std::string expected;
787    expected.append(binaryData1, sizeof(binaryData1) - 1);
788    expected.append(binaryData2, sizeof(binaryData2) - 1);
789
790    check_key_value(h, h1, "key", expected.data(), expected.length());
791    return SUCCESS;
792}
793
794static enum test_result test_prepend(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
795    item *i = NULL;
796
797    // MB-11332: prepend on non-existing key should return NOT_STORED
798    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key",
799                       "foo\r\n", 5, 82758, &i, 0, 0)
800          == ENGINE_NOT_STORED,
801          "MB-11332: Failed prepend.");
802
803
804    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
805                       "\r\n", 2, 82758, &i, 0, 0)
806          == ENGINE_SUCCESS,
807          "Failed set.");
808    h1->release(h, NULL, i);
809
810    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key",
811                       "foo\r\n", 5, 82758, &i, 0, 0)
812          == ENGINE_SUCCESS,
813          "Failed append.");
814    h1->release(h, NULL, i);
815
816    check_key_value(h, h1, "key", "foo\r\n\r\n", 7);
817
818    char binaryData1[] = "abcdefg\0gfedcba\r\n";
819    char binaryData2[] = "abzdefg\0gfedcba\r\n";
820    size_t dataSize = 20*1024*1024;
821    char *bigBinaryData3 = new char[dataSize];
822    memset(bigBinaryData3, '\0', dataSize);
823
824    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
825                       binaryData1, sizeof(binaryData1) - 1, 82758, &i, 0, 0)
826          == ENGINE_SUCCESS,
827          "Failed set.");
828    h1->release(h, NULL, i);
829
830    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key",
831                       bigBinaryData3, dataSize, 82758, &i, 0, 0)
832          == ENGINE_E2BIG,
833          "Expected prepend failure.");
834    h1->release(h, NULL, i);
835    delete[] bigBinaryData3;
836
837    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key",
838                       binaryData2, sizeof(binaryData2) - 1, 82758, &i, 0, 0)
839          == ENGINE_SUCCESS,
840          "Failed append.");
841    h1->release(h, NULL, i);
842
843    std::string expected;
844    expected.append(binaryData2, sizeof(binaryData2) - 1);
845    expected.append(binaryData1, sizeof(binaryData1) - 1);
846
847    check_key_value(h, h1, "key", expected.data(), expected.length());
848    return SUCCESS;
849}
850
851static enum test_result test_append_compressed(ENGINE_HANDLE *h,
852                                               ENGINE_HANDLE_V1 *h1) {
853
854    item *i = NULL;
855
856    size_t len = snappy_max_compressed_length(2);
857    char *newBuf = (char *) malloc (len);
858    snappy_compress("\r\n", 2, newBuf, &len);
859    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key1",
860                       (const char *)newBuf, len, 82758, &i, 0, 0, 3600,
861                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
862          == ENGINE_SUCCESS, "Failed set.");
863    h1->release(h, NULL, i);
864    free (newBuf);
865
866    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key1",
867                       "foo\r\n", 5, 82758, &i, 0, 0)
868          == ENGINE_SUCCESS,
869          "Failed append uncompressed to compressed.");
870    h1->release(h, NULL, i);
871
872    size_t len1 = snappy_max_compressed_length(7);
873    char *newBuf1 = (char *) malloc (len1);
874    snappy_compress("\r\nfoo\r\n", 7, newBuf1, &len1);
875
876    item_info info;
877    check(get_item_info(h, h1, &info, "key1", 0), "checking key and value");
878    check(info.nvalue == 1, "info.nvalue != 1");
879    check(len1 == info.value[0].iov_len, "Value length mismatch");
880    check(memcmp(info.value[0].iov_base, newBuf1, len1) == 0, "Data mismatch");
881    check(info.datatype == 0x02, "Datatype mismatch");
882    free (newBuf1);
883
884    len = snappy_max_compressed_length(3);
885    newBuf = (char *) malloc (len);
886    snappy_compress("bar", 3, newBuf, &len);
887    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key1",
888                       (const char*)newBuf, len, 82758, &i, 0, 0, 3600,
889                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
890            == ENGINE_SUCCESS,
891            "Failed append compressed to compressed.");
892    h1->release(h, NULL, i);
893    free (newBuf);
894
895    len1 = snappy_max_compressed_length(10);
896    newBuf1 = (char *) malloc (len1);
897    snappy_compress("\r\nfoo\r\nbar", 10, newBuf1, &len1);
898
899    check(get_item_info(h, h1, &info, "key1", 0), "checking key and value");
900    check(info.nvalue == 1, "info.nvalue != 1");
901    check(len1 == info.value[0].iov_len, "Value length mismatch");
902    check(memcmp(info.value[0].iov_base, newBuf1, len1) == 0, "Data mismatch");
903    check(info.datatype == 0x02, "Datatype mismatch");
904    free (newBuf1);
905
906    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key2",
907                       "foo", 3, 82758, &i, 0, 0, 3600,
908                       PROTOCOL_BINARY_RAW_BYTES)
909          == ENGINE_SUCCESS, "Failed set.");
910    h1->release(h, NULL, i);
911
912    len = snappy_max_compressed_length(3);
913    newBuf = (char *) malloc (len);
914    snappy_compress("bar", 3, newBuf, &len);
915    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key2",
916                       newBuf, len, 82758, &i, 0, 0, 3600,
917                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
918            == ENGINE_SUCCESS,
919            "Failed append compressed to uncompressed.");
920    h1->release(h, NULL, i);
921    free (newBuf);
922
923    check(get_item_info(h, h1, &info, "key2", 0), "checking key and value");
924    check(info.nvalue == 1, "info.nvalue != 1");
925    check(info.value[0].iov_len == 6, "Value length mismatch");
926    check(memcmp(info.value[0].iov_base, "foobar", 6) == 0, "Data mismatch");
927    check(info.datatype == 0x00, "Datatype mismatch");
928
929    return SUCCESS;
930}
931
932static enum test_result test_prepend_compressed(ENGINE_HANDLE *h,
933                                               ENGINE_HANDLE_V1 *h1) {
934
935    item *i = NULL;
936
937    size_t len = snappy_max_compressed_length(2);
938    char *newBuf = (char *) malloc (len);
939    snappy_compress("\r\n", 2, newBuf, &len);
940    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key1",
941                       (const char *)newBuf, len, 82758, &i, 0, 0, 3600,
942                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
943          == ENGINE_SUCCESS, "Failed set.");
944    h1->release(h, NULL, i);
945    free (newBuf);
946
947    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key1",
948                       "foo\r\n", 5, 82758, &i, 0, 0)
949          == ENGINE_SUCCESS,
950          "Failed prepend uncompressed to compressed.");
951    h1->release(h, NULL, i);
952
953    size_t len1 = snappy_max_compressed_length(7);
954    char *newBuf1 = (char *) malloc (len1);
955    snappy_compress("foo\r\n\r\n", 7, newBuf1, &len1);
956
957    item_info info;
958    check(get_item_info(h, h1, &info, "key1", 0), "checking key and value");
959    check(info.nvalue == 1, "info.nvalue != 1");
960    check(len1 == info.value[0].iov_len, "Value length mismatch");
961    check(memcmp(info.value[0].iov_base, newBuf1, len1) == 0, "Data mismatch");
962    check(info.datatype == 0x02, "Datatype mismatch");
963    free (newBuf1);
964
965    len = snappy_max_compressed_length(3);
966    newBuf = (char *) malloc (len);
967    snappy_compress("bar", 3, newBuf, &len);
968    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key1",
969                       (const char*)newBuf, len, 82758, &i, 0, 0, 3600,
970                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
971            == ENGINE_SUCCESS,
972            "Failed prepend compressed to compressed.");
973    h1->release(h, NULL, i);
974    free (newBuf);
975
976    len1 = snappy_max_compressed_length(10);
977    newBuf1 = (char *) malloc (len1);
978    snappy_compress("barfoo\r\n\r\n", 10, newBuf1, &len1);
979
980    check(get_item_info(h, h1, &info, "key1", 0), "checking key and value");
981    check(info.nvalue == 1, "info.nvalue != 1");
982    check(len1 == info.value[0].iov_len, "Value length mismatch");
983    check(memcmp(info.value[0].iov_base, newBuf1, len1) == 0, "Data mismatch");
984    check(info.datatype == 0x02, "Datatype mismatch");
985    free (newBuf1);
986
987    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key2",
988                       "foo", 3, 82758, &i, 0, 0, 3600,
989                       PROTOCOL_BINARY_RAW_BYTES)
990          == ENGINE_SUCCESS, "Failed set.");
991    h1->release(h, NULL, i);
992
993    len = snappy_max_compressed_length(3);
994    newBuf = (char *) malloc (len);
995    snappy_compress("bar", 3, newBuf, &len);
996    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key2",
997                       newBuf, len, 82758, &i, 0, 0, 3600,
998                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
999            == ENGINE_SUCCESS,
1000            "Failed prepend compressed to uncompressed.");
1001    h1->release(h, NULL, i);
1002    free (newBuf);
1003
1004    check(get_item_info(h, h1, &info, "key2", 0), "checking key and value");
1005    check(info.nvalue == 1, "info.nvalue != 1");
1006    check(info.value[0].iov_len == 6, "Value length mismatch");
1007    check(memcmp(info.value[0].iov_base, "barfoo", 6) == 0, "Data mismatch");
1008    check(info.datatype == 0x00, "Datatype mismatch");
1009
1010    return SUCCESS;
1011}
1012
1013static enum test_result test_append_prepend_to_json(ENGINE_HANDLE *h,
1014                                                    ENGINE_HANDLE_V1 *h1) {
1015    item *i = NULL;
1016    item_info info;
1017
1018    const char* key1 = "foo1";
1019    const char* key2 = "foo2";
1020    const char* value1 = "{\"foo1\":\"bar1\"}";
1021    const char* value2 = "{\"foo2\":\"bar2\"}";
1022
1023    // APPEND
1024    check(storeCasVb11(h, h1, NULL, OPERATION_SET, key1,
1025                       value1, strlen(value1), 82758, &i, 0, 0,
1026                       3600, PROTOCOL_BINARY_DATATYPE_JSON)
1027          == ENGINE_SUCCESS, "Failed set.");
1028    h1->release(h, NULL, i);
1029
1030    check(h1->get(h, NULL, &i, key1, strlen(key1), 0) == ENGINE_SUCCESS,
1031            "Unable to get stored item");
1032    h1->release(h, NULL, i);
1033    info.nvalue = 1;
1034    h1->get_item_info(h, NULL, i, &info);
1035    check(checkUTF8JSON((const unsigned char*)info.value[0].iov_base,
1036                        (int)info.value[0].iov_len) == 1, "Expected JSON");
1037    check(info.datatype == PROTOCOL_BINARY_DATATYPE_JSON, "Invalid datatype");
1038
1039    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, key1,
1040                       value2, strlen(value2), 82758, &i, 0, 0)
1041          == ENGINE_SUCCESS,
1042          "Failed append.");
1043    h1->release(h, NULL, i);
1044
1045    check(h1->get(h, NULL, &i, key1, strlen(key1), 0) == ENGINE_SUCCESS,
1046            "Unable to get stored item");
1047    info.nvalue = 1;
1048    h1->get_item_info(h, NULL, i, &info);
1049    h1->release(h, NULL, i);
1050    check(checkUTF8JSON((const unsigned char*)info.value[0].iov_base,
1051                        (int)info.value[0].iov_len) == 0, "Expected Binary");
1052    check(info.datatype == PROTOCOL_BINARY_RAW_BYTES,
1053                "Invalid datatype after append");
1054
1055    // PREPEND
1056    check(storeCasVb11(h, h1, NULL, OPERATION_SET, key2,
1057                       value1, strlen(value1), 82758, &i, 0, 0,
1058                       3600, PROTOCOL_BINARY_DATATYPE_JSON)
1059          == ENGINE_SUCCESS, "Failed set.");
1060    h1->release(h, NULL, i);
1061
1062    check(h1->get(h, NULL, &i, key2, strlen(key2), 0) == ENGINE_SUCCESS,
1063            "Unable to get stored item");
1064    info.nvalue = 1;
1065    h1->get_item_info(h, NULL, i, &info);
1066    h1->release(h, NULL, i);
1067    check(checkUTF8JSON((const unsigned char*)info.value[0].iov_base,
1068                        (int)info.value[0].iov_len) == 1, "Expected JSON");
1069    check(info.datatype == PROTOCOL_BINARY_DATATYPE_JSON, "Invalid datatype");
1070
1071    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, key2,
1072                       value2, strlen(value2), 82758, &i, 0, 0)
1073          == ENGINE_SUCCESS,
1074          "Failed prepend.");
1075    h1->release(h, NULL, i);
1076
1077    check(h1->get(h, NULL, &i, key2, strlen(key2), 0) == ENGINE_SUCCESS,
1078            "Unable to get stored item");
1079    h1->release(h, NULL, i);
1080    info.nvalue = 1;
1081    h1->get_item_info(h, NULL, i, &info);
1082    check(checkUTF8JSON((const unsigned char*)info.value[0].iov_base,
1083                        (int)info.value[0].iov_len) == 0, "Expected Binary");
1084    check(info.datatype == PROTOCOL_BINARY_RAW_BYTES,
1085                "Invalid datatype after prepend");
1086
1087    return SUCCESS;
1088}
1089
1090static enum test_result test_incr(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1091    uint64_t cas = 0, result = 0;
1092    item *i = NULL;
1093    check(store(h, h1, NULL, OPERATION_ADD,"key", "1", &i) == ENGINE_SUCCESS,
1094          "Failed to add value.");
1095    h1->release(h, NULL, i);
1096
1097    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
1098                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1099                         0) == ENGINE_SUCCESS,
1100          "Failed to incr value.");
1101
1102    check_key_value(h, h1, "key", "2", 1);
1103    return SUCCESS;
1104}
1105
1106static enum test_result test_bug2799(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1107    uint64_t cas = 0, result = 0;
1108    item *i = NULL;
1109    check(store(h, h1, NULL, OPERATION_ADD, "key", "1", &i) == ENGINE_SUCCESS,
1110          "Failed to add value.");
1111    h1->release(h, NULL, i);
1112
1113    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
1114                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1115                         0) == ENGINE_SUCCESS,
1116          "Failed to incr value.");
1117
1118    check_key_value(h, h1, "key", "2", 1);
1119
1120    testHarness.time_travel(3617);
1121
1122    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1123    return SUCCESS;
1124}
1125
1126static enum test_result test_flush(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1127    item *i = NULL;
1128    // First try to delete something we know to not be there.
1129    check(del(h, h1, "key", 0, 0) == ENGINE_KEY_ENOENT, "Failed to fail initial delete.");
1130    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1131          "Failed set.");
1132    h1->release(h, NULL, i);
1133    check_key_value(h, h1, "key", "somevalue", 9);
1134    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS,
1135          "Failed to flush");
1136    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1137
1138    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1139          "Failed post-flush set.");
1140    h1->release(h, NULL, i);
1141    check_key_value(h, h1, "key", "somevalue", 9);
1142
1143    return SUCCESS;
1144}
1145
1146static enum test_result test_flush_disabled(ENGINE_HANDLE *h,
1147                                            ENGINE_HANDLE_V1 *h1) {
1148    item *i = NULL;
1149    // start an engine with disabled flush, the flush() should be noop and
1150    // we expect to see the key after flush()
1151
1152    // store a key and check its existence
1153    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1154          "Failed set.");
1155    h1->release(h, NULL, i);
1156    check_key_value(h, h1, "key", "somevalue", 9);
1157    // expect error msg engine does not support operation
1158    check(h1->flush(h, NULL, 0) == ENGINE_ENOTSUP, "Flush should be disabled");
1159    //check the key
1160    check(ENGINE_SUCCESS == verify_key(h, h1, "key"), "Expected key");
1161
1162    // restart engine with flush enabled and redo the test, we expect flush to succeed
1163    std::string param = "flushall_enabled=false";
1164    std::string config = testHarness.get_current_testcase()->cfg;
1165    size_t found = config.find(param);
1166    if(found != config.npos) {
1167        config.replace(found, param.size(), "flushall_enabled=true");
1168    }
1169    testHarness.reload_engine(&h, &h1,
1170                              testHarness.engine_path,
1171                              config.c_str(),
1172                              true, false);
1173    wait_for_warmup_complete(h, h1);
1174
1175    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS, "Flush should be enabled");
1176
1177    //expect missing key
1178    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1179
1180    return SUCCESS;
1181}
1182
1183static enum test_result test_flush_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1184    item *i = NULL;
1185    int mem_used = get_int_stat(h, h1, "mem_used");
1186    int overhead = get_int_stat(h, h1, "ep_overhead");
1187    int cacheSize = get_int_stat(h, h1, "ep_total_cache_size");
1188    int nonResident = get_int_stat(h, h1, "ep_num_non_resident");
1189
1190    int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
1191    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1192          "Failed set.");
1193    h1->release(h, NULL, i);
1194    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i) == ENGINE_SUCCESS,
1195          "Failed set.");
1196    h1->release(h, NULL, i);
1197    testHarness.time_travel(65);
1198    wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
1199
1200    check(ENGINE_SUCCESS == verify_key(h, h1, "key"), "Expected key");
1201    check(ENGINE_SUCCESS == verify_key(h, h1, "key2"), "Expected key2");
1202
1203    check_key_value(h, h1, "key", "somevalue", 9);
1204    check_key_value(h, h1, "key2", "somevalue", 9);
1205
1206    int mem_used2 = get_int_stat(h, h1, "mem_used");
1207    int overhead2 = get_int_stat(h, h1, "ep_overhead");
1208    int cacheSize2 = get_int_stat(h, h1, "ep_total_cache_size");
1209
1210    cb_assert(mem_used2 > mem_used);
1211    // "mem_used2 - overhead2" (i.e., ep_kv_size) should be greater than the hashtable cache size
1212    // due to the checkpoint overhead
1213    cb_assert(mem_used2 - overhead2 > cacheSize2);
1214
1215    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS, "Failed to flush");
1216    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1217    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key2"), "Expected missing key");
1218
1219    wait_for_flusher_to_settle(h, h1);
1220
1221    mem_used2 = get_int_stat(h, h1, "mem_used");
1222    overhead2 = get_int_stat(h, h1, "ep_overhead");
1223    cacheSize2 = get_int_stat(h, h1, "ep_total_cache_size");
1224    int nonResident2 = get_int_stat(h, h1, "ep_num_non_resident");
1225
1226    cb_assert(mem_used2 == mem_used);
1227    cb_assert(overhead2 == overhead);
1228    cb_assert(nonResident2 == nonResident);
1229    cb_assert(cacheSize2 == cacheSize);
1230
1231    return SUCCESS;
1232}
1233
1234static enum test_result test_flush_multiv(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1235    item *i = NULL;
1236    check(set_vbucket_state(h, h1, 2, vbucket_state_active), "Failed to set vbucket state.");
1237    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1238          "Failed set.");
1239    h1->release(h, NULL, i);
1240    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i,
1241                0, 2) == ENGINE_SUCCESS,
1242          "Failed set in vb2.");
1243    h1->release(h, NULL, i);
1244
1245    check(ENGINE_SUCCESS == verify_key(h, h1, "key"), "Expected key");
1246    check(ENGINE_SUCCESS == verify_key(h, h1, "key2", 2), "Expected key2");
1247
1248    check_key_value(h, h1, "key", "somevalue", 9);
1249    check_key_value(h, h1, "key2", "somevalue", 9, 2);
1250
1251    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS, "Failed to flush");
1252
1253    vals.clear();
1254    check(h1->get_stats(h, NULL, NULL, 0, add_stats) == ENGINE_SUCCESS,
1255          "Failed to get stats.");
1256    check(vals.find("ep_flush_all") != vals.end(), "Failed to get the status of flush_all");
1257
1258    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1259    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key2", 2), "Expected missing key");
1260
1261    return SUCCESS;
1262}
1263
1264static int checkCurrItemsAfterShutdown(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1265                                       int numItems2Load, bool shutdownForce) {
1266    std::vector<std::string> keys;
1267    for (int index = 0; index < numItems2Load; ++index) {
1268        std::stringstream s;
1269        s << "keys_2_load-" << index;
1270        std::string key(s.str());
1271        keys.push_back(key);
1272    }
1273
1274    check(get_int_stat(h, h1, "ep_total_persisted") == 0,
1275          "Expected ep_total_persisted equals 0");
1276    check(get_int_stat(h, h1, "curr_items") == 0,
1277          "Expected curr_items equals 0");
1278
1279    // stop flusher before loading new items
1280    protocol_binary_request_header *pkt = createPacket(PROTOCOL_BINARY_CMD_STOP_PERSISTENCE);
1281    check(h1->unknown_command(h, NULL, pkt, add_response) == ENGINE_SUCCESS,
1282          "CMD_STOP_PERSISTENCE failed!");
1283    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
1284          "Failed to stop persistence!");
1285    free(pkt);
1286
1287    std::vector<std::string>::iterator itr;
1288    for (itr = keys.begin(); itr != keys.end(); ++itr) {
1289        item *i;
1290        check(store(h, h1, NULL, OPERATION_SET, itr->c_str(), "oracle", &i, 0, 0)
1291              == ENGINE_SUCCESS, "Failed to store a value");
1292        h1->release(h, NULL, i);
1293    }
1294
1295    check(get_int_stat(h, h1, "ep_total_persisted") == 0,
1296          "Incorrect ep_total_persisted, expected 0");
1297    std::stringstream ss;
1298    ss << "Incorrect curr_items, expected " << numItems2Load;
1299    std::string errmsg(ss.str());
1300    check(get_int_stat(h, h1, "curr_items") == numItems2Load,
1301          errmsg.c_str());
1302
1303    // resume flusher before shutdown + warmup
1304    pkt = createPacket(PROTOCOL_BINARY_CMD_START_PERSISTENCE);
1305    check(h1->unknown_command(h, NULL, pkt, add_response) == ENGINE_SUCCESS,
1306          "CMD_START_PERSISTENCE failed!");
1307    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
1308          "Failed to start persistence!");
1309    free(pkt);
1310
1311    // shutdown engine force and restart
1312    testHarness.reload_engine(&h, &h1,
1313                              testHarness.engine_path,
1314                              testHarness.get_current_testcase()->cfg,
1315                              true, shutdownForce);
1316    wait_for_warmup_complete(h, h1);
1317    return get_int_stat(h, h1, "curr_items");
1318}
1319
1320static enum test_result test_flush_shutdown_force(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1321    int numItems2load = 3000;
1322    bool shutdownForce = true;
1323    int currItems = checkCurrItemsAfterShutdown(h, h1, numItems2load, shutdownForce);
1324    check (currItems <= numItems2load,
1325           "Number of curr items should be <= 3000, unless previous "
1326           "shutdown force had to wait for the flusher");
1327    return SUCCESS;
1328}
1329
1330static enum test_result test_flush_shutdown_noforce(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1331    int numItems2load = 3000;
1332    bool shutdownForce = false;
1333    int currItems = checkCurrItemsAfterShutdown(h, h1, numItems2load, shutdownForce);
1334    check (currItems == numItems2load,
1335           "Number of curr items should be equal to 3000, unless previous "
1336           "shutdown did not wait for the flusher");
1337    return SUCCESS;
1338}
1339
1340static enum test_result test_flush_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1341    item *i = NULL;
1342    // First try to delete something we know to not be there.
1343    check(del(h, h1, "key", 0, 0) == ENGINE_KEY_ENOENT, "Failed to fail initial delete.");
1344    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1345          "Failed set.");
1346    h1->release(h, NULL, i);
1347    check_key_value(h, h1, "key", "somevalue", 9);
1348
1349    // Restart once to ensure written to disk.
1350    testHarness.reload_engine(&h, &h1,
1351                              testHarness.engine_path,
1352                              testHarness.get_current_testcase()->cfg,
1353                              true, false);
1354    wait_for_warmup_complete(h, h1);
1355
1356    // Read value from disk.
1357    check_key_value(h, h1, "key", "somevalue", 9);
1358
1359    // Flush
1360    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS,
1361          "Failed to flush");
1362
1363    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i) == ENGINE_SUCCESS,
1364          "Failed post-flush set.");
1365    h1->release(h, NULL, i);
1366    check_key_value(h, h1, "key2", "somevalue", 9);
1367
1368    // Restart again, ensure written to disk.
1369    testHarness.reload_engine(&h, &h1,
1370                              testHarness.engine_path,
1371                              testHarness.get_current_testcase()->cfg,
1372                              true, false);
1373    wait_for_warmup_complete(h, h1);
1374
1375    check(store(h, h1, NULL, OPERATION_SET, "key3", "somevalue", &i) == ENGINE_SUCCESS,
1376          "Failed post-flush, post-restart set.");
1377    h1->release(h, NULL, i);
1378    check_key_value(h, h1, "key3", "somevalue", 9);
1379
1380    // Read value again, should not be there.
1381    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1382    return SUCCESS;
1383}
1384
1385static enum test_result test_flush_multiv_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1386    item *i = NULL;
1387    check(set_vbucket_state(h, h1, 2, vbucket_state_active), "Failed to set vbucket state.");
1388    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1389          "Failed set.");
1390    h1->release(h, NULL, i);
1391    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i,
1392                0, 2) == ENGINE_SUCCESS,
1393          "Failed set in vb2.");
1394    h1->release(h, NULL, i);
1395
1396    // Restart once to ensure written to disk.
1397    testHarness.reload_engine(&h, &h1,
1398                              testHarness.engine_path,
1399                              testHarness.get_current_testcase()->cfg,
1400                              true, false);
1401    wait_for_warmup_complete(h, h1);
1402
1403    // Read value from disk.
1404    check_key_value(h, h1, "key", "somevalue", 9);
1405
1406    // Flush
1407    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS,
1408          "Failed to flush");
1409
1410    // Restart again, ensure written to disk.
1411    testHarness.reload_engine(&h, &h1,
1412                              testHarness.engine_path,
1413                              testHarness.get_current_testcase()->cfg,
1414                              true, false);
1415    wait_for_warmup_complete(h, h1);
1416
1417    // Read value again, should not be there.
1418    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1419    check(verify_vbucket_missing(h, h1, 2), "Bucket 2 came back.");
1420    return SUCCESS;
1421}
1422
1423static enum test_result test_delete(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1424    item *i = NULL;
1425    // First try to delete something we know to not be there.
1426    check(del(h, h1, "key", 0, 0) == ENGINE_KEY_ENOENT, "Failed to fail initial delete.");
1427    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1428          "Failed set.");
1429    Item *it = reinterpret_cast<Item*>(i);
1430    uint64_t orig_cas = it->getCas();
1431    h1->release(h, NULL, i);
1432    check_key_value(h, h1, "key", "somevalue", 9);
1433
1434    uint64_t cas = 0;
1435    check(h1->remove(h, NULL, "key", 3, &cas, 0) == ENGINE_SUCCESS,
1436          "Failed remove with value.");
1437    check(orig_cas + 1 == cas, "Cas mismatch on delete");
1438    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1439
1440    // Can I time travel to an expired object and delete it?
1441    checkeq(ENGINE_SUCCESS, store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1442            "Failed set.");
1443    h1->release(h, NULL, i);
1444    testHarness.time_travel(3617);
1445    checkeq(ENGINE_KEY_ENOENT, del(h, h1, "key", 0, 0),
1446            "Did not get ENOENT removing an expired object.");
1447    checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"), "Expected missing key");
1448
1449    return SUCCESS;
1450}
1451
1452static enum test_result test_set_delete(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1453    item *i = NULL;
1454    checkeq(ENGINE_SUCCESS, store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1455            "Failed set.");
1456    h1->release(h, NULL, i);
1457    check_key_value(h, h1, "key", "somevalue", 9);
1458    checkeq(ENGINE_SUCCESS, del(h, h1, "key", 0, 0),
1459            "Failed remove with value.");
1460    checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"), "Expected missing key");
1461    wait_for_flusher_to_settle(h, h1);
1462    wait_for_stat_to_be(h, h1, "curr_items", 0);
1463    return SUCCESS;
1464}
1465
1466static enum test_result test_set_delete_invalid_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1467    item *i = NULL;
1468    check(store(h, h1, NULL, OPERATION_SET, "key",
1469                "somevalue", &i) == ENGINE_SUCCESS,
1470          "Failed set.");
1471    check_key_value(h, h1, "key", "somevalue", 9);
1472    item_info info;
1473    info.nvalue = 1;
1474    check(h1->get_item_info(h, NULL, i, &info) == true,
1475          "Should be able to get info");
1476    h1->release(h, NULL, i);
1477
1478    check(del(h, h1, "key", info.cas + 1, 0) == ENGINE_KEY_EEXISTS,
1479          "Didn't expect to be able to remove the item with wrong cas");
1480    return SUCCESS;
1481}
1482
1483static enum test_result test_bug2509(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1484    for (int j = 0; j < 10000; ++j) {
1485        item *itm = NULL;
1486        checkeq(ENGINE_SUCCESS,
1487                store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &itm),
1488                "Failed set.");
1489        h1->release(h, NULL, itm);
1490        usleep(10);
1491        checkeq(ENGINE_SUCCESS, del(h, h1, "key", 0, 0), "Failed remove with value.");
1492        usleep(10);
1493    }
1494
1495    // Restart again, to verify we don't have any duplicates.
1496    testHarness.reload_engine(&h, &h1,
1497                              testHarness.engine_path,
1498                              testHarness.get_current_testcase()->cfg,
1499                              true, false);
1500    wait_for_warmup_complete(h, h1);
1501
1502    return get_int_stat(h, h1, "ep_warmup_dups") == 0 ? SUCCESS : FAIL;
1503}
1504
1505static enum test_result test_bug7023(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1506    std::vector<std::string> keys;
1507    // Make a vbucket mess.
1508    for (int j = 0; j < 10000; ++j) {
1509        std::stringstream ss;
1510        ss << "key" << j;
1511        std::string key(ss.str());
1512        keys.push_back(key);
1513    }
1514
1515    std::vector<std::string>::iterator it;
1516    for (int j = 0; j < 5; ++j) {
1517        check(set_vbucket_state(h, h1, 0, vbucket_state_dead), "Failed set set vbucket 0 dead.");
1518        vbucketDelete(h, h1, 0);
1519        check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
1520              "Expected vbucket deletion to work.");
1521        check(set_vbucket_state(h, h1, 0, vbucket_state_active), "Failed set set vbucket 0 active.");
1522        for (it = keys.begin(); it != keys.end(); ++it) {
1523            item *i;
1524            check(store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str(), &i)
1525                  == ENGINE_SUCCESS, "Failed to store a value");
1526        }
1527    }
1528    wait_for_flusher_to_settle(h, h1);
1529
1530    // Restart again, to verify no data loss.
1531    testHarness.reload_engine(&h, &h1,
1532                              testHarness.engine_path,
1533                              testHarness.get_current_testcase()->cfg,
1534                              true, false);
1535    wait_for_warmup_complete(h, h1);
1536    return get_int_stat(h, h1, "ep_warmup_value_count", "warmup") == 10000 ? SUCCESS : FAIL;
1537}
1538
1539static enum test_result test_delete_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1540    wait_for_persisted_value(h, h1, "key", "value1");
1541
1542    check(del(h, h1, "key", 0, 0) == ENGINE_SUCCESS, "Failed remove with value.");
1543
1544    wait_for_persisted_value(h, h1, "key", "value2");
1545
1546    testHarness.reload_engine(&h, &h1,
1547                              testHarness.engine_path,
1548                              testHarness.get_current_testcase()->cfg,
1549                              true, false);
1550    wait_for_warmup_complete(h, h1);
1551
1552    check_key_value(h, h1, "key", "value2", 6);
1553    check(del(h, h1, "key", 0, 0) == ENGINE_SUCCESS, "Failed remove with value.");
1554    wait_for_flusher_to_settle(h, h1);
1555
1556    testHarness.reload_engine(&h, &h1,
1557                              testHarness.engine_path,
1558                              testHarness.get_current_testcase()->cfg,
1559                              true, false);
1560    wait_for_warmup_complete(h, h1);
1561
1562    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1563
1564    return SUCCESS;
1565}
1566
1567static enum test_result test_get_delete_missing_file(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1568    const char *key = "key";
1569    wait_for_persisted_value(h, h1, key, "value2delete");
1570
1571    // whack the db file and directory where the key is stored
1572    rmdb();
1573
1574    item *i = NULL;
1575    int errorCode = h1->get(h, NULL, &i, key, strlen(key), 0);
1576    h1->release(h, NULL, i);
1577
1578    // ep engine must be unaware of well-being of the db file as long as
1579    // the item is still in the memory
1580    check(errorCode == ENGINE_SUCCESS, "Expected success for get");
1581
1582    i = NULL;
1583    evict_key(h, h1, key);
1584    errorCode = h1->get(h, NULL, &i, key, strlen(key), 0);
1585    h1->release(h, NULL, i);
1586
1587    // ep engine must be now aware of the ill-fated db file where
1588    // the item is supposedly stored
1589    check(errorCode == ENGINE_TMPFAIL, "Expected tmp fail for get");
1590
1591    return SUCCESS;
1592}
1593
1594
1595static enum test_result test_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1596    item *i = NULL;
1597    static const char val[] = "somevalue";
1598    ENGINE_ERROR_CODE ret;
1599    check((ret = store(h, h1, NULL, OPERATION_SET, "key", val, &i)) == ENGINE_SUCCESS,
1600          "Failed set.");
1601    h1->release(h, NULL, i);
1602
1603    testHarness.reload_engine(&h, &h1,
1604                              testHarness.engine_path,
1605                              testHarness.get_current_testcase()->cfg,
1606                              true, false);
1607    wait_for_warmup_complete(h, h1);
1608    check_key_value(h, h1, "key", val, strlen(val));
1609    return SUCCESS;
1610}
1611
1612static enum test_result test_restart_session_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1613    createTapConn(h, h1, "tap_client_thread");
1614
1615    testHarness.reload_engine(&h, &h1,
1616                              testHarness.engine_path,
1617                              testHarness.get_current_testcase()->cfg,
1618                              true, false);
1619    wait_for_warmup_complete(h, h1);
1620    createTapConn(h, h1, "tap_client_thread");
1621
1622    check(h1->get_stats(h, NULL, "tap", 3, add_stats) == ENGINE_SUCCESS,
1623          "Failed to get stats.");
1624    std::string val = vals["eq_tapq:tap_client_thread:backfill_completed"];
1625    check(strcmp(val.c_str(), "true") == 0, "Don't expect the backfill upon restart");
1626    return SUCCESS;
1627}
1628
1629static enum test_result test_specialKeys(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1630    item *i = NULL;
1631    ENGINE_ERROR_CODE ret;
1632
1633    // Simplified Chinese "Couchbase"
1634    static const char key0[] = "沙发数据库";
1635    static const char val0[] = "some Chinese value";
1636    check((ret = store(h, h1, NULL, OPERATION_SET, key0, val0, &i)) == ENGINE_SUCCESS,
1637          "Failed set Chinese key");
1638    check_key_value(h, h1, key0, val0, strlen(val0));
1639    h1->release(h, NULL, i);
1640    // Traditional Chinese "Couchbase"
1641    static const char key1[] = "沙發數據庫";
1642    static const char val1[] = "some Traditional Chinese value";
1643    check((ret = store(h, h1, NULL, OPERATION_SET, key1, val1, &i)) == ENGINE_SUCCESS,
1644          "Failed set Traditional Chinese key");
1645    h1->release(h, NULL, i);
1646    // Korean "couch potato"
1647    static const char key2[] = "쇼파감자";
1648    static const char val2[] = "some Korean value";
1649    check((ret = store(h, h1, NULL, OPERATION_SET, key2, val2, &i)) == ENGINE_SUCCESS,
1650          "Failed set Korean key");
1651    h1->release(h, NULL, i);
1652    // Russian "couch potato"
1653    static const char key3[] = "лодырь, лентяй";
1654    static const char val3[] = "some Russian value";
1655    check((ret = store(h, h1, NULL, OPERATION_SET, key3, val3, &i)) == ENGINE_SUCCESS,
1656          "Failed set Russian key");
1657    h1->release(h, NULL, i);
1658    // Japanese "couch potato"
1659    static const char key4[] = "カウチポテト";
1660    static const char val4[] = "some Japanese value";
1661    check((ret = store(h, h1, NULL, OPERATION_SET, key4, val4, &i)) == ENGINE_SUCCESS,
1662          "Failed set Japanese key");
1663    h1->release(h, NULL, i);
1664    // Indian char key, and no idea what it is
1665    static const char key5[] = "हरियानवी";
1666    static const char val5[] = "some Indian value";
1667    check((ret = store(h, h1, NULL, OPERATION_SET, key5, val5, &i)) == ENGINE_SUCCESS,
1668          "Failed set Indian key");
1669    h1->release(h, NULL, i);
1670    // Portuguese translation "couch potato"
1671    static const char key6[] = "sedentário";
1672    static const char val6[] = "some Portuguese value";
1673    check((ret = store(h, h1, NULL, OPERATION_SET, key6, val6, &i)) == ENGINE_SUCCESS,
1674          "Failed set Portuguese key");
1675    h1->release(h, NULL, i);
1676    // Arabic translation "couch potato"
1677    static const char key7[] = "الحافلةالبطاطة";
1678    static const char val7[] = "some Arabic value";
1679    check((ret = store(h, h1, NULL, OPERATION_SET, key7, val7, &i)) == ENGINE_SUCCESS,
1680          "Failed set Arabic key");
1681    h1->release(h, NULL, i);
1682
1683    testHarness.reload_engine(&h, &h1,
1684                              testHarness.engine_path,
1685                              testHarness.get_current_testcase()->cfg,
1686                              true, false);
1687    wait_for_warmup_complete(h, h1);
1688    check_key_value(h, h1, key0, val0, strlen(val0));
1689    check_key_value(h, h1, key1, val1, strlen(val1));
1690    check_key_value(h, h1, key2, val2, strlen(val2));
1691    check_key_value(h, h1, key3, val3, strlen(val3));
1692    check_key_value(h, h1, key4, val4, strlen(val4));
1693    check_key_value(h, h1, key5, val5, strlen(val5));
1694    check_key_value(h, h1, key6, val6, strlen(val6));
1695    check_key_value(h, h1, key7, val7, strlen(val7));
1696    return SUCCESS;
1697}
1698
1699static enum test_result test_binKeys(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1700    item *i = NULL;
1701    ENGINE_ERROR_CODE ret;
1702
1703    // binary key with char values beyond 0x7F
1704    static const char key0[] = "\xe0\xed\xf1\x6f\x7f\xf8\xfa";
1705    static const char val0[] = "some value val8";
1706    check((ret = store(h, h1, NULL, OPERATION_SET, key0, val0, &i)) == ENGINE_SUCCESS,
1707          "Failed set binary key0");
1708    check_key_value(h, h1, key0, val0, strlen(val0));
1709    h1->release(h, NULL, i);
1710    // binary keys with char values beyond 0x7F
1711    static const char key1[] = "\xf1\xfd\xfe\xff\xf0\xf8\xef";
1712    static const char val1[] = "some value val9";
1713    check((ret = store(h, h1, NULL, OPERATION_SET, key1, val1, &i)) == ENGINE_SUCCESS,
1714          "Failed set binary key1");
1715    check_key_value(h, h1, key1, val1, strlen(val1));
1716    h1->release(h, NULL, i);
1717    // binary keys with special utf-8 BOM (Byte Order Mark) values 0xBB 0xBF 0xEF
1718    static const char key2[] = "\xff\xfe\xbb\xbf\xef";
1719    static const char val2[] = "some utf-8 bom value";
1720    check((ret = store(h, h1, NULL, OPERATION_SET, key2, val2, &i)) == ENGINE_SUCCESS,
1721          "Failed set binary utf-8 bom key");
1722    check_key_value(h, h1, key2, val2, strlen(val2));
1723    h1->release(h, NULL, i);
1724    // binary keys with special utf-16BE BOM values "U+FEFF"
1725    static const char key3[] = "U+\xfe\xff\xefU+\xff\xfe";
1726    static const char val3[] = "some utf-16 bom value";
1727    check((ret = store(h, h1, NULL, OPERATION_SET, key3, val3, &i)) == ENGINE_SUCCESS,
1728          "Failed set binary utf-16 bom key");
1729    check_key_value(h, h1, key3, val3, strlen(val3));
1730    h1->release(h, NULL, i);
1731
1732    testHarness.reload_engine(&h, &h1,
1733                              testHarness.engine_path,
1734                              testHarness.get_current_testcase()->cfg,
1735                              true, false);
1736    wait_for_warmup_complete(h, h1);
1737    check_key_value(h, h1, key0, val0, strlen(val0));
1738    check_key_value(h, h1, key1, val1, strlen(val1));
1739    check_key_value(h, h1, key2, val2, strlen(val2));
1740    check_key_value(h, h1, key3, val3, strlen(val3));
1741    return SUCCESS;
1742}
1743
1744static enum test_result test_restart_bin_val(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1745
1746
1747
1748    char binaryData[] = "abcdefg\0gfedcba";
1749    cb_assert(sizeof(binaryData) != strlen(binaryData));
1750
1751    item *i = NULL;
1752    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
1753                       binaryData, sizeof(binaryData), 82758, &i, 0, 0)
1754          == ENGINE_SUCCESS,
1755          "Failed set.");
1756    h1->release(h, NULL, i);
1757
1758    testHarness.reload_engine(&h, &h1,
1759                              testHarness.engine_path,
1760                              testHarness.get_current_testcase()->cfg,
1761                              true, false);
1762    wait_for_warmup_complete(h, h1);
1763
1764    check_key_value(h, h1, "key", binaryData, sizeof(binaryData));
1765    return SUCCESS;
1766}
1767
1768static enum test_result test_wrong_vb_get(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1769    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1770    check(ENGINE_NOT_MY_VBUCKET == verify_key(h, h1, "key", 1),
1771          "Expected wrong bucket.");
1772    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1773    return SUCCESS;
1774}
1775
1776static enum test_result test_vb_get_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1777    check(set_vbucket_state(h, h1, 1, vbucket_state_pending), "Failed to set vbucket state.");
1778    const void *cookie = testHarness.create_cookie();
1779    testHarness.set_ewouldblock_handling(cookie, false);
1780
1781    item *i = NULL;
1782    check(ENGINE_EWOULDBLOCK == h1->get(h, cookie, &i, "key", strlen("key"), 1),
1783          "Expected woodblock.");
1784    h1->release(h, NULL, i);
1785
1786    testHarness.destroy_cookie(cookie);
1787    return SUCCESS;
1788}
1789
1790static enum test_result test_vb_get_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1791    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set vbucket state.");
1792    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1793    check(ENGINE_NOT_MY_VBUCKET == verify_key(h, h1, "key", 1),
1794          "Expected not my bucket.");
1795    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1796    return SUCCESS;
1797}
1798
1799static enum test_result test_wrong_vb_incr(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1800    uint64_t cas, result;
1801    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1802    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
1803                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1804                         1) == ENGINE_NOT_MY_VBUCKET,
1805          "Expected not my vbucket.");
1806    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1807    return SUCCESS;
1808}
1809
1810static enum test_result test_vb_incr_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1811    const void *cookie = testHarness.create_cookie();
1812    testHarness.set_ewouldblock_handling(cookie, false);
1813    uint64_t cas, result;
1814    check(set_vbucket_state(h, h1, 1, vbucket_state_pending), "Failed to set vbucket state.");
1815    check(h1->arithmetic(h, cookie, "key", 3, true, false, 1, 1, 0,
1816                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1817                         1) == ENGINE_EWOULDBLOCK,
1818          "Expected woodblock.");
1819    testHarness.destroy_cookie(cookie);
1820    return SUCCESS;
1821}
1822
1823static enum test_result test_vb_incr_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1824    uint64_t cas, result;
1825    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set vbucket state.");
1826    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1827    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
1828                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1829                         1) == ENGINE_NOT_MY_VBUCKET,
1830          "Expected not my bucket.");
1831    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1832    return SUCCESS;
1833}
1834
1835static enum test_result test_wrong_vb_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1836    return test_wrong_vb_mutation(h, h1, OPERATION_SET);
1837}
1838
1839static enum test_result test_wrong_vb_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1840    return test_wrong_vb_mutation(h, h1, OPERATION_CAS);
1841}
1842
1843static enum test_result test_wrong_vb_add(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1844    return test_wrong_vb_mutation(h, h1, OPERATION_ADD);
1845}
1846
1847static enum test_result test_wrong_vb_replace(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1848    return test_wrong_vb_mutation(h, h1, OPERATION_REPLACE);
1849}
1850
1851static enum test_result test_wrong_vb_append(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1852    return test_wrong_vb_mutation(h, h1, OPERATION_APPEND);
1853}
1854
1855static enum test_result test_wrong_vb_prepend(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1856    return test_wrong_vb_mutation(h, h1, OPERATION_PREPEND);
1857}
1858
1859static enum test_result test_wrong_vb_del(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1860    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1861    check(ENGINE_NOT_MY_VBUCKET == del(h, h1, "key", 0, 1), "Expected wrong bucket.");
1862    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1863    return SUCCESS;
1864}
1865
1866static enum test_result test_expiry(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1867    const char *key = "test_expiry";
1868    const char *data = "some test data here.";
1869
1870    item *it = NULL;
1871
1872    ENGINE_ERROR_CODE rv;
1873    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 2,
1874                      PROTOCOL_BINARY_RAW_BYTES);
1875    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1876
1877    item_info info;
1878    info.nvalue = 1;
1879    if (!h1->get_item_info(h, NULL, it, &info)) {
1880        abort();
1881    }
1882    memcpy(info.value[0].iov_base, data, strlen(data));
1883
1884    uint64_t cas = 0;
1885    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
1886    check(rv == ENGINE_SUCCESS, "Set failed.");
1887    check_key_value(h, h1, key, data, strlen(data));
1888    h1->release(h, NULL, it);
1889
1890    testHarness.time_travel(5);
1891    check(h1->get(h, NULL, &it, key, strlen(key), 0) == ENGINE_KEY_ENOENT,
1892          "Item didn't expire");
1893
1894    int expired_access = get_int_stat(h, h1, "ep_expired_access");
1895    int expired_pager = get_int_stat(h, h1, "ep_expired_pager");
1896    int active_expired = get_int_stat(h, h1, "vb_active_expired");
1897    check(expired_pager == 0, "Expected zero expired item by pager");
1898    check(expired_access == 1, "Expected an expired item on access");
1899    check(active_expired == 1, "Expected an expired active item");
1900    checkeq(ENGINE_SUCCESS, store(h, h1, NULL, OPERATION_SET, key, data, &it),
1901            "Failed set.");
1902    h1->release(h, NULL, it);
1903
1904    std::stringstream ss;
1905    ss << "curr_items stat should be still 1 after ";
1906    ss << "overwriting the key that was expired, but not purged yet";
1907    checkeq(1, get_int_stat(h, h1, "curr_items"), ss.str().c_str());
1908
1909    return SUCCESS;
1910}
1911
1912static enum test_result test_expiry_loader(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1913    const char *key = "test_expiry_loader";
1914    const char *data = "some test data here.";
1915
1916    item *it = NULL;
1917
1918    ENGINE_ERROR_CODE rv;
1919    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 2,
1920                      PROTOCOL_BINARY_RAW_BYTES);
1921    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1922
1923    item_info info;
1924    info.nvalue = 1;
1925    if (!h1->get_item_info(h, NULL, it, &info)) {
1926        abort();
1927    }
1928    memcpy(info.value[0].iov_base, data, strlen(data));
1929
1930    uint64_t cas = 0;
1931    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
1932    check(rv == ENGINE_SUCCESS, "Set failed.");
1933    check_key_value(h, h1, key, data, strlen(data));
1934    h1->release(h, NULL, it);
1935
1936    testHarness.time_travel(3);
1937
1938    check(h1->get(h, NULL, &it, key, strlen(key), 0) == ENGINE_KEY_ENOENT,
1939          "Item didn't expire");
1940
1941    // Restart the engine to ensure the above expired item is not loaded
1942    testHarness.reload_engine(&h, &h1,
1943                              testHarness.engine_path,
1944                              testHarness.get_current_testcase()->cfg,
1945                              true, false);
1946    wait_for_warmup_complete(h, h1);
1947    cb_assert(0 == get_int_stat(h, h1, "ep_warmup_value_count", "warmup"));
1948
1949    return SUCCESS;
1950}
1951
1952static enum test_result test_expiration_on_warmup(ENGINE_HANDLE *h,
1953                                                  ENGINE_HANDLE_V1 *h1) {
1954    const char *key = "KEY";
1955    const char *data = "VALUE";
1956
1957    item *it = NULL;
1958
1959    ENGINE_ERROR_CODE rv;
1960    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 3,
1961                      PROTOCOL_BINARY_RAW_BYTES);
1962    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1963
1964    item_info info;
1965    info.nvalue = 1;
1966    if (!h1->get_item_info(h, NULL, it, &info)) {
1967        abort();
1968    }
1969    memcpy(info.value[0].iov_base, data, strlen(data));
1970
1971    uint64_t cas = 0;
1972    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
1973    check(rv == ENGINE_SUCCESS, "Set failed.");
1974    check_key_value(h, h1, key, data, strlen(data));
1975    h1->release(h, NULL, it);
1976    wait_for_flusher_to_settle(h, h1);
1977
1978    check(get_int_stat(h, h1, "curr_items") == 1, "Failed store item");
1979    testHarness.time_travel(5);
1980
1981    // Restart the engine to ensure the above item is expired
1982    testHarness.reload_engine(&h, &h1,
1983                              testHarness.engine_path,
1984                              testHarness.get_current_testcase()->cfg,
1985                              true, false);
1986    wait_for_warmup_complete(h, h1);
1987    wait_for_flusher_to_settle(h, h1);
1988    check(get_int_stat(h, h1, "curr_items") == 0,
1989            "The item should have been expired.");
1990
1991    return SUCCESS;
1992
1993}
1994
1995static enum test_result test_bug3454(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1996    const char *key = "test_expiry_duplicate_warmup";
1997    const char *data = "some test data here.";
1998
1999    item *it = NULL;
2000
2001    ENGINE_ERROR_CODE rv;
2002    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 5,
2003                      PROTOCOL_BINARY_RAW_BYTES);
2004    check(rv == ENGINE_SUCCESS, "Allocation failed.");
2005
2006    item_info info;
2007    info.nvalue = 1;
2008    if (!h1->get_item_info(h, NULL, it, &info)) {
2009        abort();
2010    }
2011    memcpy(info.value[0].iov_base, data, strlen(data));
2012
2013    uint64_t cas = 0;
2014    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
2015    check(rv == ENGINE_SUCCESS, "Set failed.");
2016    check_key_value(h, h1, key, data, strlen(data));
2017    h1->release(h, NULL, it);
2018    wait_for_flusher_to_settle(h, h1);
2019
2020    // Advance the ep_engine time by 10 sec for the above item to be expired.
2021    testHarness.time_travel(10);
2022    check(h1->get(h, NULL, &it, key, strlen(key), 0) == ENGINE_KEY_ENOENT,
2023          "Item didn't expire");
2024
2025    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 0,
2026                      PROTOCOL_BINARY_RAW_BYTES);
2027    check(rv == ENGINE_SUCCESS, "Allocation failed.");
2028
2029    info.nvalue = 1;
2030    if (!h1->get_item_info(h, NULL, it, &info)) {
2031        abort();
2032    }
2033    memcpy(info.value[0].iov_base, data, strlen(data));
2034
2035    cas = 0;
2036    // Add a new item with the same key.
2037    rv = h1->store(h, NULL, it, &cas, OPERATION_ADD, 0);
2038    check(rv == ENGINE_SUCCESS, "Add failed.");
2039    check_key_value(h, h1, key, data, strlen(data));
2040    h1->release(h, NULL, it);
2041
2042    check(h1->get(h, NULL, &it, key, strlen(key), 0) == ENGINE_SUCCESS,
2043          "Item shouldn't expire");
2044    h1->release(h, NULL, it);
2045
2046    // Restart the engine to ensure the above unexpired new item is loaded
2047    testHarness.reload_engine(&h, &h1,
2048                              testHarness.engine_path,
2049                              testHarness.get_current_testcase()->cfg,
2050                              true, false);
2051    wait_for_warmup_complete(h, h1);
2052    cb_assert(1 == get_int_stat(h, h1, "ep_warmup_value_count", "warmup"));
2053    cb_assert(0 == get_int_stat(h, h1, "ep_warmup_dups", "warmup"));
2054
2055    return SUCCESS;
2056}
2057
2058static enum test_result test_bug3522(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2059    const char *key = "test_expiry_no_items_warmup";
2060    const char *data = "some test data here.";
2061
2062    item *it = NULL;
2063
2064    ENGINE_ERROR_CODE rv;
2065    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 0,
2066                      PROTOCOL_BINARY_RAW_BYTES);
2067    check(rv == ENGINE_SUCCESS, "Allocation failed.");
2068
2069    item_info info;
2070    info.nvalue = 1;
2071    if (!h1->get_item_info(h, NULL, it, &info)) {
2072        abort();
2073    }
2074    memcpy(info.value[0].iov_base, data, strlen(data));
2075
2076    uint64_t cas = 0;
2077    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
2078    check(rv == ENGINE_SUCCESS, "Set failed.");
2079    check_key_value(h, h1, key, data, strlen(data));
2080    h1->release(h, NULL, it);
2081    wait_for_flusher_to_settle(h, h1);
2082
2083    // Add a new item with the same key and 2 sec of expiration.
2084    const char *new_data = "new data here.";
2085    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(new_data), 0, 2,
2086                      PROTOCOL_BINARY_RAW_BYTES);
2087    check(rv == ENGINE_SUCCESS, "Allocation failed.");
2088
2089    info.nvalue = 1;
2090    if (!h1->get_item_info(h, NULL, it, &info)) {
2091        abort();
2092    }
2093    memcpy(info.value[0].iov_base, new_data, strlen(new_data));
2094
2095    int pager_runs = get_int_stat(h, h1, "ep_num_expiry_pager_runs");
2096    cas = 0;
2097    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
2098    check(rv == ENGINE_SUCCESS, "Set failed.");
2099    check_key_value(h, h1, key, new_data, strlen(new_data));
2100    h1->release(h, NULL, it);
2101    testHarness.time_travel(3);
2102    wait_for_stat_change(h, h1, "ep_num_expiry_pager_runs", pager_runs);
2103    wait_for_flusher_to_settle(h, h1);
2104
2105    // Restart the engine.
2106    testHarness.reload_engine(&h, &h1,
2107                              testHarness.engine_path,
2108                              testHarness.get_current_testcase()->cfg,
2109                              true, false);
2110    wait_for_warmup_complete(h, h1);
2111    // TODO: modify this for a better test case
2112    cb_assert(0 == get_int_stat(h, h1, "ep_warmup_dups", "warmup"));
2113
2114    return SUCCESS;
2115}
2116
2117static enum test_result test_get_replica_active_state(ENGINE_HANDLE *h,
2118                                                      ENGINE_HANDLE_V1 *h1) {
2119    protocol_binary_request_header *pkt;
2120    pkt = prepare_get_replica(h, h1, vbucket_state_active);
2121    check(h1->unknown_command(h, NULL, pkt, add_response) ==
2122          ENGINE_SUCCESS, "Get Replica Failed");
2123    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2124          "Expected PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET response.");
2125
2126    return SUCCESS;
2127}
2128
2129static enum test_result test_get_replica_pending_state(ENGINE_HANDLE *h,
2130                                                       ENGINE_HANDLE_V1 *h1) {
2131    protocol_binary_request_header *pkt;
2132
2133    const void *cookie = testHarness.create_cookie();
2134    testHarness.set_ewouldblock_handling(cookie, false);
2135    pkt = prepare_get_replica(h, h1, vbucket_state_pending);
2136    check(h1->unknown_command(h, cookie, pkt, add_response) ==
2137          ENGINE_EWOULDBLOCK, "Should have returned error for pending state");
2138    testHarness.destroy_cookie(cookie);
2139    return SUCCESS;
2140}
2141
2142static enum test_result test_get_replica_dead_state(ENGINE_HANDLE *h,
2143                                                    ENGINE_HANDLE_V1 *h1) {
2144    protocol_binary_request_header *pkt;
2145    pkt = prepare_get_replica(h, h1, vbucket_state_dead);
2146    check(h1->unknown_command(h, NULL, pkt, add_response) ==
2147          ENGINE_SUCCESS, "Get Replica Failed");
2148    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2149          "Expected PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET response.");
2150
2151    return SUCCESS;
2152}
2153
2154static enum test_result test_get_replica(ENGINE_HANDLE *h,
2155                                         ENGINE_HANDLE_V1 *h1) {
2156    protocol_binary_request_header *pkt;
2157    pkt = prepare_get_replica(h, h1, vbucket_state_replica);
2158    check(h1->unknown_command(h, NULL, pkt, add_response) == ENGINE_SUCCESS,
2159                              "Get Replica Failed");
2160    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
2161          "Expected PROTOCOL_BINARY_RESPONSE_SUCCESS response.");
2162    check(strcmp("replicadata", last_body) == 0,
2163                 "Should have returned identical value");
2164
2165    return SUCCESS;
2166}
2167
2168static enum test_result test_get_replica_non_resident(ENGINE_HANDLE *h,
2169                                                      ENGINE_HANDLE_V1 *h1) {
2170
2171    item *i = NULL;
2172    check(store(h, h1, NULL, OPERATION_SET, "key", "value", &i, 0, 0)
2173          == ENGINE_SUCCESS, "Store Failed");
2174    h1->release(h, NULL, i);
2175    wait_for_flusher_to_settle(h, h1);
2176    wait_for_stat_to_be(h, h1, "ep_total_persisted", 1);
2177
2178    evict_key(h, h1, "key", 0, "Ejected.");
2179    check(set_vbucket_state(h, h1, 0, vbucket_state_replica),
2180          "Failed to set vbucket to replica");
2181
2182    get_replica(h, h1, "key", 0);
2183    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "Expected success");
2184
2185    return SUCCESS;
2186}
2187
2188static enum test_result test_get_replica_invalid_key(ENGINE_HANDLE *h,
2189                                                     ENGINE_HANDLE_V1 *h1) {
2190    protocol_binary_request_header *pkt;
2191    bool makeinvalidkey = true;
2192    pkt = prepare_get_replica(h, h1, vbucket_state_replica, makeinvalidkey);
2193    check(h1->unknown_command(h, NULL, pkt, add_response) ==
2194          ENGINE_SUCCESS, "Get Replica Failed");
2195    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2196          "Expected PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET response.");
2197    return SUCCESS;
2198}
2199
2200static enum test_result test_vb_del_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2201    const void *cookie = testHarness.create_cookie();
2202    testHarness.set_ewouldblock_handling(cookie, false);
2203    check(set_vbucket_state(h, h1, 1, vbucket_state_pending), "Failed to set vbucket state.");
2204    check(ENGINE_EWOULDBLOCK == del(h, h1, "key", 0, 1, cookie),
2205          "Expected woodblock.");
2206    testHarness.destroy_cookie(cookie);
2207    return SUCCESS;
2208}
2209
2210static enum test_result test_vb_del_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2211    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set vbucket state.");
2212    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
2213    check(ENGINE_NOT_MY_VBUCKET == del(h, h1, "key", 0, 1),
2214          "Expected not my vbucket.");
2215    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
2216    return SUCCESS;
2217}
2218
2219static enum test_result test_touch(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2220    // key is a mandatory field!
2221    touch(h, h1, NULL, 0, (time(NULL) + 10));
2222    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2223
2224    // extlen is a mandatory field!
2225    protocol_binary_request_header *request;
2226    request = createPacket(PROTOCOL_BINARY_CMD_TOUCH, 0, 0, NULL, 0, "akey", 4);
2227    check(h1->unknown_command(h, NULL, request, add_response) == ENGINE_SUCCESS,
2228          "Failed to call touch");
2229    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2230    free(request);
2231
2232    // Try to touch an unknown item...
2233    touch(h, h1, "mykey", 0, (time(NULL) + 10));
2234    check(last_status == PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, "Testing unknown key");
2235
2236    // illegal vbucket
2237    touch(h, h1, "mykey", 5, (time(NULL) + 10));
2238    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET, "Testing illegal vbucket");
2239
2240    // Store the item!
2241    item *itm = NULL;
2242    check(store(h, h1, NULL, OPERATION_SET, "mykey", "somevalue", &itm) == ENGINE_SUCCESS,
2243          "Failed set.");
2244    h1->release(h, NULL, itm);
2245
2246    check_key_value(h, h1, "mykey", "somevalue", strlen("somevalue"));
2247
2248    touch(h, h1, "mykey", 0, (time(NULL) + 10));
2249    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch mykey");
2250
2251    // time-travel 9 secs..
2252    testHarness.time_travel(9);
2253
2254    // The item should still exist
2255    check_key_value(h, h1, "mykey", "somevalue", 9);
2256
2257    // time-travel 2 secs..
2258    testHarness.time_travel(2);
2259
2260    // The item should have expired now...
2261    check(h1->get(h, NULL, &itm, "mykey", 5, 0) == ENGINE_KEY_ENOENT, "Item should be gone");
2262    return SUCCESS;
2263}
2264
2265static enum test_result test_touch_mb7342(ENGINE_HANDLE *h,
2266                                          ENGINE_HANDLE_V1 *h1) {
2267    const char *key = "MB-7342";
2268    // Store the item!
2269    item *itm = NULL;
2270    check(store(h, h1, NULL, OPERATION_SET, key, "v", &itm) == ENGINE_SUCCESS,
2271          "Failed set.");
2272    h1->release(h, NULL, itm);
2273
2274    touch(h, h1, key, 0, 0);
2275    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch key");
2276
2277    check_key_value(h, h1, key, "v", 1);
2278
2279    // Travel a loong time to see if the object is still there (the default
2280    // store sets an exp time of 3600
2281    testHarness.time_travel(3700);
2282
2283    check_key_value(h, h1, key, "v", 1);
2284
2285    return SUCCESS;
2286}
2287
2288static enum test_result test_touch_mb10277(ENGINE_HANDLE *h,
2289                                            ENGINE_HANDLE_V1 *h1) {
2290    const char *key = "MB-10277";
2291    // Store the item!
2292    item *itm = NULL;
2293    check(store(h, h1, NULL, OPERATION_SET, key, "v", &itm) == ENGINE_SUCCESS,
2294          "Failed set.");
2295    h1->release(h, NULL, itm);
2296    wait_for_flusher_to_settle(h, h1);
2297    evict_key(h, h1, key, 0, "Ejected.");
2298
2299    touch(h, h1, key, 0, 3600); // A new expiration time remains in the same.
2300    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch key");
2301
2302    return SUCCESS;
2303}
2304
2305static enum test_result test_gat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2306    // key is a mandatory field!
2307    gat(h, h1, NULL, 0, 10);
2308    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2309
2310    // extlen is a mandatory field!
2311    protocol_binary_request_header *request;
2312    request = createPacket(PROTOCOL_BINARY_CMD_GAT, 0, 0, NULL, 0, "akey", 4);
2313    check(h1->unknown_command(h, NULL, request, add_response) == ENGINE_SUCCESS,
2314          "Failed to call gat");
2315    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2316    free(request);
2317
2318    // Try to gat an unknown item...
2319    gat(h, h1, "mykey", 0, 10);
2320    check(last_status == PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, "Testing unknown key");
2321
2322    // illegal vbucket
2323    gat(h, h1, "mykey", 5, 10);
2324    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET, "Testing illegal vbucket");
2325
2326    // Store the item!
2327    item *itm = NULL;
2328    check(store(h, h1, NULL, OPERATION_SET, "mykey", "{\"some\":\"value\"}",
2329                &itm, 0, 0, 3600, PROTOCOL_BINARY_DATATYPE_JSON) == ENGINE_SUCCESS,
2330          "Failed set.");
2331    h1->release(h, NULL, itm);
2332
2333    check_key_value(h, h1, "mykey", "{\"some\":\"value\"}",
2334            strlen("{\"some\":\"value\"}"));
2335
2336    gat(h, h1, "mykey", 0, 10);
2337    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "gat mykey");
2338    check(last_datatype == PROTOCOL_BINARY_DATATYPE_JSON, "Expected datatype to be JSON");
2339    check(memcmp(last_body, "{\"some\":\"value\"}", sizeof("{\"some\":\"value\"}")) == 0,
2340          "Invalid data returned");
2341
2342    // time-travel 9 secs..
2343    testHarness.time_travel(9);
2344
2345    // The item should still exist
2346    check_key_value(h, h1, "mykey", "{\"some\":\"value\"}",
2347                    strlen("{\"some\":\"value\"}"));
2348
2349    // time-travel 2 secs..
2350    testHarness.time_travel(2);
2351
2352    // The item should have expired now...
2353    check(h1->get(h, NULL, &itm, "mykey", 5, 0) == ENGINE_KEY_ENOENT, "Item should be gone");
2354    return SUCCESS;
2355}
2356
2357static enum test_result test_gatq(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2358    // key is a mandatory field!
2359    gat(h, h1, NULL, 0, 10, true);
2360    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2361
2362    // extlen is a mandatory field!
2363    protocol_binary_request_header *request;
2364    request = createPacket(PROTOCOL_BINARY_CMD_GATQ, 0, 0, NULL, 0, "akey", 4);
2365    check(h1->unknown_command(h, NULL, request, add_response) == ENGINE_SUCCESS,
2366          "Failed to call gatq");
2367    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2368    free(request);
2369
2370    // Try to gatq an unknown item...
2371    last_status = static_cast<protocol_binary_response_status>(0xffff);
2372    gat(h, h1, "mykey", 0, 10, true);
2373
2374    // We should not have sent any response!
2375    check(last_status == (protocol_binary_response_status)0xffff, "Testing unknown key");
2376
2377    // illegal vbucket
2378    gat(h, h1, "mykey", 5, 10, true);
2379    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2380          "Testing illegal vbucket");
2381
2382    // Store the item!
2383    item *itm = NULL;
2384    check(store(h, h1, NULL, OPERATION_SET, "mykey", "{\"some\":\"value\"}",
2385                &itm, 0, 0, 3600, PROTOCOL_BINARY_DATATYPE_JSON) == ENGINE_SUCCESS,
2386          "Failed set.");
2387    h1->release(h, NULL, itm);
2388
2389    check_key_value(h, h1, "mykey", "{\"some\":\"value\"}",
2390                    strlen("{\"some\":\"value\"}"));
2391
2392    gat(h, h1, "mykey", 0, 10, true);
2393    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "gat mykey");
2394    check(last_datatype == PROTOCOL_BINARY_DATATYPE_JSON, "Expected datatype to be JSON");
2395    check(memcmp(last_body, "{\"some\":\"value\"}", sizeof("{\"some\":\"value\"}")) == 0,
2396          "Invalid data returned");
2397
2398    // time-travel 9 secs..
2399    testHarness.time_travel(9);
2400
2401    // The item should still exist
2402    check_key_value(h, h1, "mykey", "{\"some\":\"value\"}",
2403                    strlen("{\"some\":\"value\"}"));
2404
2405    // time-travel 2 secs..
2406    testHarness.time_travel(2);
2407
2408    // The item should have expired now...
2409    check(h1->get(h, NULL, &itm, "mykey", 5, 0) == ENGINE_KEY_ENOENT, "Item should be gone");
2410    return SUCCESS;
2411}
2412
2413static enum test_result test_mb5215(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2414    item *itm = NULL;
2415    check(store(h, h1, NULL, OPERATION_SET, "coolkey", "cooler", &itm)
2416          == ENGINE_SUCCESS, "Failed set.");
2417    h1->release(h, NULL, itm);
2418
2419    check_key_value(h, h1, "coolkey", "cooler", strlen("cooler"));
2420
2421    // set new exptime to 111
2422    int expTime = time(NULL) + 111;
2423
2424    touch(h, h1, "coolkey", 0, expTime);
2425    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch coolkey");
2426
2427    //reload engine
2428    testHarness.reload_engine(&h, &h1,
2429                              testHarness.engine_path,
2430                              testHarness.get_current_testcase()->cfg,
2431                              true, false);
2432    wait_for_warmup_complete(h, h1);
2433
2434    //verify persisted expiration time
2435    const char *statkey = "key coolkey 0";
2436    int newExpTime;
2437    check(h1->get(h, NULL, &itm, "coolkey", 7, 0) == ENGINE_SUCCESS,
2438          "Missing key");
2439    h1->release(h, NULL, itm);
2440    newExpTime = get_int_stat(h, h1, "key_exptime", statkey);
2441    check(newExpTime == expTime, "Failed to persist new exptime");
2442
2443    // evict key, touch expiration time, and verify
2444    evict_key(h, h1, "coolkey", 0, "Ejected.");
2445
2446    expTime = time(NULL) + 222;
2447    touch(h, h1, "coolkey", 0, expTime);
2448    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch coolkey");
2449
2450    testHarness.reload_engine(&h, &h1,
2451                              testHarness.engine_path,
2452                              testHarness.get_current_testcase()->cfg,
2453                              true, false);
2454    wait_for_warmup_complete(h, h1);
2455
2456    check(h1->get(h, NULL, &itm, "coolkey", 7, 0) == ENGINE_SUCCESS,
2457          "Missing key");
2458    h1->release(h, NULL, itm);
2459    newExpTime = get_int_stat(h, h1, "key_exptime", statkey);
2460    check(newExpTime == expTime, "Failed to persist new exptime");
2461
2462    return SUCCESS;
2463}
2464
2465static enum test_result test_alloc_limit(ENGINE_HANDLE *h,
2466                                         ENGINE_HANDLE_V1 *h1) {
2467    item *it = NULL;
2468    ENGINE_ERROR_CODE rv;
2469
2470    rv = h1->allocate(h, NULL, &it, "key", 3, 20 * 1024 * 1024, 0, 0,
2471                      PROTOCOL_BINARY_RAW_BYTES);
2472    check(rv == ENGINE_SUCCESS, "Allocated 20MB item");
2473    h1->release(h, NULL, it);
2474
2475    rv = h1->allocate(h, NULL, &it, "key", 3, (20 * 1024 * 1024) + 1, 0, 0,
2476                      PROTOCOL_BINARY_RAW_BYTES);
2477    check(rv == ENGINE_E2BIG, "Object too big");
2478
2479    return SUCCESS;
2480}
2481
2482static enum test_result test_whitespace_db(ENGINE_HANDLE *h,
2483                                           ENGINE_HANDLE_V1 *h1) {
2484    vals.clear();
2485    check(h1->get_stats(h, NULL, NULL, 0, add_stats) == ENGINE_SUCCESS,
2486          "Failed to get stats.");
2487    if (vals["ep_dbname"] != std::string(WHITESPACE_DB)) {
2488        std::cerr << "Expected dbname = ``" << WHITESPACE_DB << "''"
2489                  << ", got ``" << vals["ep_dbname"] << "''" << std::endl;
2490        return FAIL;
2491    }
2492
2493    check(access(WHITESPACE_DB, F_OK) != -1, "I expected the whitespace db to exist");
2494    return SUCCESS;
2495}
2496
2497static enum test_result test_memory_limit(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2498    set_param(h, h1, protocol_binary_engine_param_flush, "mutation_mem_threshold", "95");
2499    int used = get_int_stat(h, h1, "mem_used");
2500    double mem_threshold =
2501        static_cast<double>(get_int_stat(h, h1, "ep_mutation_mem_threshold")) / 100;
2502    int max = static_cast<int>(get_int_stat(h, h1, "ep_max_size") * mem_threshold);
2503    check(get_int_stat(h, h1, "ep_oom_errors") == 0 &&
2504          get_int_stat(h, h1, "ep_tmp_oom_errors") == 0, "Expected no OOM errors.");
2505    cb_assert(used < max);
2506
2507    char data[8192];
2508    memset(data, 'x', sizeof(data));
2509    size_t vlen = max - used - 192;
2510    data[vlen] = 0x00;
2511
2512    item *i = NULL;
2513    // So if we add an item,
2514    check(store(h, h1, NULL, OPERATION_SET, "key", data, &i) == ENGINE_SUCCESS,
2515          "store failure");
2516    check_key_value(h, h1, "key", data, vlen);
2517    h1->release(h, NULL, i);
2518
2519    // There should be no room for another.
2520    ENGINE_ERROR_CODE second = store(h, h1, NULL, OPERATION_SET, "key2", data, &i);
2521    check(second == ENGINE_ENOMEM || second == ENGINE_TMPFAIL,
2522          "should have failed second set");
2523    h1->release(h, NULL, i);
2524    check(get_int_stat(h, h1, "ep_oom_errors") == 1 ||
2525          get_int_stat(h, h1, "ep_tmp_oom_errors") == 1, "Expected an OOM error.");
2526
2527    ENGINE_ERROR_CODE overwrite = store(h, h1, NULL, OPERATION_SET, "key", data, &i);
2528    check(overwrite == ENGINE_ENOMEM || overwrite == ENGINE_TMPFAIL,
2529          "should have failed second override");
2530    h1->release(h, NULL, i);
2531    check(get_int_stat(h, h1, "ep_oom_errors") == 2 ||
2532          get_int_stat(h, h1, "ep_tmp_oom_errors") == 2, "Expected another OOM error.");
2533    check_key_value(h, h1, "key", data, vlen);
2534    check(ENGINE_SUCCESS != verify_key(h, h1, "key2"), "Expected a failure in GET");
2535    int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
2536    // Until we remove that item
2537    check(del(h, h1, "key", 0, 0) == ENGINE_SUCCESS, "Failed remove with value.");
2538    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
2539    testHarness.time_travel(65);
2540    wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
2541
2542    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue2", &i) == ENGINE_SUCCESS,
2543          "should have succeded on the last set");
2544    check_key_value(h, h1, "key2", "somevalue2", 10);
2545    h1->release(h, NULL, i);
2546    return SUCCESS;
2547}
2548
2549static enum test_result test_vbucket_get_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2550    return verify_vbucket_missing(h, h1, 1) ? SUCCESS : FAIL;
2551}
2552
2553static enum test_result test_vbucket_get(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2554    return verify_vbucket_state(h, h1, 0, vbucket_state_active) ? SUCCESS : FAIL;
2555}
2556
2557static enum test_result test_vbucket_create(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2558    if (!verify_vbucket_missing(h, h1, 1)) {
2559        fprintf(stderr, "vbucket wasn't missing.\n");
2560        return FAIL;
2561    }
2562
2563    if (!set_vbucket_state(h, h1, 1, vbucket_state_active)) {
2564        fprintf(stderr, "set state failed.\n");
2565        return FAIL;
2566    }
2567
2568    return verify_vbucket_state(h, h1, 1, vbucket_state_active) ? SUCCESS : FAIL;
2569}
2570
2571static enum test_result test_vbucket_compact(ENGINE_HANDLE *h,
2572                                             ENGINE_HANDLE_V1 *h1) {
2573    const char *key = "Carss";
2574    const char *value = "pollute";
2575    if (!verify_vbucket_missing(h, h1, 0)) {
2576        fprintf(stderr, "vbucket wasn't missing.\n");
2577        return FAIL;
2578    }
2579
2580    if (!set_vbucket_state(h, h1, 0, vbucket_state_active)) {
2581        fprintf(stderr, "set state failed.\n");
2582        return FAIL;
2583    }
2584
2585    check(verify_vbucket_state(h, h1, 0, vbucket_state_active),
2586            "VBucket state not active");
2587
2588    // Set two keys - one to be expired and other to remain...
2589    item *itm = NULL;
2590    check(store(h, h1, NULL, OPERATION_SET, key, value, &itm)
2591          == ENGINE_SUCCESS, "Failed set.");
2592    h1->release(h, NULL, itm);
2593
2594    check_key_value(h, h1, key, value, strlen(value));
2595
2596    // Set a non-expiring key...
2597    check(store(h, h1, NULL, OPERATION_SET, "trees", "cleanse", &itm)
2598          == ENGINE_SUCCESS, "Failed set.");
2599    h1->release(h, NULL, itm);
2600
2601    check_key_value(h, h1, "trees", "cleanse", strlen("cleanse"));
2602
2603    touch(h, h1, key, 0, 11);
2604    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch Carss");
2605
2606    testHarness.time_travel(12);
2607    wait_for_flusher_to_settle(h, h1);
2608
2609    check(get_int_stat(h, h1, "vb_0:purge_seqno", "vbucket-seqno") == 0,
2610            "purge_seqno not found to be zero before compaction");
2611
2612    // Compaction on VBucket
2613    compact_db(h, h1, 0, 2, 3, 1);
2614
2615    check(get_int_stat(h, h1, "ep_pending_compactions") == 0,
2616    "ep_pending_compactions stat did not tick down after compaction command");
2617
2618    // the key tree and its value should be intact...
2619    check(verify_key(h, h1, "trees") == ENGINE_SUCCESS,
2620          "key trees should be found.");
2621    // the key Carrs should have disappeared...
2622    int val = verify_key(h, h1, "Carss");
2623    check(val == ENGINE_KEY_ENOENT, "Key Carss has not expired.");
2624
2625    check(get_int_stat(h, h1, "vb_0:purge_seqno", "vbucket-seqno") == 3,
2626        "purge_seqno didn't match expected value");
2627
2628    return SUCCESS;
2629}
2630
2631struct comp_thread_ctx {
2632    ENGINE_HANDLE *h;
2633    ENGINE_HANDLE_V1 *h1;
2634    uint16_t vbid;
2635};
2636
2637extern "C" {
2638    static void compaction_thread(void *arg) {
2639        struct comp_thread_ctx *ctx = static_cast<comp_thread_ctx *>(arg);
2640        compact_db(ctx->h, ctx->h1, ctx->vbid, 0, 0, 0);
2641    }
2642}
2643
2644static enum test_result test_multiple_vb_compactions(ENGINE_HANDLE *h,
2645                                                     ENGINE_HANDLE_V1 *h1) {
2646    for (uint16_t i = 0; i < 4; ++i) {
2647        if (!set_vbucket_state(h, h1, i, vbucket_state_active)) {
2648            fprintf(stderr, "set state failed for vbucket %d.\n", i);
2649            return FAIL;
2650        }
2651        check(verify_vbucket_state(h, h1, i, vbucket_state_active),
2652              "VBucket state not active");
2653    }
2654
2655    std::vector<std::string> keys;
2656    for (int j = 0; j < 20000; ++j) {
2657        std::stringstream ss;
2658        ss << "key" << j;
2659        std::string key(ss.str());
2660        keys.push_back(key);
2661    }
2662
2663    int count = 0;
2664    std::vector<std::string>::iterator it;
2665    for (it = keys.begin(); it != keys.end(); ++it) {
2666        uint16_t vbid = count % 4;
2667        item *i;
2668        check(store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str(), &i, 0, vbid)
2669              == ENGINE_SUCCESS, "Failed to store a value");
2670        h1->release(h, NULL, i);
2671        ++count;
2672    }
2673
2674    // Compact multiple vbuckets.
2675    const int n_threads = 4;
2676    cb_thread_t threads[n_threads];
2677    struct comp_thread_ctx ctx[n_threads];
2678
2679    for (int i = 0; i < n_threads; i++) {
2680        ctx[i].h = h;
2681        ctx[i].h1 = h1;
2682        ctx[i].vbid = static_cast<uint16_t>(i);
2683        int r = cb_create_thread(&threads[i], compaction_thread, &ctx[i], 0);
2684        cb_assert(r == 0);
2685    }
2686
2687    for (int i = 0; i < n_threads; i++) {
2688        int r = cb_join_thread(threads[i]);
2689        cb_assert(r == 0);
2690    }
2691
2692    check(get_int_stat(h, h1, "ep_pending_compactions") == 0,
2693    "ep_pending_compactions stat did not tick down after compaction command");
2694
2695    return SUCCESS;
2696}
2697
2698static enum test_result
2699test_multi_vb_compactions_with_workload(ENGINE_HANDLE *h,
2700                                        ENGINE_HANDLE_V1 *h1) {
2701    for (uint16_t i = 0; i < 4; ++i) {
2702        if (!set_vbucket_state(h, h1, i, vbucket_state_active)) {
2703            fprintf(stderr, "set state failed for vbucket %d.\n", i);
2704            return FAIL;
2705        }
2706        check(verify_vbucket_state(h, h1, i, vbucket_state_active),
2707              "VBucket state not active");
2708    }
2709
2710    std::vector<std::string> keys;
2711    for (int j = 0; j < 10000; ++j) {
2712        std::stringstream ss;
2713        ss << "key" << j;
2714        std::string key(ss.str());
2715        keys.push_back(key);
2716    }
2717
2718    int count = 0;
2719    std::vector<std::string>::iterator it;
2720    for (it = keys.begin(); it != keys.end(); ++it) {
2721        uint16_t vbid = count % 4;
2722        item *i;
2723        check(store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str(), &i, 0, vbid)
2724              == ENGINE_SUCCESS, "Failed to store a value");
2725        h1->release(h, NULL, i);
2726        ++count;
2727    }
2728    wait_for_flusher_to_settle(h, h1);
2729
2730    for (int i = 0; i < 2; ++i) {
2731        count = 0;
2732        for (it = keys.begin(); it != keys.end(); ++it) {
2733            uint16_t vbid = count % 4;
2734            item *i = NULL;
2735            check(h1->get(h, NULL, &i, it->c_str(), strlen(it->c_str()), vbid) ==
2736                  ENGINE_SUCCESS, "Unable to get stored item");
2737            h1->release(h, NULL, i);
2738            ++count;
2739        }
2740    }
2741    wait_for_str_stat_to_be(h, h1, "ep_workload_pattern", "read_heavy", NULL);
2742
2743    // Compact multiple vbuckets.
2744    const int n_threads = 4;
2745    cb_thread_t threads[n_threads];
2746    struct comp_thread_ctx ctx[n_threads];
2747
2748    for (int i = 0; i < n_threads; i++) {
2749        ctx[i].h = h;
2750        ctx[i].h1 = h1;
2751        ctx[i].vbid = static_cast<uint16_t>(i);
2752        int r = cb_create_thread(&threads[i], compaction_thread, &ctx[i], 0);
2753        cb_assert(r == 0);
2754    }
2755
2756    for (int i = 0; i < n_threads; i++) {
2757        int r = cb_join_thread(threads[i]);
2758        cb_assert(r == 0);
2759    }
2760
2761    check(get_int_stat(h, h1, "ep_pending_compactions") == 0,
2762    "ep_pending_compactions stat did not tick down after compaction command");
2763
2764    return SUCCESS;
2765}
2766
2767static enum test_result vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
2768                                             const char* value = NULL) {
2769    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed to set vbucket state.");
2770
2771    vbucketDelete(h, h1, 2, value);
2772    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2773          "Expected failure deleting non-existent bucket.");
2774
2775    check(set_vbucket_state(h, h1, 1, vbucket_state_dead), "Failed set set vbucket 1 state.");
2776
2777    vbucketDelete(h, h1, 1, value);
2778    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
2779          "Expected failure deleting non-existent bucket.");
2780
2781    check(verify_vbucket_missing(h, h1, 1),
2782          "vbucket 0 was not missing after deleting it.");
2783
2784    return SUCCESS;
2785}
2786
2787static enum test_result test_vbucket_destroy_stats(ENGINE_HANDLE *h,
2788                                                   ENGINE_HANDLE_V1 *h1) {
2789
2790    int mem_used = get_int_stat(h, h1, "mem_used");
2791    int cacheSize = get_int_stat(h, h1, "ep_total_cache_size");
2792    int overhead = get_int_stat(h, h1, "ep_overhead");
2793    int nonResident = get_int_stat(h, h1, "ep_num_non_resident");
2794
2795    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed to set vbucket state.");
2796
2797    std::vector<std::string> keys;
2798    for (int j = 0; j < 2000; ++j) {
2799        std::stringstream ss;
2800        ss << "key" << j;
2801        std::string key(ss.str());
2802        keys.push_back(key);
2803    }
2804
2805    int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
2806    std::vector<std::string>::iterator it;
2807    for (it = keys.begin(); it != keys.end(); ++it) {
2808        item *i;
2809        check(store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str(), &i, 0, 1)
2810              == ENGINE_SUCCESS, "Failed to store a value");
2811        h1->release(h, NULL, i);
2812    }
2813    wait_for_flusher_to_settle(h, h1);
2814    testHarness.time_travel(65);
2815    wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
2816
2817    check(set_vbucket_state(h, h1, 1, vbucket_state_dead), "Failed set set vbucket 1 state.");
2818
2819    int vbucketDel = get_int_stat(h, h1, "ep_vbucket_del");
2820    vbucketDelete(h, h1, 1);
2821    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
2822          "Expected failure deleting non-existent bucket.");
2823
2824    check(verify_vbucket_missing(h, h1, 1),
2825          "vbucket 1 was not missing after deleting it.");
2826
2827    wait_for_stat_change(h, h1, "ep_vbucket_del", vbucketDel);
2828
2829    wait_for_stat_to_be(h, h1, "mem_used", mem_used);
2830    wait_for_stat_to_be(h, h1, "ep_total_cache_size", cacheSize);
2831    wait_for_stat_to_be(h, h1, "ep_overhead", overhead);
2832    wait_for_stat_to_be(h, h1, "ep_num_non_resident", nonResident);
2833
2834    return SUCCESS;
2835}
2836
2837static enum test_result vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
2838                                                const char* value = NULL) {
2839    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed to set vbucket state.");
2840
2841    // Store a value so the restart will try to resurrect it.
2842    item *i = NULL;
2843    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i, 0, 1)
2844          == ENGINE_SUCCESS, "Failed to set a value");
2845    check_key_value(h, h1, "key", "somevalue", 9, 1);
2846    h1->release(h, NULL, i);
2847
2848    // Reload to get a flush forced.
2849    testHarness.reload_engine(&h, &h1,
2850                              testHarness.engine_path,
2851                              testHarness.get_current_testcase()->cfg,
2852                              true, false);
2853    wait_for_warmup_complete(h, h1);
2854
2855    check(verify_vbucket_state(h, h1, 1, vbucket_state_active),
2856          "Bucket state was what it was initially, after restart.");
2857    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed to set vbucket state.");
2858    check_key_value(h, h1, "key", "somevalue", 9, 1);
2859
2860    check(set_vbucket_state(h, h1, 1, vbucket_state_dead), "Failed set set vbucket 1 state.");
2861
2862    vbucketDelete(h, h1, 1, value);
2863    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
2864          "Expected failure deleting non-existent bucket.");
2865
2866    check(verify_vbucket_missing(h, h1, 1),
2867          "vbucket 1 was not missing after deleting it.");
2868
2869    testHarness.reload_engine(&h, &h1,
2870                              testHarness.engine_path,
2871                              testHarness.get_current_testcase()->cfg,
2872                              true, false);
2873    wait_for_warmup_complete(h, h1);
2874
2875    if (verify_vbucket_state(h, h1, 1, vbucket_state_pending, true)) {
2876        std::cerr << "Bucket came up in pending state after delete." << std::endl;
2877        abort();
2878    }
2879
2880    check(verify_vbucket_missing(h, h1, 1),
2881          "vbucket 1 was not missing after restart.");
2882
2883    return SUCCESS;
2884}
2885
2886static enum test_result test_async_vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2887    return vbucket_destroy(h, h1);
2888}
2889
2890static enum test_result test_sync_vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2891    return vbucket_destroy(h, h1, "async=0");
2892}
2893
2894static enum test_result test_async_vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2895    return vbucket_destroy_restart(h, h1);
2896}
2897
2898static enum test_result test_sync_vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2899    return vbucket_destroy_restart(h, h1, "async=0");
2900}
2901
2902static enum test_result test_vb_set_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2903    return test_pending_vb_mutation(h, h1, OPERATION_SET);
2904}
2905
2906static enum test_result test_vb_add_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2907    return test_pending_vb_mutation(h, h1, OPERATION_ADD);
2908}
2909
2910static enum test_result test_vb_cas_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2911    return test_pending_vb_mutation(h, h1, OPERATION_CAS);
2912}
2913
2914static enum test_result test_vb_append_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2915    return test_pending_vb_mutation(h, h1, OPERATION_APPEND);
2916}
2917
2918static enum test_result test_vb_prepend_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2919    return test_pending_vb_mutation(h, h1, OPERATION_PREPEND);
2920}
2921
2922static enum test_result test_vb_set_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2923    return test_replica_vb_mutation(h, h1, OPERATION_SET);
2924}
2925
2926static enum test_result test_vb_replace_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2927    return test_replica_vb_mutation(h, h1, OPERATION_REPLACE);
2928}
2929
2930static enum test_result test_vb_replace_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2931    return test_pending_vb_mutation(h, h1, OPERATION_REPLACE);
2932}
2933
2934static enum test_result test_vb_add_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2935    return test_replica_vb_mutation(h, h1, OPERATION_ADD);
2936}
2937
2938static enum test_result test_vb_cas_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2939    return test_replica_vb_mutation(h, h1, OPERATION_CAS);
2940}
2941
2942static enum test_result test_vb_append_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2943    return test_replica_vb_mutation(h, h1, OPERATION_APPEND);
2944}
2945
2946static enum test_result test_vb_prepend_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2947    return test_replica_vb_mutation(h, h1, OPERATION_PREPEND);
2948}
2949
2950static enum test_result test_stats_seqno(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2951    check(set_vbucket_state(h, h1, 1, vbucket_state_active),
2952          "Failed to set vbucket state.");
2953
2954    int num_keys = 100;
2955    for (int ii = 0; ii < num_keys; ++ii) {
2956        std::stringstream ss;
2957        ss << "key" << ii;
2958        check(store(h, h1, NULL, OPERATION_SET, ss.str().c_str(),
2959                    "value", NULL, 0, 0) == ENGINE_SUCCESS,
2960              "Failed to store an item.");
2961    }
2962
2963    check(get_int_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno") == 100,
2964          "Invalid seqno");
2965    check(get_int_stat(h, h1, "vb_1:high_seqno", "vbucket-seqno") == 0,
2966          "Invalid seqno");
2967    check(get_int_stat(h, h1, "vb_1:high_seqno", "vbucket-seqno 1") == 0,
2968          "Invalid seqno");
2969
2970    uint64_t vb_uuid = get_ull_stat(h, h1, "vb_1:0:id", "failovers");
2971    check(get_ull_stat(h, h1, "vb_1:uuid", "vbucket-seqno 1") == vb_uuid,
2972          "Invalid uuid");
2973    check(vals.size() == 4, "Expected four stats");
2974
2975    // Check invalid vbucket
2976    check(h1->get_stats(h, NULL, "vbucket-seqno 2", 15, add_stats)
2977          == ENGINE_NOT_MY_VBUCKET, "Expected not my vbucket");
2978
2979    // Check bad vbucket parameter (not numeric)
2980    check(h1->get_stats(h, NULL, "vbucket-seqno tt2", 17, add_stats)
2981          == ENGINE_EINVAL, "Expected invalid");
2982
2983    // Check extra spaces at the end
2984    check(h1->get_stats(h, NULL, "vbucket-seqno    ", 17, add_stats)
2985          == ENGINE_EINVAL, "Expected invalid");
2986
2987    return SUCCESS;
2988}
2989
2990static enum test_result test_stats_diskinfo(ENGINE_HANDLE *h,
2991