xref: /3.0.3-GA/ep-engine/tests/ep_testsuite.cc (revision 3c05d1fa)
1/* -*- MODE: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/*
3 *     Copyright 2010 Couchbase, Inc
4 *
5 *   Licensed under the Apache License, Version 2.0 (the "License");
6 *   you may not use this file except in compliance with the License.
7 *   You may obtain a copy of the License at
8 *
9 *       http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *   Unless required by applicable law or agreed to in writing, software
12 *   distributed under the License is distributed on an "AS IS" BASIS,
13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *   See the License for the specific language governing permissions and
15 *   limitations under the License.
16 */
17
18// Usage: (to repeatedly run just a single test case)
19// make engine_tests IS_LOOP=-L EP_TEST_NUM=3
20
21#include "config.h"
22
23#include <stdio.h>
24#include <stdlib.h>
25#include <string.h>
26#include <sys/stat.h>
27#ifdef _MSC_VER
28#include <direct.h>
29#define mkdir(a, b) _mkdir(a)
30#else
31#include <sys/wait.h>
32#endif
33
34#include <cstdlib>
35#include <iostream>
36#include <map>
37#include <set>
38#include <sstream>
39#include <string>
40#include <vector>
41
42#include <platform/dirutils.h>
43
44#include "atomic.h"
45#include "ep-engine/command_ids.h"
46#include "ep_test_apis.h"
47#include "ep_testsuite.h"
48#include "locks.h"
49#include "mock/mock_dcp.h"
50#include "mutex.h"
51
52#include <snappy-c.h>
53#include <JSON_checker.h>
54
55#ifdef linux
56/* /usr/include/netinet/in.h defines macros from ntohs() to _bswap_nn to
57 * optimize the conversion functions, but the prototypes generate warnings
58 * from gcc. The conversion methods isn't the bottleneck for my app, so
59 * just remove the warnings by undef'ing the optimization ..
60 */
61#undef ntohs
62#undef ntohl
63#undef htons
64#undef htonl
65#endif
66
67// ptr_fun don't like the extern "C" thing for unlock cookie.. cast it
68// away ;)
69typedef void (*UNLOCK_COOKIE_T)(const void *cookie);
70
71extern "C" bool abort_msg(const char *expr, const char *msg, int line);
72
73
74template <typename T>
75static void checkeqfn(T exp, T got, const char *msg, const char *file, const int linenum) {
76    if (exp != got) {
77        std::stringstream ss;
78        ss << "Expected `" << exp << "', got `" << got << "' - " << msg;
79        abort_msg(ss.str().c_str(), file, linenum);
80    }
81}
82
83#define checkeq(a, b, c) checkeqfn(a, b, c, __FILE__, __LINE__)
84
85extern "C" {
86
87#define check(expr, msg) \
88    static_cast<void>((expr) ? 0 : abort_msg(#expr, msg, __LINE__))
89
90#define WHITESPACE_DB "whitespace sucks.db"
91#define MULTI_DISPATCHER_CONFIG \
92    "ht_size=129;ht_locks=3;chk_remover_stime=1;chk_period=60"
93
94struct test_harness testHarness;
95
96class ThreadData {
97public:
98    ThreadData(ENGINE_HANDLE *eh, ENGINE_HANDLE_V1 *ehv1,
99               int e=0) : h(eh), h1(ehv1), extra(e) {}
100    ENGINE_HANDLE    *h;
101    ENGINE_HANDLE_V1 *h1;
102    int               extra;
103};
104
105bool abort_msg(const char *expr, const char *msg, int line) {
106    fprintf(stderr, "%s:%d Test failed: `%s' (%s)\n",
107            __FILE__, line, msg, expr);
108    abort();
109    // UNREACHABLE
110    return false;
111}
112
113static const char *dbname_env;
114static enum test_result rmdb(void)
115{
116    const char *files[] = { WHITESPACE_DB,
117                            "/tmp/test",
118                            "/tmp/mutation.log",
119                            dbname_env,
120                            NULL };
121    int ii = 0;
122    while (files[ii] != NULL) {
123        CouchbaseDirectoryUtilities::rmrf(files[ii]);
124        if (access(files[ii], F_OK) != -1) {
125            std::cerr << "Failed to remove: " << files[ii] << " " << std::endl;
126            return FAIL;
127        }
128        ++ii;
129    }
130
131    return SUCCESS;
132}
133
134static enum test_result skipped_test_function(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
135    (void) h;
136    (void) h1;
137    return SKIPPED;
138}
139
140static bool teardown(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
141    (void)h; (void)h1;
142    vals.clear();
143    return true;
144}
145
146static const void* createTapConn(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
147                                 const char *name) {
148    const void *cookie = testHarness.create_cookie();
149    testHarness.lock_cookie(cookie);
150    TAP_ITERATOR iter = h1->get_tap_iterator(h, cookie, name,
151                                             strlen(name),
152                                             TAP_CONNECT_FLAG_DUMP, NULL,
153                                             0);
154    check(iter != NULL, "Failed to create a tap iterator");
155    return cookie;
156}
157
158static void check_key_value(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
159                            const char* key, const char* val, size_t vlen,
160                            uint16_t vbucket = 0) {
161    item_info info;
162    check(get_item_info(h, h1, &info, key, vbucket), "checking key and value");
163    check(info.nvalue == 1, "info.nvalue != 1");
164    check(vlen == info.value[0].iov_len, "Value length mismatch");
165    check(memcmp(info.value[0].iov_base, val, vlen) == 0, "Data mismatch");
166}
167
168static bool test_setup(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
169    wait_for_warmup_complete(h, h1);
170
171    check(h1->get_stats(h, NULL, "prev-vbucket", 12, add_stats) == ENGINE_SUCCESS,
172          "Failed to get the previous state of vbuckets");
173    if (vals.find("vb_0") == vals.end()) {
174        check(set_vbucket_state(h, h1, 0, vbucket_state_active),
175              "Failed to set VB0 state.");
176    }
177
178    wait_for_stat_change(h, h1, "ep_vb_snapshot_total", 0);
179
180    // warmup is complete, notify ep engine that it must now enable
181    // data traffic
182    protocol_binary_request_header *pkt = createPacket(PROTOCOL_BINARY_CMD_ENABLE_TRAFFIC);
183    check(h1->unknown_command(h, NULL, pkt, add_response) == ENGINE_SUCCESS,
184          "Failed to enable data traffic");
185    free(pkt);
186
187    return true;
188}
189
190static enum test_result test_getl(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
191    const char *key = "k1";
192    uint16_t vbucketId = 0;
193    uint32_t expiration = 25;
194
195    getl(h, h1, key, vbucketId, expiration);
196    check(last_status == PROTOCOL_BINARY_RESPONSE_KEY_ENOENT,
197          "expected the key to be missing...");
198    if (last_body != NULL && (strcmp(last_body, "NOT_FOUND") != 0)) {
199        fprintf(stderr, "Should have returned NOT_FOUND. Getl Failed");
200        abort();
201    }
202
203    item *i = NULL;
204    check(store(h, h1, NULL, OPERATION_SET, key, "{\"lock\":\"data\"}",
205                &i, 0, vbucketId, 3600, PROTOCOL_BINARY_DATATYPE_JSON)
206          == ENGINE_SUCCESS, "Failed to store an item.");
207    h1->release(h, NULL, i);
208
209    /* retry getl, should succeed */
210    getl(h, h1, key, vbucketId, expiration);
211    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
212          "Expected to be able to getl on first try");
213    check(strcmp("{\"lock\":\"data\"}", last_body) == 0, "Body was malformed.");
214    check(last_datatype == PROTOCOL_BINARY_DATATYPE_JSON,
215            "Expected datatype to be JSON");
216
217    /* wait 16 seconds */
218    testHarness.time_travel(16);
219
220    /* lock's taken so this should fail */
221    getl(h, h1, key, vbucketId, expiration);
222    check(last_status == PROTOCOL_BINARY_RESPONSE_ETMPFAIL,
223          "Expected to fail getl on second try");
224
225    if (last_body != NULL && (strcmp(last_body, "LOCK_ERROR") != 0)) {
226        fprintf(stderr, "Should have returned LOCK_ERROR. Getl Failed");
227        abort();
228    }
229
230    check(store(h, h1, NULL, OPERATION_SET, key, "lockdata2", &i, 0, vbucketId)
231          != ENGINE_SUCCESS, "Should have failed to store an item.");
232    h1->release(h, NULL, i);
233
234    /* wait another 10 seconds */
235    testHarness.time_travel(10);
236
237    /* retry set, should succeed */
238    check(store(h, h1, NULL, OPERATION_SET, key, "lockdata", &i, 0, vbucketId)
239          == ENGINE_SUCCESS, "Failed to store an item.");
240    h1->release(h, NULL, i);
241
242    /* point to wrong vbucket, to test NOT_MY_VB response */
243    getl(h, h1, key, 10, expiration);
244    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
245          "Should have received not my vbucket response");
246
247    /* acquire lock, should succeed */
248    getl(h, h1, key, vbucketId, expiration);
249    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
250          "Aquire lock should have succeeded");
251    check(last_datatype == PROTOCOL_BINARY_RAW_BYTES,
252            "Expected datatype to be RAW BYTES");
253
254    /* try an incr operation followed by a delete, both of which should fail */
255    uint64_t cas = 0;
256    uint64_t result = 0;
257
258    check(h1->arithmetic(h, NULL, key, 2, true, false, 1, 1, 0,
259                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
260                         0)  == ENGINE_TMPFAIL, "Incr failed");
261
262
263    check(del(h, h1, key, 0, 0) == ENGINE_TMPFAIL, "Delete failed");
264
265
266    /* bug MB 2699 append after getl should fail with ENGINE_TMPFAIL */
267
268    testHarness.time_travel(26);
269
270    char binaryData1[] = "abcdefg\0gfedcba";
271    char binaryData2[] = "abzdefg\0gfedcba";
272
273    check(storeCasVb11(h, h1, NULL, OPERATION_SET, key,
274                       binaryData1, sizeof(binaryData1) - 1, 82758, &i, 0, 0)
275          == ENGINE_SUCCESS,
276          "Failed set.");
277    h1->release(h, NULL, i);
278
279    /* acquire lock, should succeed */
280    getl(h, h1, key, vbucketId, expiration);
281    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
282          "Aquire lock should have succeeded");
283
284    /* append should fail */
285    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, key,
286                       binaryData2, sizeof(binaryData2) - 1, 82758, &i, 0, 0)
287          == ENGINE_TMPFAIL,
288          "Append should fail.");
289    h1->release(h, NULL, i);
290
291    /* bug MB 3252 & MB 3354.
292     * 1. Set a key with an expiry value.
293     * 2. Take a lock on the item before it expires
294     * 3. Wait for the item to expire
295     * 4. Perform a CAS operation, should fail
296     * 5. Perform a set operation, should succeed
297     */
298    const char *ekey = "test_expiry";
299    const char *edata = "some test data here.";
300
301    item *it = NULL;
302
303    check(h1->allocate(h, NULL, &it, ekey, strlen(ekey), strlen(edata), 0, 2,
304          PROTOCOL_BINARY_RAW_BYTES) == ENGINE_SUCCESS, "Allocation Failed");
305
306    item_info info;
307    info.nvalue = 1;
308    if (!h1->get_item_info(h, NULL, it, &info)) {
309        abort();
310    }
311    memcpy(info.value[0].iov_base, edata, strlen(edata));
312
313    check(h1->store(h, NULL, it, &cas, OPERATION_SET, 0) ==
314        ENGINE_SUCCESS, "Failed to Store item");
315    check_key_value(h, h1, ekey, edata, strlen(edata));
316    h1->release(h, NULL, it);
317
318    testHarness.time_travel(3);
319    cas = last_cas;
320
321    /* cas should fail */
322    check(storeCasVb11(h, h1, NULL, OPERATION_CAS, ekey,
323                       binaryData1, sizeof(binaryData1) - 1, 82758, &i, cas, 0)
324          != ENGINE_SUCCESS,
325          "CAS succeeded.");
326    h1->release(h, NULL, i);
327
328    /* but a simple store should succeed */
329    check(store(h, h1, NULL, OPERATION_SET, ekey, edata, &i, 0, vbucketId)
330          == ENGINE_SUCCESS, "Failed to store an item.");
331    h1->release(h, NULL, i);
332    return SUCCESS;
333}
334
335static enum test_result test_unl(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
336
337    const char *key = "k2";
338    uint16_t vbucketId = 0;
339
340    unl(h, h1, key, vbucketId);
341    check(last_status != PROTOCOL_BINARY_RESPONSE_SUCCESS,
342          "expected the key to be missing...");
343
344    item *i = NULL;
345    check(store(h, h1, NULL, OPERATION_SET, key, "lockdata", &i, 0, vbucketId)
346          == ENGINE_SUCCESS, "Failed to store an item.");
347    h1->release(h, NULL, i);
348
349    /* getl, should succeed */
350    getl(h, h1, key, vbucketId, 0);
351    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
352          "Expected to be able to getl on first try");
353
354    /* save the returned cas value for later */
355    uint64_t cas = last_cas;
356
357    /* lock's taken unlocking with a random cas value should fail */
358    unl(h, h1, key, vbucketId);
359    check(last_status == PROTOCOL_BINARY_RESPONSE_ETMPFAIL,
360          "Expected to fail getl on second try");
361
362    if (last_body != NULL && (strcmp(last_body, "UNLOCK_ERROR") != 0)) {
363        fprintf(stderr, "Should have returned UNLOCK_ERROR. Unl Failed");
364        abort();
365    }
366
367    unl(h, h1, key, vbucketId, cas);
368    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
369          "Expected to succed unl with correct cas");
370
371    /* acquire lock, should succeed */
372    getl(h, h1, key, vbucketId, 0);
373
374    /* wait 16 seconds */
375    testHarness.time_travel(16);
376
377    /* lock has expired, unl should fail */
378    unl(h, h1, key, vbucketId, last_cas);
379    check(last_status == PROTOCOL_BINARY_RESPONSE_ETMPFAIL,
380          "Expected to fail unl on lock timeout");
381
382    return SUCCESS;
383}
384
385static enum test_result test_wrong_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
386                                               ENGINE_STORE_OPERATION op) {
387    item *i = NULL;
388    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
389    uint64_t cas = 11;
390    if (op == OPERATION_ADD) {
391        // Add operation with cas != 0 doesn't make sense
392        cas = 0;
393    }
394    check(store(h, h1, NULL, op,
395                "key", "somevalue", &i, cas, 1) == ENGINE_NOT_MY_VBUCKET,
396        "Expected not_my_vbucket");
397    h1->release(h, NULL, i);
398    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
399    return SUCCESS;
400}
401
402static enum test_result test_pending_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
403                                                 ENGINE_STORE_OPERATION op) {
404    const void *cookie = testHarness.create_cookie();
405    testHarness.set_ewouldblock_handling(cookie, false);
406    item *i = NULL;
407    check(set_vbucket_state(h, h1, 1, vbucket_state_pending), "Failed to set vbucket state.");
408    check(verify_vbucket_state(h, h1, 1, vbucket_state_pending), "Bucket state was not set to pending.");
409    uint64_t cas = 11;
410    if (op == OPERATION_ADD) {
411        // Add operation with cas != 0 doesn't make sense..
412        cas = 0;
413    }
414    check(store(h, h1, cookie, op,
415                "key", "somevalue", &i, cas, 1) == ENGINE_EWOULDBLOCK,
416        "Expected woodblock");
417    h1->release(h, NULL, i);
418    testHarness.destroy_cookie(cookie);
419    return SUCCESS;
420}
421
422static enum test_result test_replica_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
423                                                 ENGINE_STORE_OPERATION op) {
424    item *i = NULL;
425    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set vbucket state.");
426    check(verify_vbucket_state(h, h1, 1, vbucket_state_replica), "Bucket state was not set to replica.");
427    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
428
429    uint64_t cas = 11;
430    if (op == OPERATION_ADD) {
431        // performing add with a CAS != 0 doesn't make sense...
432        cas = 0;
433    }
434    check(store(h, h1, NULL, op,
435                "key", "somevalue", &i, cas, 1) == ENGINE_NOT_MY_VBUCKET,
436        "Expected not my vbucket");
437    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
438    h1->release(h, NULL, i);
439    return SUCCESS;
440}
441
442//
443// ----------------------------------------------------------------------
444// The actual tests are below.
445// ----------------------------------------------------------------------
446//
447
448static enum test_result test_get_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
449    check(verify_key(h, h1, "k") == ENGINE_KEY_ENOENT, "Expected miss.");
450    return SUCCESS;
451}
452
453static enum test_result test_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
454    item *i = NULL;
455    check(ENGINE_SUCCESS ==
456          store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
457          "Error setting.");
458    h1->release(h, NULL, i);
459    return SUCCESS;
460}
461
462struct handle_pair {
463    ENGINE_HANDLE *h;
464    ENGINE_HANDLE_V1 *h1;
465};
466
467extern "C" {
468    static void conc_del_set_thread(void *arg) {
469        struct handle_pair *hp = static_cast<handle_pair *>(arg);
470        item *it = NULL;
471
472        for (int i = 0; i < 5000; ++i) {
473            store(hp->h, hp->h1, NULL, OPERATION_ADD,
474                  "key", "somevalue", &it);
475            hp->h1->release(hp->h, NULL, it);
476            usleep(10);
477            checkeq(ENGINE_SUCCESS,
478                    store(hp->h, hp->h1, NULL, OPERATION_SET,
479                          "key", "somevalue", &it),
480                    "Error setting.");
481            hp->h1->release(hp->h, NULL, it);
482            usleep(10);
483            // Ignoring the result here -- we're racing.
484            del(hp->h, hp->h1, "key", 0, 0);
485            usleep(10);
486        }
487    }
488}
489
490static enum test_result test_conc_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
491
492    const int n_threads = 8;
493    cb_thread_t threads[n_threads];
494    struct handle_pair hp = {h, h1};
495
496    wait_for_persisted_value(h, h1, "key", "value1");
497
498    for (int i = 0; i < n_threads; i++) {
499        int r = cb_create_thread(&threads[i], conc_del_set_thread, &hp, 0);
500        cb_assert(r == 0);
501    }
502
503    for (int i = 0; i < n_threads; i++) {
504        int r = cb_join_thread(threads[i]);
505        cb_assert(r == 0);
506    }
507
508    wait_for_flusher_to_settle(h, h1);
509
510    testHarness.reload_engine(&h, &h1,
511                              testHarness.engine_path,
512                              testHarness.get_current_testcase()->cfg,
513                              true, false);
514    wait_for_warmup_complete(h, h1);
515
516    cb_assert(0 == get_int_stat(h, h1, "ep_warmed_dups"));
517
518    return SUCCESS;
519}
520
521static enum test_result test_set_get_hit(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
522    item *i = NULL;
523    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
524          "store failure");
525    check_key_value(h, h1, "key", "somevalue", 9);
526    h1->release(h, NULL, i);
527    return SUCCESS;
528}
529
530static enum test_result test_set_get_hit_bin(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
531    char binaryData[] = "abcdefg\0gfedcba";
532    cb_assert(sizeof(binaryData) != strlen(binaryData));
533
534    item *i = NULL;
535    check(ENGINE_SUCCESS ==
536          storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
537                       binaryData, sizeof(binaryData), 82758, &i, 0, 0),
538          "Failed to set.");
539    h1->release(h, NULL, i);
540    check_key_value(h, h1, "key", binaryData, sizeof(binaryData));
541    return SUCCESS;
542}
543
544static enum test_result test_set_with_cas_non_existent(ENGINE_HANDLE *h,
545                                                       ENGINE_HANDLE_V1 *h1) {
546    const char *key = "test_expiry_flush";
547    item *i = NULL;
548
549    check(h1->allocate(h, NULL, &i, key, strlen(key), 10, 0, 0,
550          PROTOCOL_BINARY_RAW_BYTES) == ENGINE_SUCCESS, "Allocation failed.");
551
552    Item *it = reinterpret_cast<Item*>(i);
553    it->setCas(1234);
554
555    uint64_t cas = 0;
556    check(h1->store(h, NULL, i, &cas, OPERATION_SET, 0) == ENGINE_KEY_ENOENT,
557          "Expected not found");
558    h1->release(h, NULL, i);
559
560    return SUCCESS;
561}
562
563static enum test_result test_set_change_flags(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
564    item *i = NULL;
565    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
566          "Failed to set.");
567    h1->release(h, NULL, i);
568
569    item_info info;
570    uint32_t flags = 828258;
571    check(get_item_info(h, h1, &info, "key"), "Failed to get value.");
572    cb_assert(info.flags != flags);
573
574    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
575                       "newvalue", strlen("newvalue"), flags, &i, 0, 0) == ENGINE_SUCCESS,
576          "Failed to set again.");
577    h1->release(h, NULL, i);
578
579    check(get_item_info(h, h1, &info, "key"), "Failed to get value.");
580
581    return info.flags == flags ? SUCCESS : FAIL;
582}
583
584static enum test_result test_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
585    item *i = NULL;
586    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
587          "Failed to do initial set.");
588    h1->release(h, NULL, i);
589    check(store(h, h1, NULL, OPERATION_CAS, "key", "failcas", &i) != ENGINE_SUCCESS,
590          "Failed to fail initial CAS.");
591    h1->release(h, NULL, i);
592    check_key_value(h, h1, "key", "somevalue", 9);
593
594    check(h1->get(h, NULL, &i, "key", 3, 0) == ENGINE_SUCCESS,
595          "Failed to get value.");
596
597    item_info info;
598    info.nvalue = 1;
599    check(h1->get_item_info(h, NULL, i, &info), "Failed to get item info.");
600    h1->release(h, NULL, i);
601
602    check(store(h, h1, NULL, OPERATION_CAS, "key", "winCas", &i,
603                info.cas) == ENGINE_SUCCESS,
604          "Failed to store CAS");
605    h1->release(h, NULL, i);
606    check_key_value(h, h1, "key", "winCas", 6);
607
608    uint64_t cval = 99999;
609    check(store(h, h1, NULL, OPERATION_CAS, "non-existing", "winCas", &i,
610                cval) == ENGINE_KEY_ENOENT,
611          "CAS for non-existing key returned the wrong error code");
612    h1->release(h, NULL, i);
613    return SUCCESS;
614}
615
616static enum test_result test_add(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
617    item *i = NULL;
618    check(store(h, h1, NULL, OPERATION_ADD,"key", "somevalue", &i) == ENGINE_SUCCESS,
619          "Failed to add value.");
620    h1->release(h, NULL, i);
621    check(store(h, h1, NULL, OPERATION_ADD,"key", "somevalue", &i) == ENGINE_NOT_STORED,
622          "Failed to fail to re-add value.");
623    h1->release(h, NULL, i);
624
625    // This aborts on failure.
626    check_key_value(h, h1, "key", "somevalue", 9);
627
628    // Expiration above was an hour, so let's go to The Future
629    testHarness.time_travel(3800);
630
631    check(store(h, h1, NULL, OPERATION_ADD,"key", "newvalue", &i) == ENGINE_SUCCESS,
632          "Failed to add value again.");
633    h1->release(h, NULL, i);
634    check_key_value(h, h1, "key", "newvalue", 8);
635    return SUCCESS;
636}
637
638static enum test_result test_add_add_with_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
639    item *i = NULL;
640    check(store(h, h1, NULL, OPERATION_ADD, "key",
641                "somevalue", &i) == ENGINE_SUCCESS,
642          "Failed set.");
643    check_key_value(h, h1, "key", "somevalue", 9);
644    item_info info;
645    info.nvalue = 1;
646    info.nvalue = 1;
647    check(h1->get_item_info(h, NULL, i, &info) == true,
648          "Should be able to get info");
649
650    item *i2 = NULL;
651    ENGINE_ERROR_CODE ret;
652    check((ret = store(h, h1, NULL, OPERATION_ADD, "key",
653                       "somevalue", &i2, info.cas)) == ENGINE_KEY_EEXISTS,
654          "Should not be able to add the key two times");
655
656    h1->release(h, NULL, i);
657    h1->release(h, NULL, i2);
658    return SUCCESS;
659}
660
661static enum test_result test_replace(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
662    item *i = NULL;
663    check(store(h, h1, NULL, OPERATION_REPLACE,"key", "somevalue", &i) != ENGINE_SUCCESS,
664          "Failed to fail to replace non-existing value.");
665    h1->release(h, NULL, i);
666    check(store(h, h1, NULL, OPERATION_SET,"key", "somevalue", &i) == ENGINE_SUCCESS,
667          "Failed to set value.");
668    h1->release(h, NULL, i);
669    check(store(h, h1, NULL, OPERATION_REPLACE,"key", "somevalue", &i) == ENGINE_SUCCESS,
670          "Failed to replace existing value.");
671    h1->release(h, NULL, i);
672    check_key_value(h, h1, "key", "somevalue", 9);
673    return SUCCESS;
674}
675
676static enum test_result test_replace_with_eviction(ENGINE_HANDLE *h,
677                                                   ENGINE_HANDLE_V1 *h1) {
678    item *i = NULL;
679    check(store(h, h1, NULL, OPERATION_SET,"key", "somevalue", &i) == ENGINE_SUCCESS,
680          "Failed to set value.");
681    h1->release(h, NULL, i);
682    wait_for_flusher_to_settle(h, h1);
683    evict_key(h, h1, "key");
684    int numBgFetched = get_int_stat(h, h1, "ep_bg_fetched");
685
686    check(store(h, h1, NULL, OPERATION_REPLACE,"key", "somevalue1", &i) == ENGINE_SUCCESS,
687          "Failed to replace existing value.");
688
689    check(h1->get_stats(h, NULL, NULL, 0, add_stats) == ENGINE_SUCCESS,
690          "Failed to get stats.");
691    std::string eviction_policy = vals.find("ep_item_eviction_policy")->second;
692    if (eviction_policy == "full_eviction") {
693        numBgFetched++;
694    }
695
696    check(get_int_stat(h, h1, "ep_bg_fetched") == numBgFetched,
697          "Bg fetched value didn't match");
698
699    h1->release(h, NULL, i);
700    check_key_value(h, h1, "key", "somevalue1", 10);
701    return SUCCESS;
702}
703
704static enum test_result test_incr_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
705    uint64_t cas = 0, result = 0;
706    h1->arithmetic(h, NULL, "key", 3, true, false, 1, 0, 0,
707                   &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
708                   0);
709    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected to not find key");
710    return SUCCESS;
711}
712
713static enum test_result test_incr_default(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
714    uint64_t cas = 0, result = 0;
715    check(h1->arithmetic(h, NULL, "key", 3, true, true, 1, 1, 0,
716                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
717                         0) == ENGINE_SUCCESS,
718          "Failed first arith");
719    check(result == 1, "Failed result verification.");
720
721    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
722                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
723                         0) == ENGINE_SUCCESS,
724          "Failed second arith.");
725    check(result == 2, "Failed second result verification.");
726
727    check(h1->arithmetic(h, NULL, "key", 3, true, true, 1, 1, 0,
728                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
729                         0) == ENGINE_SUCCESS,
730          "Failed third arith.");
731    check(result == 3, "Failed third result verification.");
732
733    check_key_value(h, h1, "key", "3", 1);
734    return SUCCESS;
735}
736
737static enum test_result test_append(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
738    item *i = NULL;
739
740    // MB-11332: append on non-existing key should return NOT_STORED
741    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key",
742                       "foo\r\n", 5, 82758, &i, 0, 0)
743          == ENGINE_NOT_STORED,
744          "MB-11332: Failed append.");
745
746    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
747                       "\r\n", 2, 82758, &i, 0, 0)
748          == ENGINE_SUCCESS,
749          "Failed set.");
750    h1->release(h, NULL, i);
751
752    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key",
753                       "foo\r\n", 5, 82758, &i, 0, 0)
754          == ENGINE_SUCCESS,
755          "Failed append.");
756    h1->release(h, NULL, i);
757
758    check_key_value(h, h1, "key", "\r\nfoo\r\n", 7);
759
760    char binaryData1[] = "abcdefg\0gfedcba\r\n";
761    char binaryData2[] = "abzdefg\0gfedcba\r\n";
762    size_t dataSize = 20*1024*1024;
763    char *bigBinaryData3 = new char[dataSize];
764    memset(bigBinaryData3, '\0', dataSize);
765
766    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
767                       binaryData1, sizeof(binaryData1) - 1, 82758, &i, 0, 0)
768          == ENGINE_SUCCESS,
769          "Failed set.");
770    h1->release(h, NULL, i);
771
772    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key",
773                       bigBinaryData3, dataSize, 82758, &i, 0, 0)
774          == ENGINE_E2BIG,
775          "Expected append failure.");
776    h1->release(h, NULL, i);
777    delete[] bigBinaryData3;
778
779    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key",
780                       binaryData2, sizeof(binaryData2) - 1, 82758, &i, 0, 0)
781          == ENGINE_SUCCESS,
782          "Failed append.");
783    h1->release(h, NULL, i);
784
785    std::string expected;
786    expected.append(binaryData1, sizeof(binaryData1) - 1);
787    expected.append(binaryData2, sizeof(binaryData2) - 1);
788
789    check_key_value(h, h1, "key", expected.data(), expected.length());
790    return SUCCESS;
791}
792
793static enum test_result test_prepend(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
794    item *i = NULL;
795
796    // MB-11332: prepend on non-existing key should return NOT_STORED
797    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key",
798                       "foo\r\n", 5, 82758, &i, 0, 0)
799          == ENGINE_NOT_STORED,
800          "MB-11332: Failed prepend.");
801
802
803    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
804                       "\r\n", 2, 82758, &i, 0, 0)
805          == ENGINE_SUCCESS,
806          "Failed set.");
807    h1->release(h, NULL, i);
808
809    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key",
810                       "foo\r\n", 5, 82758, &i, 0, 0)
811          == ENGINE_SUCCESS,
812          "Failed append.");
813    h1->release(h, NULL, i);
814
815    check_key_value(h, h1, "key", "foo\r\n\r\n", 7);
816
817    char binaryData1[] = "abcdefg\0gfedcba\r\n";
818    char binaryData2[] = "abzdefg\0gfedcba\r\n";
819    size_t dataSize = 20*1024*1024;
820    char *bigBinaryData3 = new char[dataSize];
821    memset(bigBinaryData3, '\0', dataSize);
822
823    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
824                       binaryData1, sizeof(binaryData1) - 1, 82758, &i, 0, 0)
825          == ENGINE_SUCCESS,
826          "Failed set.");
827    h1->release(h, NULL, i);
828
829    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key",
830                       bigBinaryData3, dataSize, 82758, &i, 0, 0)
831          == ENGINE_E2BIG,
832          "Expected prepend failure.");
833    h1->release(h, NULL, i);
834    delete[] bigBinaryData3;
835
836    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key",
837                       binaryData2, sizeof(binaryData2) - 1, 82758, &i, 0, 0)
838          == ENGINE_SUCCESS,
839          "Failed append.");
840    h1->release(h, NULL, i);
841
842    std::string expected;
843    expected.append(binaryData2, sizeof(binaryData2) - 1);
844    expected.append(binaryData1, sizeof(binaryData1) - 1);
845
846    check_key_value(h, h1, "key", expected.data(), expected.length());
847    return SUCCESS;
848}
849
850static enum test_result test_append_compressed(ENGINE_HANDLE *h,
851                                               ENGINE_HANDLE_V1 *h1) {
852
853    item *i = NULL;
854
855    size_t len = snappy_max_compressed_length(2);
856    char *newBuf = (char *) malloc (len);
857    snappy_compress("\r\n", 2, newBuf, &len);
858    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key1",
859                       (const char *)newBuf, len, 82758, &i, 0, 0, 3600,
860                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
861          == ENGINE_SUCCESS, "Failed set.");
862    h1->release(h, NULL, i);
863    free (newBuf);
864
865    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key1",
866                       "foo\r\n", 5, 82758, &i, 0, 0)
867          == ENGINE_SUCCESS,
868          "Failed append uncompressed to compressed.");
869    h1->release(h, NULL, i);
870
871    size_t len1 = snappy_max_compressed_length(7);
872    char *newBuf1 = (char *) malloc (len1);
873    snappy_compress("\r\nfoo\r\n", 7, newBuf1, &len1);
874
875    item_info info;
876    check(get_item_info(h, h1, &info, "key1", 0), "checking key and value");
877    check(info.nvalue == 1, "info.nvalue != 1");
878    check(len1 == info.value[0].iov_len, "Value length mismatch");
879    check(memcmp(info.value[0].iov_base, newBuf1, len1) == 0, "Data mismatch");
880    check(info.datatype == 0x02, "Datatype mismatch");
881    free (newBuf1);
882
883    len = snappy_max_compressed_length(3);
884    newBuf = (char *) malloc (len);
885    snappy_compress("bar", 3, newBuf, &len);
886    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key1",
887                       (const char*)newBuf, len, 82758, &i, 0, 0, 3600,
888                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
889            == ENGINE_SUCCESS,
890            "Failed append compressed to compressed.");
891    h1->release(h, NULL, i);
892    free (newBuf);
893
894    len1 = snappy_max_compressed_length(10);
895    newBuf1 = (char *) malloc (len1);
896    snappy_compress("\r\nfoo\r\nbar", 10, newBuf1, &len1);
897
898    check(get_item_info(h, h1, &info, "key1", 0), "checking key and value");
899    check(info.nvalue == 1, "info.nvalue != 1");
900    check(len1 == info.value[0].iov_len, "Value length mismatch");
901    check(memcmp(info.value[0].iov_base, newBuf1, len1) == 0, "Data mismatch");
902    check(info.datatype == 0x02, "Datatype mismatch");
903    free (newBuf1);
904
905    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key2",
906                       "foo", 3, 82758, &i, 0, 0, 3600,
907                       PROTOCOL_BINARY_RAW_BYTES)
908          == ENGINE_SUCCESS, "Failed set.");
909    h1->release(h, NULL, i);
910
911    len = snappy_max_compressed_length(3);
912    newBuf = (char *) malloc (len);
913    snappy_compress("bar", 3, newBuf, &len);
914    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key2",
915                       newBuf, len, 82758, &i, 0, 0, 3600,
916                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
917            == ENGINE_SUCCESS,
918            "Failed append compressed to uncompressed.");
919    h1->release(h, NULL, i);
920    free (newBuf);
921
922    check(get_item_info(h, h1, &info, "key2", 0), "checking key and value");
923    check(info.nvalue == 1, "info.nvalue != 1");
924    check(info.value[0].iov_len == 6, "Value length mismatch");
925    check(memcmp(info.value[0].iov_base, "foobar", 6) == 0, "Data mismatch");
926    check(info.datatype == 0x00, "Datatype mismatch");
927
928    return SUCCESS;
929}
930
931static enum test_result test_prepend_compressed(ENGINE_HANDLE *h,
932                                               ENGINE_HANDLE_V1 *h1) {
933
934    item *i = NULL;
935
936    size_t len = snappy_max_compressed_length(2);
937    char *newBuf = (char *) malloc (len);
938    snappy_compress("\r\n", 2, newBuf, &len);
939    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key1",
940                       (const char *)newBuf, len, 82758, &i, 0, 0, 3600,
941                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
942          == ENGINE_SUCCESS, "Failed set.");
943    h1->release(h, NULL, i);
944    free (newBuf);
945
946    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key1",
947                       "foo\r\n", 5, 82758, &i, 0, 0)
948          == ENGINE_SUCCESS,
949          "Failed prepend uncompressed to compressed.");
950    h1->release(h, NULL, i);
951
952    size_t len1 = snappy_max_compressed_length(7);
953    char *newBuf1 = (char *) malloc (len1);
954    snappy_compress("foo\r\n\r\n", 7, newBuf1, &len1);
955
956    item_info info;
957    check(get_item_info(h, h1, &info, "key1", 0), "checking key and value");
958    check(info.nvalue == 1, "info.nvalue != 1");
959    check(len1 == info.value[0].iov_len, "Value length mismatch");
960    check(memcmp(info.value[0].iov_base, newBuf1, len1) == 0, "Data mismatch");
961    check(info.datatype == 0x02, "Datatype mismatch");
962    free (newBuf1);
963
964    len = snappy_max_compressed_length(3);
965    newBuf = (char *) malloc (len);
966    snappy_compress("bar", 3, newBuf, &len);
967    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key1",
968                       (const char*)newBuf, len, 82758, &i, 0, 0, 3600,
969                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
970            == ENGINE_SUCCESS,
971            "Failed prepend compressed to compressed.");
972    h1->release(h, NULL, i);
973    free (newBuf);
974
975    len1 = snappy_max_compressed_length(10);
976    newBuf1 = (char *) malloc (len1);
977    snappy_compress("barfoo\r\n\r\n", 10, newBuf1, &len1);
978
979    check(get_item_info(h, h1, &info, "key1", 0), "checking key and value");
980    check(info.nvalue == 1, "info.nvalue != 1");
981    check(len1 == info.value[0].iov_len, "Value length mismatch");
982    check(memcmp(info.value[0].iov_base, newBuf1, len1) == 0, "Data mismatch");
983    check(info.datatype == 0x02, "Datatype mismatch");
984    free (newBuf1);
985
986    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key2",
987                       "foo", 3, 82758, &i, 0, 0, 3600,
988                       PROTOCOL_BINARY_RAW_BYTES)
989          == ENGINE_SUCCESS, "Failed set.");
990    h1->release(h, NULL, i);
991
992    len = snappy_max_compressed_length(3);
993    newBuf = (char *) malloc (len);
994    snappy_compress("bar", 3, newBuf, &len);
995    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key2",
996                       newBuf, len, 82758, &i, 0, 0, 3600,
997                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
998            == ENGINE_SUCCESS,
999            "Failed prepend compressed to uncompressed.");
1000    h1->release(h, NULL, i);
1001    free (newBuf);
1002
1003    check(get_item_info(h, h1, &info, "key2", 0), "checking key and value");
1004    check(info.nvalue == 1, "info.nvalue != 1");
1005    check(info.value[0].iov_len == 6, "Value length mismatch");
1006    check(memcmp(info.value[0].iov_base, "barfoo", 6) == 0, "Data mismatch");
1007    check(info.datatype == 0x00, "Datatype mismatch");
1008
1009    return SUCCESS;
1010}
1011
1012static enum test_result test_append_prepend_to_json(ENGINE_HANDLE *h,
1013                                                    ENGINE_HANDLE_V1 *h1) {
1014    item *i = NULL;
1015    item_info info;
1016
1017    const char* key1 = "foo1";
1018    const char* key2 = "foo2";
1019    const char* value1 = "{\"foo1\":\"bar1\"}";
1020    const char* value2 = "{\"foo2\":\"bar2\"}";
1021
1022    // APPEND
1023    check(storeCasVb11(h, h1, NULL, OPERATION_SET, key1,
1024                       value1, strlen(value1), 82758, &i, 0, 0,
1025                       3600, PROTOCOL_BINARY_DATATYPE_JSON)
1026          == ENGINE_SUCCESS, "Failed set.");
1027    h1->release(h, NULL, i);
1028
1029    check(h1->get(h, NULL, &i, key1, strlen(key1), 0) == ENGINE_SUCCESS,
1030            "Unable to get stored item");
1031    h1->release(h, NULL, i);
1032    info.nvalue = 1;
1033    h1->get_item_info(h, NULL, i, &info);
1034    check(checkUTF8JSON((const unsigned char*)info.value[0].iov_base,
1035                        (int)info.value[0].iov_len) == 1, "Expected JSON");
1036    check(info.datatype == PROTOCOL_BINARY_DATATYPE_JSON, "Invalid datatype");
1037
1038    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, key1,
1039                       value2, strlen(value2), 82758, &i, 0, 0)
1040          == ENGINE_SUCCESS,
1041          "Failed append.");
1042    h1->release(h, NULL, i);
1043
1044    check(h1->get(h, NULL, &i, key1, strlen(key1), 0) == ENGINE_SUCCESS,
1045            "Unable to get stored item");
1046    info.nvalue = 1;
1047    h1->get_item_info(h, NULL, i, &info);
1048    h1->release(h, NULL, i);
1049    check(checkUTF8JSON((const unsigned char*)info.value[0].iov_base,
1050                        (int)info.value[0].iov_len) == 0, "Expected Binary");
1051    check(info.datatype == PROTOCOL_BINARY_RAW_BYTES,
1052                "Invalid datatype after append");
1053
1054    // PREPEND
1055    check(storeCasVb11(h, h1, NULL, OPERATION_SET, key2,
1056                       value1, strlen(value1), 82758, &i, 0, 0,
1057                       3600, PROTOCOL_BINARY_DATATYPE_JSON)
1058          == ENGINE_SUCCESS, "Failed set.");
1059    h1->release(h, NULL, i);
1060
1061    check(h1->get(h, NULL, &i, key2, strlen(key2), 0) == ENGINE_SUCCESS,
1062            "Unable to get stored item");
1063    info.nvalue = 1;
1064    h1->get_item_info(h, NULL, i, &info);
1065    h1->release(h, NULL, i);
1066    check(checkUTF8JSON((const unsigned char*)info.value[0].iov_base,
1067                        (int)info.value[0].iov_len) == 1, "Expected JSON");
1068    check(info.datatype == PROTOCOL_BINARY_DATATYPE_JSON, "Invalid datatype");
1069
1070    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, key2,
1071                       value2, strlen(value2), 82758, &i, 0, 0)
1072          == ENGINE_SUCCESS,
1073          "Failed prepend.");
1074    h1->release(h, NULL, i);
1075
1076    check(h1->get(h, NULL, &i, key2, strlen(key2), 0) == ENGINE_SUCCESS,
1077            "Unable to get stored item");
1078    h1->release(h, NULL, i);
1079    info.nvalue = 1;
1080    h1->get_item_info(h, NULL, i, &info);
1081    check(checkUTF8JSON((const unsigned char*)info.value[0].iov_base,
1082                        (int)info.value[0].iov_len) == 0, "Expected Binary");
1083    check(info.datatype == PROTOCOL_BINARY_RAW_BYTES,
1084                "Invalid datatype after prepend");
1085
1086    return SUCCESS;
1087}
1088
1089static enum test_result test_incr(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1090    uint64_t cas = 0, result = 0;
1091    item *i = NULL;
1092    check(store(h, h1, NULL, OPERATION_ADD,"key", "1", &i) == ENGINE_SUCCESS,
1093          "Failed to add value.");
1094    h1->release(h, NULL, i);
1095
1096    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
1097                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1098                         0) == ENGINE_SUCCESS,
1099          "Failed to incr value.");
1100
1101    check_key_value(h, h1, "key", "2", 1);
1102    return SUCCESS;
1103}
1104
1105static enum test_result test_bug2799(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1106    uint64_t cas = 0, result = 0;
1107    item *i = NULL;
1108    check(store(h, h1, NULL, OPERATION_ADD, "key", "1", &i) == ENGINE_SUCCESS,
1109          "Failed to add value.");
1110    h1->release(h, NULL, i);
1111
1112    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
1113                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1114                         0) == ENGINE_SUCCESS,
1115          "Failed to incr value.");
1116
1117    check_key_value(h, h1, "key", "2", 1);
1118
1119    testHarness.time_travel(3617);
1120
1121    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1122    return SUCCESS;
1123}
1124
1125static enum test_result test_flush(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1126    item *i = NULL;
1127    // First try to delete something we know to not be there.
1128    check(del(h, h1, "key", 0, 0) == ENGINE_KEY_ENOENT, "Failed to fail initial delete.");
1129    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1130          "Failed set.");
1131    h1->release(h, NULL, i);
1132    check_key_value(h, h1, "key", "somevalue", 9);
1133    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS,
1134          "Failed to flush");
1135    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1136
1137    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1138          "Failed post-flush set.");
1139    h1->release(h, NULL, i);
1140    check_key_value(h, h1, "key", "somevalue", 9);
1141
1142    return SUCCESS;
1143}
1144
1145static enum test_result test_flush_disabled(ENGINE_HANDLE *h,
1146                                            ENGINE_HANDLE_V1 *h1) {
1147    item *i = NULL;
1148    // start an engine with disabled flush, the flush() should be noop and
1149    // we expect to see the key after flush()
1150
1151    // store a key and check its existence
1152    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1153          "Failed set.");
1154    h1->release(h, NULL, i);
1155    check_key_value(h, h1, "key", "somevalue", 9);
1156    // expect error msg engine does not support operation
1157    check(h1->flush(h, NULL, 0) == ENGINE_ENOTSUP, "Flush should be disabled");
1158    //check the key
1159    check(ENGINE_SUCCESS == verify_key(h, h1, "key"), "Expected key");
1160
1161    // restart engine with flush enabled and redo the test, we expect flush to succeed
1162    std::string param = "flushall_enabled=false";
1163    std::string config = testHarness.get_current_testcase()->cfg;
1164    size_t found = config.find(param);
1165    if(found != config.npos) {
1166        config.replace(found, param.size(), "flushall_enabled=true");
1167    }
1168    testHarness.reload_engine(&h, &h1,
1169                              testHarness.engine_path,
1170                              config.c_str(),
1171                              true, false);
1172    wait_for_warmup_complete(h, h1);
1173
1174    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS, "Flush should be enabled");
1175
1176    //expect missing key
1177    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1178
1179    return SUCCESS;
1180}
1181
1182static enum test_result test_flush_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1183    item *i = NULL;
1184    int mem_used = get_int_stat(h, h1, "mem_used");
1185    int overhead = get_int_stat(h, h1, "ep_overhead");
1186    int cacheSize = get_int_stat(h, h1, "ep_total_cache_size");
1187    int nonResident = get_int_stat(h, h1, "ep_num_non_resident");
1188
1189    int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
1190    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1191          "Failed set.");
1192    h1->release(h, NULL, i);
1193    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i) == ENGINE_SUCCESS,
1194          "Failed set.");
1195    h1->release(h, NULL, i);
1196    testHarness.time_travel(65);
1197    wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
1198
1199    check(ENGINE_SUCCESS == verify_key(h, h1, "key"), "Expected key");
1200    check(ENGINE_SUCCESS == verify_key(h, h1, "key2"), "Expected key2");
1201
1202    check_key_value(h, h1, "key", "somevalue", 9);
1203    check_key_value(h, h1, "key2", "somevalue", 9);
1204
1205    int mem_used2 = get_int_stat(h, h1, "mem_used");
1206    int overhead2 = get_int_stat(h, h1, "ep_overhead");
1207    int cacheSize2 = get_int_stat(h, h1, "ep_total_cache_size");
1208
1209    cb_assert(mem_used2 > mem_used);
1210    // "mem_used2 - overhead2" (i.e., ep_kv_size) should be greater than the hashtable cache size
1211    // due to the checkpoint overhead
1212    cb_assert(mem_used2 - overhead2 > cacheSize2);
1213
1214    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS, "Failed to flush");
1215    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1216    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key2"), "Expected missing key");
1217
1218    wait_for_flusher_to_settle(h, h1);
1219
1220    mem_used2 = get_int_stat(h, h1, "mem_used");
1221    overhead2 = get_int_stat(h, h1, "ep_overhead");
1222    cacheSize2 = get_int_stat(h, h1, "ep_total_cache_size");
1223    int nonResident2 = get_int_stat(h, h1, "ep_num_non_resident");
1224
1225    cb_assert(mem_used2 == mem_used);
1226    cb_assert(overhead2 == overhead);
1227    cb_assert(nonResident2 == nonResident);
1228    cb_assert(cacheSize2 == cacheSize);
1229
1230    return SUCCESS;
1231}
1232
1233static enum test_result test_flush_multiv(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1234    item *i = NULL;
1235    check(set_vbucket_state(h, h1, 2, vbucket_state_active), "Failed to set vbucket state.");
1236    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1237          "Failed set.");
1238    h1->release(h, NULL, i);
1239    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i,
1240                0, 2) == ENGINE_SUCCESS,
1241          "Failed set in vb2.");
1242    h1->release(h, NULL, i);
1243
1244    check(ENGINE_SUCCESS == verify_key(h, h1, "key"), "Expected key");
1245    check(ENGINE_SUCCESS == verify_key(h, h1, "key2", 2), "Expected key2");
1246
1247    check_key_value(h, h1, "key", "somevalue", 9);
1248    check_key_value(h, h1, "key2", "somevalue", 9, 2);
1249
1250    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS, "Failed to flush");
1251
1252    vals.clear();
1253    check(h1->get_stats(h, NULL, NULL, 0, add_stats) == ENGINE_SUCCESS,
1254          "Failed to get stats.");
1255    check(vals.find("ep_flush_all") != vals.end(), "Failed to get the status of flush_all");
1256
1257    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1258    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key2", 2), "Expected missing key");
1259
1260    return SUCCESS;
1261}
1262
1263static int checkCurrItemsAfterShutdown(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1264                                       int numItems2Load, bool shutdownForce) {
1265    std::vector<std::string> keys;
1266    for (int index = 0; index < numItems2Load; ++index) {
1267        std::stringstream s;
1268        s << "keys_2_load-" << index;
1269        std::string key(s.str());
1270        keys.push_back(key);
1271    }
1272
1273    check(get_int_stat(h, h1, "ep_total_persisted") == 0,
1274          "Expected ep_total_persisted equals 0");
1275    check(get_int_stat(h, h1, "curr_items") == 0,
1276          "Expected curr_items equals 0");
1277
1278    // stop flusher before loading new items
1279    protocol_binary_request_header *pkt = createPacket(PROTOCOL_BINARY_CMD_STOP_PERSISTENCE);
1280    check(h1->unknown_command(h, NULL, pkt, add_response) == ENGINE_SUCCESS,
1281          "CMD_STOP_PERSISTENCE failed!");
1282    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
1283          "Failed to stop persistence!");
1284    free(pkt);
1285
1286    std::vector<std::string>::iterator itr;
1287    for (itr = keys.begin(); itr != keys.end(); ++itr) {
1288        item *i;
1289        check(store(h, h1, NULL, OPERATION_SET, itr->c_str(), "oracle", &i, 0, 0)
1290              == ENGINE_SUCCESS, "Failed to store a value");
1291        h1->release(h, NULL, i);
1292    }
1293
1294    check(get_int_stat(h, h1, "ep_total_persisted") == 0,
1295          "Incorrect ep_total_persisted, expected 0");
1296    std::stringstream ss;
1297    ss << "Incorrect curr_items, expected " << numItems2Load;
1298    std::string errmsg(ss.str());
1299    check(get_int_stat(h, h1, "curr_items") == numItems2Load,
1300          errmsg.c_str());
1301
1302    // resume flusher before shutdown + warmup
1303    pkt = createPacket(PROTOCOL_BINARY_CMD_START_PERSISTENCE);
1304    check(h1->unknown_command(h, NULL, pkt, add_response) == ENGINE_SUCCESS,
1305          "CMD_START_PERSISTENCE failed!");
1306    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
1307          "Failed to start persistence!");
1308    free(pkt);
1309
1310    // shutdown engine force and restart
1311    testHarness.reload_engine(&h, &h1,
1312                              testHarness.engine_path,
1313                              testHarness.get_current_testcase()->cfg,
1314                              true, shutdownForce);
1315    wait_for_warmup_complete(h, h1);
1316    return get_int_stat(h, h1, "curr_items");
1317}
1318
1319static enum test_result test_flush_shutdown_force(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1320    int numItems2load = 3000;
1321    bool shutdownForce = true;
1322    int currItems = checkCurrItemsAfterShutdown(h, h1, numItems2load, shutdownForce);
1323    check (currItems <= numItems2load,
1324           "Number of curr items should be <= 3000, unless previous "
1325           "shutdown force had to wait for the flusher");
1326    return SUCCESS;
1327}
1328
1329static enum test_result test_flush_shutdown_noforce(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1330    int numItems2load = 3000;
1331    bool shutdownForce = false;
1332    int currItems = checkCurrItemsAfterShutdown(h, h1, numItems2load, shutdownForce);
1333    check (currItems == numItems2load,
1334           "Number of curr items should be equal to 3000, unless previous "
1335           "shutdown did not wait for the flusher");
1336    return SUCCESS;
1337}
1338
1339static enum test_result test_flush_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1340    item *i = NULL;
1341    // First try to delete something we know to not be there.
1342    check(del(h, h1, "key", 0, 0) == ENGINE_KEY_ENOENT, "Failed to fail initial delete.");
1343    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1344          "Failed set.");
1345    h1->release(h, NULL, i);
1346    check_key_value(h, h1, "key", "somevalue", 9);
1347
1348    // Restart once to ensure written to disk.
1349    testHarness.reload_engine(&h, &h1,
1350                              testHarness.engine_path,
1351                              testHarness.get_current_testcase()->cfg,
1352                              true, false);
1353    wait_for_warmup_complete(h, h1);
1354
1355    // Read value from disk.
1356    check_key_value(h, h1, "key", "somevalue", 9);
1357
1358    // Flush
1359    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS,
1360          "Failed to flush");
1361
1362    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i) == ENGINE_SUCCESS,
1363          "Failed post-flush set.");
1364    h1->release(h, NULL, i);
1365    check_key_value(h, h1, "key2", "somevalue", 9);
1366
1367    // Restart again, ensure written to disk.
1368    testHarness.reload_engine(&h, &h1,
1369                              testHarness.engine_path,
1370                              testHarness.get_current_testcase()->cfg,
1371                              true, false);
1372    wait_for_warmup_complete(h, h1);
1373
1374    check(store(h, h1, NULL, OPERATION_SET, "key3", "somevalue", &i) == ENGINE_SUCCESS,
1375          "Failed post-flush, post-restart set.");
1376    h1->release(h, NULL, i);
1377    check_key_value(h, h1, "key3", "somevalue", 9);
1378
1379    // Read value again, should not be there.
1380    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1381    return SUCCESS;
1382}
1383
1384static enum test_result test_flush_multiv_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1385    item *i = NULL;
1386    check(set_vbucket_state(h, h1, 2, vbucket_state_active), "Failed to set vbucket state.");
1387    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1388          "Failed set.");
1389    h1->release(h, NULL, i);
1390    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i,
1391                0, 2) == ENGINE_SUCCESS,
1392          "Failed set in vb2.");
1393    h1->release(h, NULL, i);
1394
1395    // Restart once to ensure written to disk.
1396    testHarness.reload_engine(&h, &h1,
1397                              testHarness.engine_path,
1398                              testHarness.get_current_testcase()->cfg,
1399                              true, false);
1400    wait_for_warmup_complete(h, h1);
1401
1402    // Read value from disk.
1403    check_key_value(h, h1, "key", "somevalue", 9);
1404
1405    // Flush
1406    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS,
1407          "Failed to flush");
1408
1409    // Restart again, ensure written to disk.
1410    testHarness.reload_engine(&h, &h1,
1411                              testHarness.engine_path,
1412                              testHarness.get_current_testcase()->cfg,
1413                              true, false);
1414    wait_for_warmup_complete(h, h1);
1415
1416    // Read value again, should not be there.
1417    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1418    check(verify_vbucket_missing(h, h1, 2), "Bucket 2 came back.");
1419    return SUCCESS;
1420}
1421
1422static enum test_result test_delete(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1423    item *i = NULL;
1424    // First try to delete something we know to not be there.
1425    check(del(h, h1, "key", 0, 0) == ENGINE_KEY_ENOENT, "Failed to fail initial delete.");
1426    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1427          "Failed set.");
1428    Item *it = reinterpret_cast<Item*>(i);
1429    uint64_t orig_cas = it->getCas();
1430    h1->release(h, NULL, i);
1431    check_key_value(h, h1, "key", "somevalue", 9);
1432
1433    uint64_t cas = 0;
1434    check(h1->remove(h, NULL, "key", 3, &cas, 0) == ENGINE_SUCCESS,
1435          "Failed remove with value.");
1436    check(orig_cas + 1 == cas, "Cas mismatch on delete");
1437    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1438
1439    // Can I time travel to an expired object and delete it?
1440    checkeq(ENGINE_SUCCESS, store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1441            "Failed set.");
1442    h1->release(h, NULL, i);
1443    testHarness.time_travel(3617);
1444    checkeq(ENGINE_KEY_ENOENT, del(h, h1, "key", 0, 0),
1445            "Did not get ENOENT removing an expired object.");
1446    checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"), "Expected missing key");
1447
1448    return SUCCESS;
1449}
1450
1451static enum test_result test_set_delete(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1452    item *i = NULL;
1453    checkeq(ENGINE_SUCCESS, store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1454            "Failed set.");
1455    h1->release(h, NULL, i);
1456    check_key_value(h, h1, "key", "somevalue", 9);
1457    checkeq(ENGINE_SUCCESS, del(h, h1, "key", 0, 0),
1458            "Failed remove with value.");
1459    checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"), "Expected missing key");
1460    wait_for_flusher_to_settle(h, h1);
1461    wait_for_stat_to_be(h, h1, "curr_items", 0);
1462    return SUCCESS;
1463}
1464
1465static enum test_result test_set_delete_invalid_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1466    item *i = NULL;
1467    check(store(h, h1, NULL, OPERATION_SET, "key",
1468                "somevalue", &i) == ENGINE_SUCCESS,
1469          "Failed set.");
1470    check_key_value(h, h1, "key", "somevalue", 9);
1471    item_info info;
1472    info.nvalue = 1;
1473    check(h1->get_item_info(h, NULL, i, &info) == true,
1474          "Should be able to get info");
1475    h1->release(h, NULL, i);
1476
1477    check(del(h, h1, "key", info.cas + 1, 0) == ENGINE_KEY_EEXISTS,
1478          "Didn't expect to be able to remove the item with wrong cas");
1479    return SUCCESS;
1480}
1481
1482static enum test_result test_bug2509(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1483    for (int j = 0; j < 10000; ++j) {
1484        item *itm = NULL;
1485        checkeq(ENGINE_SUCCESS,
1486                store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &itm),
1487                "Failed set.");
1488        h1->release(h, NULL, itm);
1489        usleep(10);
1490        checkeq(ENGINE_SUCCESS, del(h, h1, "key", 0, 0), "Failed remove with value.");
1491        usleep(10);
1492    }
1493
1494    // Restart again, to verify we don't have any duplicates.
1495    testHarness.reload_engine(&h, &h1,
1496                              testHarness.engine_path,
1497                              testHarness.get_current_testcase()->cfg,
1498                              true, false);
1499    wait_for_warmup_complete(h, h1);
1500
1501    return get_int_stat(h, h1, "ep_warmup_dups") == 0 ? SUCCESS : FAIL;
1502}
1503
1504static enum test_result test_bug7023(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1505    std::vector<std::string> keys;
1506    // Make a vbucket mess.
1507    for (int j = 0; j < 10000; ++j) {
1508        std::stringstream ss;
1509        ss << "key" << j;
1510        std::string key(ss.str());
1511        keys.push_back(key);
1512    }
1513
1514    std::vector<std::string>::iterator it;
1515    for (int j = 0; j < 5; ++j) {
1516        check(set_vbucket_state(h, h1, 0, vbucket_state_dead), "Failed set set vbucket 0 dead.");
1517        vbucketDelete(h, h1, 0);
1518        check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
1519              "Expected vbucket deletion to work.");
1520        check(set_vbucket_state(h, h1, 0, vbucket_state_active), "Failed set set vbucket 0 active.");
1521        for (it = keys.begin(); it != keys.end(); ++it) {
1522            item *i;
1523            check(store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str(), &i)
1524                  == ENGINE_SUCCESS, "Failed to store a value");
1525        }
1526    }
1527    wait_for_flusher_to_settle(h, h1);
1528
1529    // Restart again, to verify no data loss.
1530    testHarness.reload_engine(&h, &h1,
1531                              testHarness.engine_path,
1532                              testHarness.get_current_testcase()->cfg,
1533                              true, false);
1534    wait_for_warmup_complete(h, h1);
1535    return get_int_stat(h, h1, "ep_warmup_value_count", "warmup") == 10000 ? SUCCESS : FAIL;
1536}
1537
1538static enum test_result test_delete_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1539    wait_for_persisted_value(h, h1, "key", "value1");
1540
1541    check(del(h, h1, "key", 0, 0) == ENGINE_SUCCESS, "Failed remove with value.");
1542
1543    wait_for_persisted_value(h, h1, "key", "value2");
1544
1545    testHarness.reload_engine(&h, &h1,
1546                              testHarness.engine_path,
1547                              testHarness.get_current_testcase()->cfg,
1548                              true, false);
1549    wait_for_warmup_complete(h, h1);
1550
1551    check_key_value(h, h1, "key", "value2", 6);
1552    check(del(h, h1, "key", 0, 0) == ENGINE_SUCCESS, "Failed remove with value.");
1553    wait_for_flusher_to_settle(h, h1);
1554
1555    testHarness.reload_engine(&h, &h1,
1556                              testHarness.engine_path,
1557                              testHarness.get_current_testcase()->cfg,
1558                              true, false);
1559    wait_for_warmup_complete(h, h1);
1560
1561    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1562
1563    return SUCCESS;
1564}
1565
1566static enum test_result test_get_delete_missing_file(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1567    const char *key = "key";
1568    wait_for_persisted_value(h, h1, key, "value2delete");
1569
1570    // whack the db file and directory where the key is stored
1571    rmdb();
1572
1573    item *i = NULL;
1574    int errorCode = h1->get(h, NULL, &i, key, strlen(key), 0);
1575    h1->release(h, NULL, i);
1576
1577    // ep engine must be unaware of well-being of the db file as long as
1578    // the item is still in the memory
1579    check(errorCode == ENGINE_SUCCESS, "Expected success for get");
1580
1581    i = NULL;
1582    evict_key(h, h1, key);
1583    errorCode = h1->get(h, NULL, &i, key, strlen(key), 0);
1584    h1->release(h, NULL, i);
1585
1586    // ep engine must be now aware of the ill-fated db file where
1587    // the item is supposedly stored
1588    check(errorCode == ENGINE_TMPFAIL, "Expected tmp fail for get");
1589
1590    return SUCCESS;
1591}
1592
1593
1594static enum test_result test_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1595    item *i = NULL;
1596    static const char val[] = "somevalue";
1597    ENGINE_ERROR_CODE ret;
1598    check((ret = store(h, h1, NULL, OPERATION_SET, "key", val, &i)) == ENGINE_SUCCESS,
1599          "Failed set.");
1600    h1->release(h, NULL, i);
1601
1602    testHarness.reload_engine(&h, &h1,
1603                              testHarness.engine_path,
1604                              testHarness.get_current_testcase()->cfg,
1605                              true, false);
1606    wait_for_warmup_complete(h, h1);
1607    check_key_value(h, h1, "key", val, strlen(val));
1608    return SUCCESS;
1609}
1610
1611static enum test_result test_restart_session_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1612    createTapConn(h, h1, "tap_client_thread");
1613
1614    testHarness.reload_engine(&h, &h1,
1615                              testHarness.engine_path,
1616                              testHarness.get_current_testcase()->cfg,
1617                              true, false);
1618    wait_for_warmup_complete(h, h1);
1619    createTapConn(h, h1, "tap_client_thread");
1620
1621    check(h1->get_stats(h, NULL, "tap", 3, add_stats) == ENGINE_SUCCESS,
1622          "Failed to get stats.");
1623    std::string val = vals["eq_tapq:tap_client_thread:backfill_completed"];
1624    check(strcmp(val.c_str(), "true") == 0, "Don't expect the backfill upon restart");
1625    return SUCCESS;
1626}
1627
1628static enum test_result test_specialKeys(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1629    item *i = NULL;
1630    ENGINE_ERROR_CODE ret;
1631
1632    // Simplified Chinese "Couchbase"
1633    static const char key0[] = "沙发数据库";
1634    static const char val0[] = "some Chinese value";
1635    check((ret = store(h, h1, NULL, OPERATION_SET, key0, val0, &i)) == ENGINE_SUCCESS,
1636          "Failed set Chinese key");
1637    check_key_value(h, h1, key0, val0, strlen(val0));
1638    h1->release(h, NULL, i);
1639    // Traditional Chinese "Couchbase"
1640    static const char key1[] = "沙發數據庫";
1641    static const char val1[] = "some Traditional Chinese value";
1642    check((ret = store(h, h1, NULL, OPERATION_SET, key1, val1, &i)) == ENGINE_SUCCESS,
1643          "Failed set Traditional Chinese key");
1644    h1->release(h, NULL, i);
1645    // Korean "couch potato"
1646    static const char key2[] = "쇼파감자";
1647    static const char val2[] = "some Korean value";
1648    check((ret = store(h, h1, NULL, OPERATION_SET, key2, val2, &i)) == ENGINE_SUCCESS,
1649          "Failed set Korean key");
1650    h1->release(h, NULL, i);
1651    // Russian "couch potato"
1652    static const char key3[] = "лодырь, лентяй";
1653    static const char val3[] = "some Russian value";
1654    check((ret = store(h, h1, NULL, OPERATION_SET, key3, val3, &i)) == ENGINE_SUCCESS,
1655          "Failed set Russian key");
1656    h1->release(h, NULL, i);
1657    // Japanese "couch potato"
1658    static const char key4[] = "カウチポテト";
1659    static const char val4[] = "some Japanese value";
1660    check((ret = store(h, h1, NULL, OPERATION_SET, key4, val4, &i)) == ENGINE_SUCCESS,
1661          "Failed set Japanese key");
1662    h1->release(h, NULL, i);
1663    // Indian char key, and no idea what it is
1664    static const char key5[] = "हरियानवी";
1665    static const char val5[] = "some Indian value";
1666    check((ret = store(h, h1, NULL, OPERATION_SET, key5, val5, &i)) == ENGINE_SUCCESS,
1667          "Failed set Indian key");
1668    h1->release(h, NULL, i);
1669    // Portuguese translation "couch potato"
1670    static const char key6[] = "sedentário";
1671    static const char val6[] = "some Portuguese value";
1672    check((ret = store(h, h1, NULL, OPERATION_SET, key6, val6, &i)) == ENGINE_SUCCESS,
1673          "Failed set Portuguese key");
1674    h1->release(h, NULL, i);
1675    // Arabic translation "couch potato"
1676    static const char key7[] = "الحافلةالبطاطة";
1677    static const char val7[] = "some Arabic value";
1678    check((ret = store(h, h1, NULL, OPERATION_SET, key7, val7, &i)) == ENGINE_SUCCESS,
1679          "Failed set Arabic key");
1680    h1->release(h, NULL, i);
1681
1682    testHarness.reload_engine(&h, &h1,
1683                              testHarness.engine_path,
1684                              testHarness.get_current_testcase()->cfg,
1685                              true, false);
1686    wait_for_warmup_complete(h, h1);
1687    check_key_value(h, h1, key0, val0, strlen(val0));
1688    check_key_value(h, h1, key1, val1, strlen(val1));
1689    check_key_value(h, h1, key2, val2, strlen(val2));
1690    check_key_value(h, h1, key3, val3, strlen(val3));
1691    check_key_value(h, h1, key4, val4, strlen(val4));
1692    check_key_value(h, h1, key5, val5, strlen(val5));
1693    check_key_value(h, h1, key6, val6, strlen(val6));
1694    check_key_value(h, h1, key7, val7, strlen(val7));
1695    return SUCCESS;
1696}
1697
1698static enum test_result test_binKeys(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1699    item *i = NULL;
1700    ENGINE_ERROR_CODE ret;
1701
1702    // binary key with char values beyond 0x7F
1703    static const char key0[] = "\xe0\xed\xf1\x6f\x7f\xf8\xfa";
1704    static const char val0[] = "some value val8";
1705    check((ret = store(h, h1, NULL, OPERATION_SET, key0, val0, &i)) == ENGINE_SUCCESS,
1706          "Failed set binary key0");
1707    check_key_value(h, h1, key0, val0, strlen(val0));
1708    h1->release(h, NULL, i);
1709    // binary keys with char values beyond 0x7F
1710    static const char key1[] = "\xf1\xfd\xfe\xff\xf0\xf8\xef";
1711    static const char val1[] = "some value val9";
1712    check((ret = store(h, h1, NULL, OPERATION_SET, key1, val1, &i)) == ENGINE_SUCCESS,
1713          "Failed set binary key1");
1714    check_key_value(h, h1, key1, val1, strlen(val1));
1715    h1->release(h, NULL, i);
1716    // binary keys with special utf-8 BOM (Byte Order Mark) values 0xBB 0xBF 0xEF
1717    static const char key2[] = "\xff\xfe\xbb\xbf\xef";
1718    static const char val2[] = "some utf-8 bom value";
1719    check((ret = store(h, h1, NULL, OPERATION_SET, key2, val2, &i)) == ENGINE_SUCCESS,
1720          "Failed set binary utf-8 bom key");
1721    check_key_value(h, h1, key2, val2, strlen(val2));
1722    h1->release(h, NULL, i);
1723    // binary keys with special utf-16BE BOM values "U+FEFF"
1724    static const char key3[] = "U+\xfe\xff\xefU+\xff\xfe";
1725    static const char val3[] = "some utf-16 bom value";
1726    check((ret = store(h, h1, NULL, OPERATION_SET, key3, val3, &i)) == ENGINE_SUCCESS,
1727          "Failed set binary utf-16 bom key");
1728    check_key_value(h, h1, key3, val3, strlen(val3));
1729    h1->release(h, NULL, i);
1730
1731    testHarness.reload_engine(&h, &h1,
1732                              testHarness.engine_path,
1733                              testHarness.get_current_testcase()->cfg,
1734                              true, false);
1735    wait_for_warmup_complete(h, h1);
1736    check_key_value(h, h1, key0, val0, strlen(val0));
1737    check_key_value(h, h1, key1, val1, strlen(val1));
1738    check_key_value(h, h1, key2, val2, strlen(val2));
1739    check_key_value(h, h1, key3, val3, strlen(val3));
1740    return SUCCESS;
1741}
1742
1743static enum test_result test_restart_bin_val(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1744
1745
1746
1747    char binaryData[] = "abcdefg\0gfedcba";
1748    cb_assert(sizeof(binaryData) != strlen(binaryData));
1749
1750    item *i = NULL;
1751    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
1752                       binaryData, sizeof(binaryData), 82758, &i, 0, 0)
1753          == ENGINE_SUCCESS,
1754          "Failed set.");
1755    h1->release(h, NULL, i);
1756
1757    testHarness.reload_engine(&h, &h1,
1758                              testHarness.engine_path,
1759                              testHarness.get_current_testcase()->cfg,
1760                              true, false);
1761    wait_for_warmup_complete(h, h1);
1762
1763    check_key_value(h, h1, "key", binaryData, sizeof(binaryData));
1764    return SUCCESS;
1765}
1766
1767static enum test_result test_wrong_vb_get(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1768    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1769    check(ENGINE_NOT_MY_VBUCKET == verify_key(h, h1, "key", 1),
1770          "Expected wrong bucket.");
1771    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1772    return SUCCESS;
1773}
1774
1775static enum test_result test_vb_get_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1776    check(set_vbucket_state(h, h1, 1, vbucket_state_pending), "Failed to set vbucket state.");
1777    const void *cookie = testHarness.create_cookie();
1778    testHarness.set_ewouldblock_handling(cookie, false);
1779
1780    item *i = NULL;
1781    check(ENGINE_EWOULDBLOCK == h1->get(h, cookie, &i, "key", strlen("key"), 1),
1782          "Expected woodblock.");
1783    h1->release(h, NULL, i);
1784
1785    testHarness.destroy_cookie(cookie);
1786    return SUCCESS;
1787}
1788
1789static enum test_result test_vb_get_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1790    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set vbucket state.");
1791    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1792    check(ENGINE_NOT_MY_VBUCKET == verify_key(h, h1, "key", 1),
1793          "Expected not my bucket.");
1794    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1795    return SUCCESS;
1796}
1797
1798static enum test_result test_wrong_vb_incr(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1799    uint64_t cas, result;
1800    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1801    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
1802                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1803                         1) == ENGINE_NOT_MY_VBUCKET,
1804          "Expected not my vbucket.");
1805    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1806    return SUCCESS;
1807}
1808
1809static enum test_result test_vb_incr_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1810    const void *cookie = testHarness.create_cookie();
1811    testHarness.set_ewouldblock_handling(cookie, false);
1812    uint64_t cas, result;
1813    check(set_vbucket_state(h, h1, 1, vbucket_state_pending), "Failed to set vbucket state.");
1814    check(h1->arithmetic(h, cookie, "key", 3, true, false, 1, 1, 0,
1815                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1816                         1) == ENGINE_EWOULDBLOCK,
1817          "Expected woodblock.");
1818    testHarness.destroy_cookie(cookie);
1819    return SUCCESS;
1820}
1821
1822static enum test_result test_vb_incr_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1823    uint64_t cas, result;
1824    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set vbucket state.");
1825    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1826    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
1827                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1828                         1) == ENGINE_NOT_MY_VBUCKET,
1829          "Expected not my bucket.");
1830    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1831    return SUCCESS;
1832}
1833
1834static enum test_result test_wrong_vb_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1835    return test_wrong_vb_mutation(h, h1, OPERATION_SET);
1836}
1837
1838static enum test_result test_wrong_vb_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1839    return test_wrong_vb_mutation(h, h1, OPERATION_CAS);
1840}
1841
1842static enum test_result test_wrong_vb_add(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1843    return test_wrong_vb_mutation(h, h1, OPERATION_ADD);
1844}
1845
1846static enum test_result test_wrong_vb_replace(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1847    return test_wrong_vb_mutation(h, h1, OPERATION_REPLACE);
1848}
1849
1850static enum test_result test_wrong_vb_append(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1851    return test_wrong_vb_mutation(h, h1, OPERATION_APPEND);
1852}
1853
1854static enum test_result test_wrong_vb_prepend(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1855    return test_wrong_vb_mutation(h, h1, OPERATION_PREPEND);
1856}
1857
1858static enum test_result test_wrong_vb_del(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1859    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1860    check(ENGINE_NOT_MY_VBUCKET == del(h, h1, "key", 0, 1), "Expected wrong bucket.");
1861    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1862    return SUCCESS;
1863}
1864
1865static enum test_result test_expiry(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1866    const char *key = "test_expiry";
1867    const char *data = "some test data here.";
1868
1869    item *it = NULL;
1870
1871    ENGINE_ERROR_CODE rv;
1872    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 2,
1873                      PROTOCOL_BINARY_RAW_BYTES);
1874    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1875
1876    item_info info;
1877    info.nvalue = 1;
1878    if (!h1->get_item_info(h, NULL, it, &info)) {
1879        abort();
1880    }
1881    memcpy(info.value[0].iov_base, data, strlen(data));
1882
1883    uint64_t cas = 0;
1884    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
1885    check(rv == ENGINE_SUCCESS, "Set failed.");
1886    check_key_value(h, h1, key, data, strlen(data));
1887    h1->release(h, NULL, it);
1888
1889    testHarness.time_travel(5);
1890    check(h1->get(h, NULL, &it, key, strlen(key), 0) == ENGINE_KEY_ENOENT,
1891          "Item didn't expire");
1892
1893    int expired_access = get_int_stat(h, h1, "ep_expired_access");
1894    int expired_pager = get_int_stat(h, h1, "ep_expired_pager");
1895    int active_expired = get_int_stat(h, h1, "vb_active_expired");
1896    check(expired_pager == 0, "Expected zero expired item by pager");
1897    check(expired_access == 1, "Expected an expired item on access");
1898    check(active_expired == 1, "Expected an expired active item");
1899    checkeq(ENGINE_SUCCESS, store(h, h1, NULL, OPERATION_SET, key, data, &it),
1900            "Failed set.");
1901    h1->release(h, NULL, it);
1902
1903    std::stringstream ss;
1904    ss << "curr_items stat should be still 1 after ";
1905    ss << "overwriting the key that was expired, but not purged yet";
1906    checkeq(1, get_int_stat(h, h1, "curr_items"), ss.str().c_str());
1907
1908    return SUCCESS;
1909}
1910
1911static enum test_result test_expiry_loader(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1912    const char *key = "test_expiry_loader";
1913    const char *data = "some test data here.";
1914
1915    item *it = NULL;
1916
1917    ENGINE_ERROR_CODE rv;
1918    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 2,
1919                      PROTOCOL_BINARY_RAW_BYTES);
1920    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1921
1922    item_info info;
1923    info.nvalue = 1;
1924    if (!h1->get_item_info(h, NULL, it, &info)) {
1925        abort();
1926    }
1927    memcpy(info.value[0].iov_base, data, strlen(data));
1928
1929    uint64_t cas = 0;
1930    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
1931    check(rv == ENGINE_SUCCESS, "Set failed.");
1932    check_key_value(h, h1, key, data, strlen(data));
1933    h1->release(h, NULL, it);
1934
1935    testHarness.time_travel(3);
1936
1937    check(h1->get(h, NULL, &it, key, strlen(key), 0) == ENGINE_KEY_ENOENT,
1938          "Item didn't expire");
1939
1940    // Restart the engine to ensure the above expired item is not loaded
1941    testHarness.reload_engine(&h, &h1,
1942                              testHarness.engine_path,
1943                              testHarness.get_current_testcase()->cfg,
1944                              true, false);
1945    wait_for_warmup_complete(h, h1);
1946    cb_assert(0 == get_int_stat(h, h1, "ep_warmup_value_count", "warmup"));
1947
1948    return SUCCESS;
1949}
1950
1951static enum test_result test_expiration_on_warmup(ENGINE_HANDLE *h,
1952                                                  ENGINE_HANDLE_V1 *h1) {
1953    const char *key = "KEY";
1954    const char *data = "VALUE";
1955
1956    item *it = NULL;
1957
1958    ENGINE_ERROR_CODE rv;
1959    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 3,
1960                      PROTOCOL_BINARY_RAW_BYTES);
1961    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1962
1963    item_info info;
1964    info.nvalue = 1;
1965    if (!h1->get_item_info(h, NULL, it, &info)) {
1966        abort();
1967    }
1968    memcpy(info.value[0].iov_base, data, strlen(data));
1969
1970    uint64_t cas = 0;
1971    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
1972    check(rv == ENGINE_SUCCESS, "Set failed.");
1973    check_key_value(h, h1, key, data, strlen(data));
1974    h1->release(h, NULL, it);
1975    wait_for_flusher_to_settle(h, h1);
1976
1977    check(get_int_stat(h, h1, "curr_items") == 1, "Failed store item");
1978    testHarness.time_travel(5);
1979
1980    // Restart the engine to ensure the above item is expired
1981    testHarness.reload_engine(&h, &h1,
1982                              testHarness.engine_path,
1983                              testHarness.get_current_testcase()->cfg,
1984                              true, false);
1985    wait_for_warmup_complete(h, h1);
1986    int pager_runs = get_int_stat(h, h1, "ep_num_expiry_pager_runs");
1987    wait_for_stat_change(h, h1, "ep_num_expiry_pager_runs", pager_runs);
1988    wait_for_flusher_to_settle(h, h1);
1989    check(get_int_stat(h, h1, "curr_items") == 0,
1990            "The item should have been expired.");
1991
1992    return SUCCESS;
1993
1994}
1995
1996static enum test_result test_bug3454(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1997    const char *key = "test_expiry_duplicate_warmup";
1998    const char *data = "some test data here.";
1999
2000    item *it = NULL;
2001
2002    ENGINE_ERROR_CODE rv;
2003    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 5,
2004                      PROTOCOL_BINARY_RAW_BYTES);
2005    check(rv == ENGINE_SUCCESS, "Allocation failed.");
2006
2007    item_info info;
2008    info.nvalue = 1;
2009    if (!h1->get_item_info(h, NULL, it, &info)) {
2010        abort();
2011    }
2012    memcpy(info.value[0].iov_base, data, strlen(data));
2013
2014    uint64_t cas = 0;
2015    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
2016    check(rv == ENGINE_SUCCESS, "Set failed.");
2017    check_key_value(h, h1, key, data, strlen(data));
2018    h1->release(h, NULL, it);
2019    wait_for_flusher_to_settle(h, h1);
2020
2021    // Advance the ep_engine time by 10 sec for the above item to be expired.
2022    testHarness.time_travel(10);
2023    check(h1->get(h, NULL, &it, key, strlen(key), 0) == ENGINE_KEY_ENOENT,
2024          "Item didn't expire");
2025
2026    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 0,
2027                      PROTOCOL_BINARY_RAW_BYTES);
2028    check(rv == ENGINE_SUCCESS, "Allocation failed.");
2029
2030    info.nvalue = 1;
2031    if (!h1->get_item_info(h, NULL, it, &info)) {
2032        abort();
2033    }
2034    memcpy(info.value[0].iov_base, data, strlen(data));
2035
2036    cas = 0;
2037    // Add a new item with the same key.
2038    rv = h1->store(h, NULL, it, &cas, OPERATION_ADD, 0);
2039    check(rv == ENGINE_SUCCESS, "Add failed.");
2040    check_key_value(h, h1, key, data, strlen(data));
2041    h1->release(h, NULL, it);
2042
2043    check(h1->get(h, NULL, &it, key, strlen(key), 0) == ENGINE_SUCCESS,
2044          "Item shouldn't expire");
2045    h1->release(h, NULL, it);
2046
2047    // Restart the engine to ensure the above unexpired new item is loaded
2048    testHarness.reload_engine(&h, &h1,
2049                              testHarness.engine_path,
2050                              testHarness.get_current_testcase()->cfg,
2051                              true, false);
2052    wait_for_warmup_complete(h, h1);
2053    cb_assert(1 == get_int_stat(h, h1, "ep_warmup_value_count", "warmup"));
2054    cb_assert(0 == get_int_stat(h, h1, "ep_warmup_dups", "warmup"));
2055
2056    return SUCCESS;
2057}
2058
2059static enum test_result test_bug3522(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2060    const char *key = "test_expiry_no_items_warmup";
2061    const char *data = "some test data here.";
2062
2063    item *it = NULL;
2064
2065    ENGINE_ERROR_CODE rv;
2066    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 0,
2067                      PROTOCOL_BINARY_RAW_BYTES);
2068    check(rv == ENGINE_SUCCESS, "Allocation failed.");
2069
2070    item_info info;
2071    info.nvalue = 1;
2072    if (!h1->get_item_info(h, NULL, it, &info)) {
2073        abort();
2074    }
2075    memcpy(info.value[0].iov_base, data, strlen(data));
2076
2077    uint64_t cas = 0;
2078    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
2079    check(rv == ENGINE_SUCCESS, "Set failed.");
2080    check_key_value(h, h1, key, data, strlen(data));
2081    h1->release(h, NULL, it);
2082    wait_for_flusher_to_settle(h, h1);
2083
2084    // Add a new item with the same key and 2 sec of expiration.
2085    const char *new_data = "new data here.";
2086    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(new_data), 0, 2,
2087                      PROTOCOL_BINARY_RAW_BYTES);
2088    check(rv == ENGINE_SUCCESS, "Allocation failed.");
2089
2090    info.nvalue = 1;
2091    if (!h1->get_item_info(h, NULL, it, &info)) {
2092        abort();
2093    }
2094    memcpy(info.value[0].iov_base, new_data, strlen(new_data));
2095
2096    int pager_runs = get_int_stat(h, h1, "ep_num_expiry_pager_runs");
2097    cas = 0;
2098    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
2099    check(rv == ENGINE_SUCCESS, "Set failed.");
2100    check_key_value(h, h1, key, new_data, strlen(new_data));
2101    h1->release(h, NULL, it);
2102    testHarness.time_travel(3);
2103    wait_for_stat_change(h, h1, "ep_num_expiry_pager_runs", pager_runs);
2104    wait_for_flusher_to_settle(h, h1);
2105
2106    // Restart the engine.
2107    testHarness.reload_engine(&h, &h1,
2108                              testHarness.engine_path,
2109                              testHarness.get_current_testcase()->cfg,
2110                              true, false);
2111    wait_for_warmup_complete(h, h1);
2112    // TODO: modify this for a better test case
2113    cb_assert(0 == get_int_stat(h, h1, "ep_warmup_dups", "warmup"));
2114
2115    return SUCCESS;
2116}
2117
2118static enum test_result test_get_replica_active_state(ENGINE_HANDLE *h,
2119                                                      ENGINE_HANDLE_V1 *h1) {
2120    protocol_binary_request_header *pkt;
2121    pkt = prepare_get_replica(h, h1, vbucket_state_active);
2122    check(h1->unknown_command(h, NULL, pkt, add_response) ==
2123          ENGINE_SUCCESS, "Get Replica Failed");
2124    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2125          "Expected PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET response.");
2126
2127    return SUCCESS;
2128}
2129
2130static enum test_result test_get_replica_pending_state(ENGINE_HANDLE *h,
2131                                                       ENGINE_HANDLE_V1 *h1) {
2132    protocol_binary_request_header *pkt;
2133
2134    const void *cookie = testHarness.create_cookie();
2135    testHarness.set_ewouldblock_handling(cookie, false);
2136    pkt = prepare_get_replica(h, h1, vbucket_state_pending);
2137    check(h1->unknown_command(h, cookie, pkt, add_response) ==
2138          ENGINE_EWOULDBLOCK, "Should have returned error for pending state");
2139    testHarness.destroy_cookie(cookie);
2140    return SUCCESS;
2141}
2142
2143static enum test_result test_get_replica_dead_state(ENGINE_HANDLE *h,
2144                                                    ENGINE_HANDLE_V1 *h1) {
2145    protocol_binary_request_header *pkt;
2146    pkt = prepare_get_replica(h, h1, vbucket_state_dead);
2147    check(h1->unknown_command(h, NULL, pkt, add_response) ==
2148          ENGINE_SUCCESS, "Get Replica Failed");
2149    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2150          "Expected PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET response.");
2151
2152    return SUCCESS;
2153}
2154
2155static enum test_result test_get_replica(ENGINE_HANDLE *h,
2156                                         ENGINE_HANDLE_V1 *h1) {
2157    protocol_binary_request_header *pkt;
2158    pkt = prepare_get_replica(h, h1, vbucket_state_replica);
2159    check(h1->unknown_command(h, NULL, pkt, add_response) == ENGINE_SUCCESS,
2160                              "Get Replica Failed");
2161    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
2162          "Expected PROTOCOL_BINARY_RESPONSE_SUCCESS response.");
2163    check(strcmp("replicadata", last_body) == 0,
2164                 "Should have returned identical value");
2165
2166    return SUCCESS;
2167}
2168
2169static enum test_result test_get_replica_non_resident(ENGINE_HANDLE *h,
2170                                                      ENGINE_HANDLE_V1 *h1) {
2171
2172    item *i = NULL;
2173    check(store(h, h1, NULL, OPERATION_SET, "key", "value", &i, 0, 0)
2174          == ENGINE_SUCCESS, "Store Failed");
2175    h1->release(h, NULL, i);
2176    wait_for_flusher_to_settle(h, h1);
2177    wait_for_stat_to_be(h, h1, "ep_total_persisted", 1);
2178
2179    evict_key(h, h1, "key", 0, "Ejected.");
2180    check(set_vbucket_state(h, h1, 0, vbucket_state_replica),
2181          "Failed to set vbucket to replica");
2182
2183    get_replica(h, h1, "key", 0);
2184    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "Expected success");
2185
2186    return SUCCESS;
2187}
2188
2189static enum test_result test_get_replica_invalid_key(ENGINE_HANDLE *h,
2190                                                     ENGINE_HANDLE_V1 *h1) {
2191    protocol_binary_request_header *pkt;
2192    bool makeinvalidkey = true;
2193    pkt = prepare_get_replica(h, h1, vbucket_state_replica, makeinvalidkey);
2194    check(h1->unknown_command(h, NULL, pkt, add_response) ==
2195          ENGINE_SUCCESS, "Get Replica Failed");
2196    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2197          "Expected PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET response.");
2198    return SUCCESS;
2199}
2200
2201static enum test_result test_vb_del_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2202    const void *cookie = testHarness.create_cookie();
2203    testHarness.set_ewouldblock_handling(cookie, false);
2204    check(set_vbucket_state(h, h1, 1, vbucket_state_pending), "Failed to set vbucket state.");
2205    check(ENGINE_EWOULDBLOCK == del(h, h1, "key", 0, 1, cookie),
2206          "Expected woodblock.");
2207    testHarness.destroy_cookie(cookie);
2208    return SUCCESS;
2209}
2210
2211static enum test_result test_vb_del_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2212    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set vbucket state.");
2213    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
2214    check(ENGINE_NOT_MY_VBUCKET == del(h, h1, "key", 0, 1),
2215          "Expected not my vbucket.");
2216    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
2217    return SUCCESS;
2218}
2219
2220static enum test_result test_touch(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2221    // key is a mandatory field!
2222    touch(h, h1, NULL, 0, (time(NULL) + 10));
2223    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2224
2225    // extlen is a mandatory field!
2226    protocol_binary_request_header *request;
2227    request = createPacket(PROTOCOL_BINARY_CMD_TOUCH, 0, 0, NULL, 0, "akey", 4);
2228    check(h1->unknown_command(h, NULL, request, add_response) == ENGINE_SUCCESS,
2229          "Failed to call touch");
2230    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2231    free(request);
2232
2233    // Try to touch an unknown item...
2234    touch(h, h1, "mykey", 0, (time(NULL) + 10));
2235    check(last_status == PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, "Testing unknown key");
2236
2237    // illegal vbucket
2238    touch(h, h1, "mykey", 5, (time(NULL) + 10));
2239    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET, "Testing illegal vbucket");
2240
2241    // Store the item!
2242    item *itm = NULL;
2243    check(store(h, h1, NULL, OPERATION_SET, "mykey", "somevalue", &itm) == ENGINE_SUCCESS,
2244          "Failed set.");
2245    h1->release(h, NULL, itm);
2246
2247    check_key_value(h, h1, "mykey", "somevalue", strlen("somevalue"));
2248
2249    touch(h, h1, "mykey", 0, (time(NULL) + 10));
2250    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch mykey");
2251
2252    // time-travel 9 secs..
2253    testHarness.time_travel(9);
2254
2255    // The item should still exist
2256    check_key_value(h, h1, "mykey", "somevalue", 9);
2257
2258    // time-travel 2 secs..
2259    testHarness.time_travel(2);
2260
2261    // The item should have expired now...
2262    check(h1->get(h, NULL, &itm, "mykey", 5, 0) == ENGINE_KEY_ENOENT, "Item should be gone");
2263    return SUCCESS;
2264}
2265
2266static enum test_result test_touch_mb7342(ENGINE_HANDLE *h,
2267                                          ENGINE_HANDLE_V1 *h1) {
2268    const char *key = "MB-7342";
2269    // Store the item!
2270    item *itm = NULL;
2271    check(store(h, h1, NULL, OPERATION_SET, key, "v", &itm) == ENGINE_SUCCESS,
2272          "Failed set.");
2273    h1->release(h, NULL, itm);
2274
2275    touch(h, h1, key, 0, 0);
2276    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch key");
2277
2278    check_key_value(h, h1, key, "v", 1);
2279
2280    // Travel a loong time to see if the object is still there (the default
2281    // store sets an exp time of 3600
2282    testHarness.time_travel(3700);
2283
2284    check_key_value(h, h1, key, "v", 1);
2285
2286    return SUCCESS;
2287}
2288
2289static enum test_result test_touch_mb10277(ENGINE_HANDLE *h,
2290                                            ENGINE_HANDLE_V1 *h1) {
2291    const char *key = "MB-10277";
2292    // Store the item!
2293    item *itm = NULL;
2294    check(store(h, h1, NULL, OPERATION_SET, key, "v", &itm) == ENGINE_SUCCESS,
2295          "Failed set.");
2296    h1->release(h, NULL, itm);
2297    wait_for_flusher_to_settle(h, h1);
2298    evict_key(h, h1, key, 0, "Ejected.");
2299
2300    touch(h, h1, key, 0, 3600); // A new expiration time remains in the same.
2301    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch key");
2302
2303    return SUCCESS;
2304}
2305
2306static enum test_result test_gat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2307    // key is a mandatory field!
2308    gat(h, h1, NULL, 0, 10);
2309    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2310
2311    // extlen is a mandatory field!
2312    protocol_binary_request_header *request;
2313    request = createPacket(PROTOCOL_BINARY_CMD_GAT, 0, 0, NULL, 0, "akey", 4);
2314    check(h1->unknown_command(h, NULL, request, add_response) == ENGINE_SUCCESS,
2315          "Failed to call gat");
2316    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2317    free(request);
2318
2319    // Try to gat an unknown item...
2320    gat(h, h1, "mykey", 0, 10);
2321    check(last_status == PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, "Testing unknown key");
2322
2323    // illegal vbucket
2324    gat(h, h1, "mykey", 5, 10);
2325    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET, "Testing illegal vbucket");
2326
2327    // Store the item!
2328    item *itm = NULL;
2329    check(store(h, h1, NULL, OPERATION_SET, "mykey", "{\"some\":\"value\"}",
2330                &itm, 0, 0, 3600, PROTOCOL_BINARY_DATATYPE_JSON) == ENGINE_SUCCESS,
2331          "Failed set.");
2332    h1->release(h, NULL, itm);
2333
2334    check_key_value(h, h1, "mykey", "{\"some\":\"value\"}",
2335            strlen("{\"some\":\"value\"}"));
2336
2337    gat(h, h1, "mykey", 0, 10);
2338    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "gat mykey");
2339    check(last_datatype == PROTOCOL_BINARY_DATATYPE_JSON, "Expected datatype to be JSON");
2340    check(memcmp(last_body, "{\"some\":\"value\"}", sizeof("{\"some\":\"value\"}")) == 0,
2341          "Invalid data returned");
2342
2343    // time-travel 9 secs..
2344    testHarness.time_travel(9);
2345
2346    // The item should still exist
2347    check_key_value(h, h1, "mykey", "{\"some\":\"value\"}",
2348                    strlen("{\"some\":\"value\"}"));
2349
2350    // time-travel 2 secs..
2351    testHarness.time_travel(2);
2352
2353    // The item should have expired now...
2354    check(h1->get(h, NULL, &itm, "mykey", 5, 0) == ENGINE_KEY_ENOENT, "Item should be gone");
2355    return SUCCESS;
2356}
2357
2358static enum test_result test_gatq(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2359    // key is a mandatory field!
2360    gat(h, h1, NULL, 0, 10, true);
2361    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2362
2363    // extlen is a mandatory field!
2364    protocol_binary_request_header *request;
2365    request = createPacket(PROTOCOL_BINARY_CMD_GATQ, 0, 0, NULL, 0, "akey", 4);
2366    check(h1->unknown_command(h, NULL, request, add_response) == ENGINE_SUCCESS,
2367          "Failed to call gatq");
2368    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2369    free(request);
2370
2371    // Try to gatq an unknown item...
2372    last_status = static_cast<protocol_binary_response_status>(0xffff);
2373    gat(h, h1, "mykey", 0, 10, true);
2374
2375    // We should not have sent any response!
2376    check(last_status == (protocol_binary_response_status)0xffff, "Testing unknown key");
2377
2378    // illegal vbucket
2379    gat(h, h1, "mykey", 5, 10, true);
2380    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2381          "Testing illegal vbucket");
2382
2383    // Store the item!
2384    item *itm = NULL;
2385    check(store(h, h1, NULL, OPERATION_SET, "mykey", "{\"some\":\"value\"}",
2386                &itm, 0, 0, 3600, PROTOCOL_BINARY_DATATYPE_JSON) == ENGINE_SUCCESS,
2387          "Failed set.");
2388    h1->release(h, NULL, itm);
2389
2390    check_key_value(h, h1, "mykey", "{\"some\":\"value\"}",
2391                    strlen("{\"some\":\"value\"}"));
2392
2393    gat(h, h1, "mykey", 0, 10, true);
2394    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "gat mykey");
2395    check(last_datatype == PROTOCOL_BINARY_DATATYPE_JSON, "Expected datatype to be JSON");
2396    check(memcmp(last_body, "{\"some\":\"value\"}", sizeof("{\"some\":\"value\"}")) == 0,
2397          "Invalid data returned");
2398
2399    // time-travel 9 secs..
2400    testHarness.time_travel(9);
2401
2402    // The item should still exist
2403    check_key_value(h, h1, "mykey", "{\"some\":\"value\"}",
2404                    strlen("{\"some\":\"value\"}"));
2405
2406    // time-travel 2 secs..
2407    testHarness.time_travel(2);
2408
2409    // The item should have expired now...
2410    check(h1->get(h, NULL, &itm, "mykey", 5, 0) == ENGINE_KEY_ENOENT, "Item should be gone");
2411    return SUCCESS;
2412}
2413
2414static enum test_result test_mb5215(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2415    item *itm = NULL;
2416    check(store(h, h1, NULL, OPERATION_SET, "coolkey", "cooler", &itm)
2417          == ENGINE_SUCCESS, "Failed set.");
2418    h1->release(h, NULL, itm);
2419
2420    check_key_value(h, h1, "coolkey", "cooler", strlen("cooler"));
2421
2422    // set new exptime to 111
2423    int expTime = time(NULL) + 111;
2424
2425    touch(h, h1, "coolkey", 0, expTime);
2426    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch coolkey");
2427
2428    //reload engine
2429    testHarness.reload_engine(&h, &h1,
2430                              testHarness.engine_path,
2431                              testHarness.get_current_testcase()->cfg,
2432                              true, false);
2433    wait_for_warmup_complete(h, h1);
2434
2435    //verify persisted expiration time
2436    const char *statkey = "key coolkey 0";
2437    int newExpTime;
2438    check(h1->get(h, NULL, &itm, "coolkey", 7, 0) == ENGINE_SUCCESS,
2439          "Missing key");
2440    h1->release(h, NULL, itm);
2441    newExpTime = get_int_stat(h, h1, "key_exptime", statkey);
2442    check(newExpTime == expTime, "Failed to persist new exptime");
2443
2444    // evict key, touch expiration time, and verify
2445    evict_key(h, h1, "coolkey", 0, "Ejected.");
2446
2447    expTime = time(NULL) + 222;
2448    touch(h, h1, "coolkey", 0, expTime);
2449    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch coolkey");
2450
2451    testHarness.reload_engine(&h, &h1,
2452                              testHarness.engine_path,
2453                              testHarness.get_current_testcase()->cfg,
2454                              true, false);
2455    wait_for_warmup_complete(h, h1);
2456
2457    check(h1->get(h, NULL, &itm, "coolkey", 7, 0) == ENGINE_SUCCESS,
2458          "Missing key");
2459    h1->release(h, NULL, itm);
2460    newExpTime = get_int_stat(h, h1, "key_exptime", statkey);
2461    check(newExpTime == expTime, "Failed to persist new exptime");
2462
2463    return SUCCESS;
2464}
2465
2466static enum test_result test_alloc_limit(ENGINE_HANDLE *h,
2467                                         ENGINE_HANDLE_V1 *h1) {
2468    item *it = NULL;
2469    ENGINE_ERROR_CODE rv;
2470
2471    rv = h1->allocate(h, NULL, &it, "key", 3, 20 * 1024 * 1024, 0, 0,
2472                      PROTOCOL_BINARY_RAW_BYTES);
2473    check(rv == ENGINE_SUCCESS, "Allocated 20MB item");
2474    h1->release(h, NULL, it);
2475
2476    rv = h1->allocate(h, NULL, &it, "key", 3, (20 * 1024 * 1024) + 1, 0, 0,
2477                      PROTOCOL_BINARY_RAW_BYTES);
2478    check(rv == ENGINE_E2BIG, "Object too big");
2479
2480    return SUCCESS;
2481}
2482
2483static enum test_result test_whitespace_db(ENGINE_HANDLE *h,
2484                                           ENGINE_HANDLE_V1 *h1) {
2485    vals.clear();
2486    check(h1->get_stats(h, NULL, NULL, 0, add_stats) == ENGINE_SUCCESS,
2487          "Failed to get stats.");
2488    if (vals["ep_dbname"] != std::string(WHITESPACE_DB)) {
2489        std::cerr << "Expected dbname = ``" << WHITESPACE_DB << "''"
2490                  << ", got ``" << vals["ep_dbname"] << "''" << std::endl;
2491        return FAIL;
2492    }
2493
2494    check(access(WHITESPACE_DB, F_OK) != -1, "I expected the whitespace db to exist");
2495    return SUCCESS;
2496}
2497
2498static enum test_result test_memory_limit(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2499    set_param(h, h1, protocol_binary_engine_param_flush, "mutation_mem_threshold", "95");
2500    int used = get_int_stat(h, h1, "mem_used");
2501    double mem_threshold =
2502        static_cast<double>(get_int_stat(h, h1, "ep_mutation_mem_threshold")) / 100;
2503    int max = static_cast<int>(get_int_stat(h, h1, "ep_max_size") * mem_threshold);
2504    check(get_int_stat(h, h1, "ep_oom_errors") == 0 &&
2505          get_int_stat(h, h1, "ep_tmp_oom_errors") == 0, "Expected no OOM errors.");
2506    cb_assert(used < max);
2507
2508    char data[8192];
2509    memset(data, 'x', sizeof(data));
2510    size_t vlen = max - used - 192;
2511    data[vlen] = 0x00;
2512
2513    item *i = NULL;
2514    // So if we add an item,
2515    check(store(h, h1, NULL, OPERATION_SET, "key", data, &i) == ENGINE_SUCCESS,
2516          "store failure");
2517    check_key_value(h, h1, "key", data, vlen);
2518    h1->release(h, NULL, i);
2519
2520    // There should be no room for another.
2521    ENGINE_ERROR_CODE second = store(h, h1, NULL, OPERATION_SET, "key2", data, &i);
2522    check(second == ENGINE_ENOMEM || second == ENGINE_TMPFAIL,
2523          "should have failed second set");
2524    h1->release(h, NULL, i);
2525    check(get_int_stat(h, h1, "ep_oom_errors") == 1 ||
2526          get_int_stat(h, h1, "ep_tmp_oom_errors") == 1, "Expected an OOM error.");
2527
2528    ENGINE_ERROR_CODE overwrite = store(h, h1, NULL, OPERATION_SET, "key", data, &i);
2529    check(overwrite == ENGINE_ENOMEM || overwrite == ENGINE_TMPFAIL,
2530          "should have failed second override");
2531    h1->release(h, NULL, i);
2532    check(get_int_stat(h, h1, "ep_oom_errors") == 2 ||
2533          get_int_stat(h, h1, "ep_tmp_oom_errors") == 2, "Expected another OOM error.");
2534    check_key_value(h, h1, "key", data, vlen);
2535    check(ENGINE_SUCCESS != verify_key(h, h1, "key2"), "Expected a failure in GET");
2536    int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
2537    // Until we remove that item
2538    check(del(h, h1, "key", 0, 0) == ENGINE_SUCCESS, "Failed remove with value.");
2539    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
2540    testHarness.time_travel(65);
2541    wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
2542
2543    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue2", &i) == ENGINE_SUCCESS,
2544          "should have succeded on the last set");
2545    check_key_value(h, h1, "key2", "somevalue2", 10);
2546    h1->release(h, NULL, i);
2547    return SUCCESS;
2548}
2549
2550static enum test_result test_vbucket_get_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2551    return verify_vbucket_missing(h, h1, 1) ? SUCCESS : FAIL;
2552}
2553
2554static enum test_result test_vbucket_get(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2555    return verify_vbucket_state(h, h1, 0, vbucket_state_active) ? SUCCESS : FAIL;
2556}
2557
2558static enum test_result test_vbucket_create(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2559    if (!verify_vbucket_missing(h, h1, 1)) {
2560        fprintf(stderr, "vbucket wasn't missing.\n");
2561        return FAIL;
2562    }
2563
2564    if (!set_vbucket_state(h, h1, 1, vbucket_state_active)) {
2565        fprintf(stderr, "set state failed.\n");
2566        return FAIL;
2567    }
2568
2569    return verify_vbucket_state(h, h1, 1, vbucket_state_active) ? SUCCESS : FAIL;
2570}
2571
2572static enum test_result test_vbucket_compact(ENGINE_HANDLE *h,
2573                                             ENGINE_HANDLE_V1 *h1) {
2574    const char *key = "Carss";
2575    const char *value = "pollute";
2576    if (!verify_vbucket_missing(h, h1, 0)) {
2577        fprintf(stderr, "vbucket wasn't missing.\n");
2578        return FAIL;
2579    }
2580
2581    if (!set_vbucket_state(h, h1, 0, vbucket_state_active)) {
2582        fprintf(stderr, "set state failed.\n");
2583        return FAIL;
2584    }
2585
2586    check(verify_vbucket_state(h, h1, 0, vbucket_state_active),
2587            "VBucket state not active");
2588
2589    // Set two keys - one to be expired and other to remain...
2590    item *itm = NULL;
2591    check(store(h, h1, NULL, OPERATION_SET, key, value, &itm)
2592          == ENGINE_SUCCESS, "Failed set.");
2593    h1->release(h, NULL, itm);
2594
2595    check_key_value(h, h1, key, value, strlen(value));
2596
2597    // Set a non-expiring key...
2598    check(store(h, h1, NULL, OPERATION_SET, "trees", "cleanse", &itm)
2599          == ENGINE_SUCCESS, "Failed set.");
2600    h1->release(h, NULL, itm);
2601
2602    check_key_value(h, h1, "trees", "cleanse", strlen("cleanse"));
2603
2604    touch(h, h1, key, 0, 11);
2605    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch Carss");
2606
2607    testHarness.time_travel(12);
2608    wait_for_flusher_to_settle(h, h1);
2609
2610    check(get_int_stat(h, h1, "vb_0:purge_seqno", "vbucket-seqno") == 0,
2611            "purge_seqno not found to be zero before compaction");
2612
2613    // Compaction on VBucket
2614    compact_db(h, h1, 0, 2, 3, 1);
2615
2616    check(get_int_stat(h, h1, "ep_pending_compactions") == 0,
2617    "ep_pending_compactions stat did not tick down after compaction command");
2618
2619    // the key tree and its value should be intact...
2620    check(verify_key(h, h1, "trees") == ENGINE_SUCCESS,
2621          "key trees should be found.");
2622    // the key Carrs should have disappeared...
2623    int val = verify_key(h, h1, "Carss");
2624    check(val == ENGINE_KEY_ENOENT, "Key Carss has not expired.");
2625
2626    check(get_int_stat(h, h1, "vb_0:purge_seqno", "vbucket-seqno") == 3,
2627        "purge_seqno didn't match expected value");
2628
2629    return SUCCESS;
2630}
2631
2632struct comp_thread_ctx {
2633    ENGINE_HANDLE *h;
2634    ENGINE_HANDLE_V1 *h1;
2635    uint16_t vbid;
2636};
2637
2638extern "C" {
2639    static void compaction_thread(void *arg) {
2640        struct comp_thread_ctx *ctx = static_cast<comp_thread_ctx *>(arg);
2641        compact_db(ctx->h, ctx->h1, ctx->vbid, 0, 0, 0);
2642    }
2643}
2644
2645static enum test_result test_multiple_vb_compactions(ENGINE_HANDLE *h,
2646                                                     ENGINE_HANDLE_V1 *h1) {
2647    for (uint16_t i = 0; i < 4; ++i) {
2648        if (!set_vbucket_state(h, h1, i, vbucket_state_active)) {
2649            fprintf(stderr, "set state failed for vbucket %d.\n", i);
2650            return FAIL;
2651        }
2652        check(verify_vbucket_state(h, h1, i, vbucket_state_active),
2653              "VBucket state not active");
2654    }
2655
2656    std::vector<std::string> keys;
2657    for (int j = 0; j < 20000; ++j) {
2658        std::stringstream ss;
2659        ss << "key" << j;
2660        std::string key(ss.str());
2661        keys.push_back(key);
2662    }
2663
2664    int count = 0;
2665    std::vector<std::string>::iterator it;
2666    for (it = keys.begin(); it != keys.end(); ++it) {
2667        uint16_t vbid = count % 4;
2668        item *i;
2669        check(store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str(), &i, 0, vbid)
2670              == ENGINE_SUCCESS, "Failed to store a value");
2671        h1->release(h, NULL, i);
2672        ++count;
2673    }
2674
2675    // Compact multiple vbuckets.
2676    const int n_threads = 4;
2677    cb_thread_t threads[n_threads];
2678    struct comp_thread_ctx ctx[n_threads];
2679
2680    for (int i = 0; i < n_threads; i++) {
2681        ctx[i].h = h;
2682        ctx[i].h1 = h1;
2683        ctx[i].vbid = static_cast<uint16_t>(i);
2684        int r = cb_create_thread(&threads[i], compaction_thread, &ctx[i], 0);
2685        cb_assert(r == 0);
2686    }
2687
2688    for (int i = 0; i < n_threads; i++) {
2689        int r = cb_join_thread(threads[i]);
2690        cb_assert(r == 0);
2691    }
2692
2693    check(get_int_stat(h, h1, "ep_pending_compactions") == 0,
2694    "ep_pending_compactions stat did not tick down after compaction command");
2695
2696    return SUCCESS;
2697}
2698
2699static enum test_result
2700test_multi_vb_compactions_with_workload(ENGINE_HANDLE *h,
2701                                        ENGINE_HANDLE_V1 *h1) {
2702    for (uint16_t i = 0; i < 4; ++i) {
2703        if (!set_vbucket_state(h, h1, i, vbucket_state_active)) {
2704            fprintf(stderr, "set state failed for vbucket %d.\n", i);
2705            return FAIL;
2706        }
2707        check(verify_vbucket_state(h, h1, i, vbucket_state_active),
2708              "VBucket state not active");
2709    }
2710
2711    std::vector<std::string> keys;
2712    for (int j = 0; j < 10000; ++j) {
2713        std::stringstream ss;
2714        ss << "key" << j;
2715        std::string key(ss.str());
2716        keys.push_back(key);
2717    }
2718
2719    int count = 0;
2720    std::vector<std::string>::iterator it;
2721    for (it = keys.begin(); it != keys.end(); ++it) {
2722        uint16_t vbid = count % 4;
2723        item *i;
2724        check(store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str(), &i, 0, vbid)
2725              == ENGINE_SUCCESS, "Failed to store a value");
2726        h1->release(h, NULL, i);
2727        ++count;
2728    }
2729    wait_for_flusher_to_settle(h, h1);
2730
2731    for (int i = 0; i < 2; ++i) {
2732        count = 0;
2733        for (it = keys.begin(); it != keys.end(); ++it) {
2734            uint16_t vbid = count % 4;
2735            item *i = NULL;
2736            check(h1->get(h, NULL, &i, it->c_str(), strlen(it->c_str()), vbid) ==
2737                  ENGINE_SUCCESS, "Unable to get stored item");
2738            h1->release(h, NULL, i);
2739            ++count;
2740        }
2741    }
2742    wait_for_str_stat_to_be(h, h1, "ep_workload_pattern", "read_heavy", NULL);
2743
2744    // Compact multiple vbuckets.
2745    const int n_threads = 4;
2746    cb_thread_t threads[n_threads];
2747    struct comp_thread_ctx ctx[n_threads];
2748
2749    for (int i = 0; i < n_threads; i++) {
2750        ctx[i].h = h;
2751        ctx[i].h1 = h1;
2752        ctx[i].vbid = static_cast<uint16_t>(i);
2753        int r = cb_create_thread(&threads[i], compaction_thread, &ctx[i], 0);
2754        cb_assert(r == 0);
2755    }
2756
2757    for (int i = 0; i < n_threads; i++) {
2758        int r = cb_join_thread(threads[i]);
2759        cb_assert(r == 0);
2760    }
2761
2762    check(get_int_stat(h, h1, "ep_pending_compactions") == 0,
2763    "ep_pending_compactions stat did not tick down after compaction command");
2764
2765    return SUCCESS;
2766}
2767
2768static enum test_result vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
2769                                             const char* value = NULL) {
2770    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed to set vbucket state.");
2771
2772    vbucketDelete(h, h1, 2, value);
2773    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2774          "Expected failure deleting non-existent bucket.");
2775
2776    check(set_vbucket_state(h, h1, 1, vbucket_state_dead), "Failed set set vbucket 1 state.");
2777
2778    vbucketDelete(h, h1, 1, value);
2779    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
2780          "Expected failure deleting non-existent bucket.");
2781
2782    check(verify_vbucket_missing(h, h1, 1),
2783          "vbucket 0 was not missing after deleting it.");
2784
2785    return SUCCESS;
2786}
2787
2788static enum test_result test_vbucket_destroy_stats(ENGINE_HANDLE *h,
2789                                                   ENGINE_HANDLE_V1 *h1) {
2790
2791    int mem_used = get_int_stat(h, h1, "mem_used");
2792    int cacheSize = get_int_stat(h, h1, "ep_total_cache_size");
2793    int overhead = get_int_stat(h, h1, "ep_overhead");
2794    int nonResident = get_int_stat(h, h1, "ep_num_non_resident");
2795
2796    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed to set vbucket state.");
2797
2798    std::vector<std::string> keys;
2799    for (int j = 0; j < 2000; ++j) {
2800        std::stringstream ss;
2801        ss << "key" << j;
2802        std::string key(ss.str());
2803        keys.push_back(key);
2804    }
2805
2806    int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
2807    std::vector<std::string>::iterator it;
2808    for (it = keys.begin(); it != keys.end(); ++it) {
2809        item *i;
2810        check(store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str(), &i, 0, 1)
2811              == ENGINE_SUCCESS, "Failed to store a value");
2812        h1->release(h, NULL, i);
2813    }
2814    wait_for_flusher_to_settle(h, h1);
2815    testHarness.time_travel(65);
2816    wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
2817
2818    check(set_vbucket_state(h, h1, 1, vbucket_state_dead), "Failed set set vbucket 1 state.");
2819
2820    int vbucketDel = get_int_stat(h, h1, "ep_vbucket_del");
2821    vbucketDelete(h, h1, 1);
2822    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
2823          "Expected failure deleting non-existent bucket.");
2824
2825    check(verify_vbucket_missing(h, h1, 1),
2826          "vbucket 1 was not missing after deleting it.");
2827
2828    wait_for_stat_change(h, h1, "ep_vbucket_del", vbucketDel);
2829
2830    wait_for_stat_to_be(h, h1, "mem_used", mem_used);
2831    wait_for_stat_to_be(h, h1, "ep_total_cache_size", cacheSize);
2832    wait_for_stat_to_be(h, h1, "ep_overhead", overhead);
2833    wait_for_stat_to_be(h, h1, "ep_num_non_resident", nonResident);
2834
2835    return SUCCESS;
2836}
2837
2838static enum test_result vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
2839                                                const char* value = NULL) {
2840    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed to set vbucket state.");
2841
2842    // Store a value so the restart will try to resurrect it.
2843    item *i = NULL;
2844    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i, 0, 1)
2845          == ENGINE_SUCCESS, "Failed to set a value");
2846    check_key_value(h, h1, "key", "somevalue", 9, 1);
2847    h1->release(h, NULL, i);
2848
2849    // Reload to get a flush forced.
2850    testHarness.reload_engine(&h, &h1,
2851                              testHarness.engine_path,
2852                              testHarness.get_current_testcase()->cfg,
2853                              true, false);
2854    wait_for_warmup_complete(h, h1);
2855
2856    check(verify_vbucket_state(h, h1, 1, vbucket_state_active),
2857          "Bucket state was what it was initially, after restart.");
2858    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed to set vbucket state.");
2859    check_key_value(h, h1, "key", "somevalue", 9, 1);
2860
2861    check(set_vbucket_state(h, h1, 1, vbucket_state_dead), "Failed set set vbucket 1 state.");
2862
2863    vbucketDelete(h, h1, 1, value);
2864    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
2865          "Expected failure deleting non-existent bucket.");
2866
2867    check(verify_vbucket_missing(h, h1, 1),
2868          "vbucket 1 was not missing after deleting it.");
2869
2870    testHarness.reload_engine(&h, &h1,
2871                              testHarness.engine_path,
2872                              testHarness.get_current_testcase()->cfg,
2873                              true, false);
2874    wait_for_warmup_complete(h, h1);
2875
2876    if (verify_vbucket_state(h, h1, 1, vbucket_state_pending, true)) {
2877        std::cerr << "Bucket came up in pending state after delete." << std::endl;
2878        abort();
2879    }
2880
2881    check(verify_vbucket_missing(h, h1, 1),
2882          "vbucket 1 was not missing after restart.");
2883
2884    return SUCCESS;
2885}
2886
2887static enum test_result test_async_vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2888    return vbucket_destroy(h, h1);
2889}
2890
2891static enum test_result test_sync_vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2892    return vbucket_destroy(h, h1, "async=0");
2893}
2894
2895static enum test_result test_async_vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2896    return vbucket_destroy_restart(h, h1);
2897}
2898
2899static enum test_result test_sync_vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2900    return vbucket_destroy_restart(h, h1, "async=0");
2901}
2902
2903static enum test_result test_vb_set_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2904    return test_pending_vb_mutation(h, h1, OPERATION_SET);
2905}
2906
2907static enum test_result test_vb_add_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2908    return test_pending_vb_mutation(h, h1, OPERATION_ADD);
2909}
2910
2911static enum test_result test_vb_cas_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2912    return test_pending_vb_mutation(h, h1, OPERATION_CAS);
2913}
2914
2915static enum test_result test_vb_append_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2916    return test_pending_vb_mutation(h, h1, OPERATION_APPEND);
2917}
2918
2919static enum test_result test_vb_prepend_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2920    return test_pending_vb_mutation(h, h1, OPERATION_PREPEND);
2921}
2922
2923static enum test_result test_vb_set_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2924    return test_replica_vb_mutation(h, h1, OPERATION_SET);
2925}
2926
2927static enum test_result test_vb_replace_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2928    return test_replica_vb_mutation(h, h1, OPERATION_REPLACE);
2929}
2930
2931static enum test_result test_vb_replace_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2932    return test_pending_vb_mutation(h, h1, OPERATION_REPLACE);
2933}
2934
2935static enum test_result test_vb_add_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2936    return test_replica_vb_mutation(h, h1, OPERATION_ADD);
2937}
2938
2939static enum test_result test_vb_cas_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2940    return test_replica_vb_mutation(h, h1, OPERATION_CAS);
2941}
2942
2943static enum test_result test_vb_append_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2944    return test_replica_vb_mutation(h, h1, OPERATION_APPEND);
2945}
2946
2947static enum test_result test_vb_prepend_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2948    return test_replica_vb_mutation(h, h1, OPERATION_PREPEND);
2949}
2950
2951static enum test_result test_stats_seqno(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2952    check(set_vbucket_state(h, h1, 1, vbucket_state_active),
2953          "Failed to set vbucket state.");
2954
2955    int num_keys = 100;
2956    for (int ii = 0; ii < num_keys; ++ii) {
2957        std::stringstream ss;
2958        ss << "key" << ii;
2959        check(store(h, h1, NULL, OPERATION_SET, ss.str().c_str(),
2960                    "value", NULL, 0, 0) == ENGINE_SUCCESS,
2961              "Failed to store an item.");
2962    }
2963
2964    check(get_int_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno") == 100,
2965          "Invalid seqno");
2966    check(get_int_stat(h, h1, "vb_1:high_seqno", "vbucket-seqno") == 0,
2967          "Invalid seqno");
2968    check(get_int_stat(h, h1, "vb_1:high_seqno", "vbucket-seqno 1") == 0,
2969          "Invalid seqno");
2970
2971    uint64_t vb_uuid = get_ull_stat(h, h1, "vb_1:0:id", "failovers");
2972    check(get_ull_stat(h, h1, "vb_1:uuid", "vbucket-seqno 1") == vb_uuid,
2973          "Invalid uuid");
2974    check(vals.size() == 4, "Expected four stats");
2975
2976    // Check invalid vbucket
2977    check(h1->get_stats(h, NULL, "vbucket-seqno 2", 15, add_stats)
2978          == ENGINE_NOT_MY_VBUCKET, "Expected not my vbucket");
2979
2980    // Check bad vbucket parameter (not numeric)
2981    check(h1->get_stats(h, NULL, "vbucket-seqno tt2", 17, add_stats)
2982          == ENGINE_EINVAL, "Expected invalid");
2983
2984    // Check extra spaces at the end
2985    check(h1->get_stats(h, NULL, "vbucket-seqno    ", 17, add_stats)
2986          == ENGINE_EINVAL, "Expected invalid");
2987
2988