xref: /3.0.3-GA/ep-engine/tests/ep_testsuite.cc (revision 5ad7924a)
1/* -*- MODE: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/*
3 *     Copyright 2010 Couchbase, Inc
4 *
5 *   Licensed under the Apache License, Version 2.0 (the "License");
6 *   you may not use this file except in compliance with the License.
7 *   You may obtain a copy of the License at
8 *
9 *       http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *   Unless required by applicable law or agreed to in writing, software
12 *   distributed under the License is distributed on an "AS IS" BASIS,
13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *   See the License for the specific language governing permissions and
15 *   limitations under the License.
16 */
17
18// Usage: (to repeatedly run just a single test case)
19// make engine_tests IS_LOOP=-L EP_TEST_NUM=3
20
21#include "config.h"
22
23#include <stdio.h>
24#include <stdlib.h>
25#include <string.h>
26#include <sys/stat.h>
27#ifdef _MSC_VER
28#include <direct.h>
29#define mkdir(a, b) _mkdir(a)
30#else
31#include <sys/wait.h>
32#endif
33
34#include <cstdlib>
35#include <iostream>
36#include <map>
37#include <set>
38#include <sstream>
39#include <string>
40#include <vector>
41
42#include <platform/dirutils.h>
43
44#include "atomic.h"
45#include "ep-engine/command_ids.h"
46#include "ep_test_apis.h"
47#include "ep_testsuite.h"
48#include "locks.h"
49#include "mock/mock_dcp.h"
50#include "mutex.h"
51
52#include <snappy-c.h>
53#include <JSON_checker.h>
54
55#ifdef linux
56/* /usr/include/netinet/in.h defines macros from ntohs() to _bswap_nn to
57 * optimize the conversion functions, but the prototypes generate warnings
58 * from gcc. The conversion methods isn't the bottleneck for my app, so
59 * just remove the warnings by undef'ing the optimization ..
60 */
61#undef ntohs
62#undef ntohl
63#undef htons
64#undef htonl
65#endif
66
67// ptr_fun don't like the extern "C" thing for unlock cookie.. cast it
68// away ;)
69typedef void (*UNLOCK_COOKIE_T)(const void *cookie);
70
71extern "C" bool abort_msg(const char *expr, const char *msg, int line);
72
73
74template <typename T>
75static void checkeqfn(T exp, T got, const char *msg, const char *file, const int linenum) {
76    if (exp != got) {
77        std::stringstream ss;
78        ss << "Expected `" << exp << "', got `" << got << "' - " << msg;
79        abort_msg(ss.str().c_str(), file, linenum);
80    }
81}
82
83#define checkeq(a, b, c) checkeqfn(a, b, c, __FILE__, __LINE__)
84
85extern "C" {
86
87#define check(expr, msg) \
88    static_cast<void>((expr) ? 0 : abort_msg(#expr, msg, __LINE__))
89
90#define WHITESPACE_DB "whitespace sucks.db"
91#define MULTI_DISPATCHER_CONFIG \
92    "ht_size=129;ht_locks=3;chk_remover_stime=1;chk_period=60"
93
94struct test_harness testHarness;
95
96class ThreadData {
97public:
98    ThreadData(ENGINE_HANDLE *eh, ENGINE_HANDLE_V1 *ehv1,
99               int e=0) : h(eh), h1(ehv1), extra(e) {}
100    ENGINE_HANDLE    *h;
101    ENGINE_HANDLE_V1 *h1;
102    int               extra;
103};
104
105bool abort_msg(const char *expr, const char *msg, int line) {
106    fprintf(stderr, "%s:%d Test failed: `%s' (%s)\n",
107            __FILE__, line, msg, expr);
108    abort();
109    // UNREACHABLE
110    return false;
111}
112
113static const char *dbname_env;
114static enum test_result rmdb(void)
115{
116    const char *files[] = { WHITESPACE_DB,
117                            "/tmp/test",
118                            "/tmp/mutation.log",
119                            dbname_env,
120                            NULL };
121    int ii = 0;
122    while (files[ii] != NULL) {
123        CouchbaseDirectoryUtilities::rmrf(files[ii]);
124        if (access(files[ii], F_OK) != -1) {
125            std::cerr << "Failed to remove: " << files[ii] << " " << std::endl;
126            return FAIL;
127        }
128        ++ii;
129    }
130
131    return SUCCESS;
132}
133
134static enum test_result skipped_test_function(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
135    (void) h;
136    (void) h1;
137    return SKIPPED;
138}
139
140static bool teardown(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
141    (void)h; (void)h1;
142    vals.clear();
143    return true;
144}
145
146static const void* createTapConn(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
147                                 const char *name) {
148    const void *cookie = testHarness.create_cookie();
149    testHarness.lock_cookie(cookie);
150    TAP_ITERATOR iter = h1->get_tap_iterator(h, cookie, name,
151                                             strlen(name),
152                                             TAP_CONNECT_FLAG_DUMP, NULL,
153                                             0);
154    check(iter != NULL, "Failed to create a tap iterator");
155    return cookie;
156}
157
158static void check_key_value(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
159                            const char* key, const char* val, size_t vlen,
160                            uint16_t vbucket = 0) {
161    item_info info;
162    check(get_item_info(h, h1, &info, key, vbucket), "checking key and value");
163    check(info.nvalue == 1, "info.nvalue != 1");
164    check(vlen == info.value[0].iov_len, "Value length mismatch");
165    check(memcmp(info.value[0].iov_base, val, vlen) == 0, "Data mismatch");
166}
167
168static bool test_setup(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
169    wait_for_warmup_complete(h, h1);
170    wait_for_stat_change(h, h1, "ep_vb_snapshot_total", 0);
171
172    // warmup is complete, notify ep engine that it must now enable
173    // data traffic
174    protocol_binary_request_header *pkt = createPacket(PROTOCOL_BINARY_CMD_ENABLE_TRAFFIC);
175    check(h1->unknown_command(h, NULL, pkt, add_response) == ENGINE_SUCCESS,
176          "Failed to enable data traffic");
177    free(pkt);
178
179    return true;
180}
181
182static enum test_result test_getl(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
183    const char *key = "k1";
184    uint16_t vbucketId = 0;
185    uint32_t expiration = 25;
186
187    getl(h, h1, key, vbucketId, expiration);
188    check(last_status == PROTOCOL_BINARY_RESPONSE_KEY_ENOENT,
189          "expected the key to be missing...");
190    if (last_body != NULL && (strcmp(last_body, "NOT_FOUND") != 0)) {
191        fprintf(stderr, "Should have returned NOT_FOUND. Getl Failed");
192        abort();
193    }
194
195    item *i = NULL;
196    check(store(h, h1, NULL, OPERATION_SET, key, "{\"lock\":\"data\"}",
197                &i, 0, vbucketId, 3600, PROTOCOL_BINARY_DATATYPE_JSON)
198          == ENGINE_SUCCESS, "Failed to store an item.");
199    h1->release(h, NULL, i);
200
201    /* retry getl, should succeed */
202    getl(h, h1, key, vbucketId, expiration);
203    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
204          "Expected to be able to getl on first try");
205    check(strcmp("{\"lock\":\"data\"}", last_body) == 0, "Body was malformed.");
206    check(last_datatype == PROTOCOL_BINARY_DATATYPE_JSON,
207            "Expected datatype to be JSON");
208
209    /* wait 16 seconds */
210    testHarness.time_travel(16);
211
212    /* lock's taken so this should fail */
213    getl(h, h1, key, vbucketId, expiration);
214    check(last_status == PROTOCOL_BINARY_RESPONSE_ETMPFAIL,
215          "Expected to fail getl on second try");
216
217    if (last_body != NULL && (strcmp(last_body, "LOCK_ERROR") != 0)) {
218        fprintf(stderr, "Should have returned LOCK_ERROR. Getl Failed");
219        abort();
220    }
221
222    check(store(h, h1, NULL, OPERATION_SET, key, "lockdata2", &i, 0, vbucketId)
223          != ENGINE_SUCCESS, "Should have failed to store an item.");
224    h1->release(h, NULL, i);
225
226    /* wait another 10 seconds */
227    testHarness.time_travel(10);
228
229    /* retry set, should succeed */
230    check(store(h, h1, NULL, OPERATION_SET, key, "lockdata", &i, 0, vbucketId)
231          == ENGINE_SUCCESS, "Failed to store an item.");
232    h1->release(h, NULL, i);
233
234    /* point to wrong vbucket, to test NOT_MY_VB response */
235    getl(h, h1, key, 10, expiration);
236    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
237          "Should have received not my vbucket response");
238
239    /* acquire lock, should succeed */
240    getl(h, h1, key, vbucketId, expiration);
241    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
242          "Aquire lock should have succeeded");
243    check(last_datatype == PROTOCOL_BINARY_RAW_BYTES,
244            "Expected datatype to be RAW BYTES");
245
246    /* try an incr operation followed by a delete, both of which should fail */
247    uint64_t cas = 0;
248    uint64_t result = 0;
249
250    check(h1->arithmetic(h, NULL, key, 2, true, false, 1, 1, 0,
251                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
252                         0)  == ENGINE_TMPFAIL, "Incr failed");
253
254
255    check(del(h, h1, key, 0, 0) == ENGINE_TMPFAIL, "Delete failed");
256
257
258    /* bug MB 2699 append after getl should fail with ENGINE_TMPFAIL */
259
260    testHarness.time_travel(26);
261
262    char binaryData1[] = "abcdefg\0gfedcba";
263    char binaryData2[] = "abzdefg\0gfedcba";
264
265    check(storeCasVb11(h, h1, NULL, OPERATION_SET, key,
266                       binaryData1, sizeof(binaryData1) - 1, 82758, &i, 0, 0)
267          == ENGINE_SUCCESS,
268          "Failed set.");
269    h1->release(h, NULL, i);
270
271    /* acquire lock, should succeed */
272    getl(h, h1, key, vbucketId, expiration);
273    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
274          "Aquire lock should have succeeded");
275
276    /* append should fail */
277    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, key,
278                       binaryData2, sizeof(binaryData2) - 1, 82758, &i, 0, 0)
279          == ENGINE_TMPFAIL,
280          "Append should fail.");
281    h1->release(h, NULL, i);
282
283    /* bug MB 3252 & MB 3354.
284     * 1. Set a key with an expiry value.
285     * 2. Take a lock on the item before it expires
286     * 3. Wait for the item to expire
287     * 4. Perform a CAS operation, should fail
288     * 5. Perform a set operation, should succeed
289     */
290    const char *ekey = "test_expiry";
291    const char *edata = "some test data here.";
292
293    item *it = NULL;
294
295    check(h1->allocate(h, NULL, &it, ekey, strlen(ekey), strlen(edata), 0, 2,
296          PROTOCOL_BINARY_RAW_BYTES) == ENGINE_SUCCESS, "Allocation Failed");
297
298    item_info info;
299    info.nvalue = 1;
300    if (!h1->get_item_info(h, NULL, it, &info)) {
301        abort();
302    }
303    memcpy(info.value[0].iov_base, edata, strlen(edata));
304
305    check(h1->store(h, NULL, it, &cas, OPERATION_SET, 0) ==
306        ENGINE_SUCCESS, "Failed to Store item");
307    check_key_value(h, h1, ekey, edata, strlen(edata));
308    h1->release(h, NULL, it);
309
310    testHarness.time_travel(3);
311    cas = last_cas;
312
313    /* cas should fail */
314    check(storeCasVb11(h, h1, NULL, OPERATION_CAS, ekey,
315                       binaryData1, sizeof(binaryData1) - 1, 82758, &i, cas, 0)
316          != ENGINE_SUCCESS,
317          "CAS succeeded.");
318    h1->release(h, NULL, i);
319
320    /* but a simple store should succeed */
321    check(store(h, h1, NULL, OPERATION_SET, ekey, edata, &i, 0, vbucketId)
322          == ENGINE_SUCCESS, "Failed to store an item.");
323    h1->release(h, NULL, i);
324    return SUCCESS;
325}
326
327static enum test_result test_unl(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
328
329    const char *key = "k2";
330    uint16_t vbucketId = 0;
331
332    unl(h, h1, key, vbucketId);
333    check(last_status != PROTOCOL_BINARY_RESPONSE_SUCCESS,
334          "expected the key to be missing...");
335
336    item *i = NULL;
337    check(store(h, h1, NULL, OPERATION_SET, key, "lockdata", &i, 0, vbucketId)
338          == ENGINE_SUCCESS, "Failed to store an item.");
339    h1->release(h, NULL, i);
340
341    /* getl, should succeed */
342    getl(h, h1, key, vbucketId, 0);
343    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
344          "Expected to be able to getl on first try");
345
346    /* save the returned cas value for later */
347    uint64_t cas = last_cas;
348
349    /* lock's taken unlocking with a random cas value should fail */
350    unl(h, h1, key, vbucketId);
351    check(last_status == PROTOCOL_BINARY_RESPONSE_ETMPFAIL,
352          "Expected to fail getl on second try");
353
354    if (last_body != NULL && (strcmp(last_body, "UNLOCK_ERROR") != 0)) {
355        fprintf(stderr, "Should have returned UNLOCK_ERROR. Unl Failed");
356        abort();
357    }
358
359    unl(h, h1, key, vbucketId, cas);
360    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
361          "Expected to succed unl with correct cas");
362
363    /* acquire lock, should succeed */
364    getl(h, h1, key, vbucketId, 0);
365
366    /* wait 16 seconds */
367    testHarness.time_travel(16);
368
369    /* lock has expired, unl should fail */
370    unl(h, h1, key, vbucketId, last_cas);
371    check(last_status == PROTOCOL_BINARY_RESPONSE_ETMPFAIL,
372          "Expected to fail unl on lock timeout");
373
374    return SUCCESS;
375}
376
377static enum test_result test_wrong_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
378                                               ENGINE_STORE_OPERATION op) {
379    item *i = NULL;
380    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
381    uint64_t cas = 11;
382    if (op == OPERATION_ADD) {
383        // Add operation with cas != 0 doesn't make sense
384        cas = 0;
385    }
386    check(store(h, h1, NULL, op,
387                "key", "somevalue", &i, cas, 1) == ENGINE_NOT_MY_VBUCKET,
388        "Expected not_my_vbucket");
389    h1->release(h, NULL, i);
390    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
391    return SUCCESS;
392}
393
394static enum test_result test_pending_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
395                                                 ENGINE_STORE_OPERATION op) {
396    const void *cookie = testHarness.create_cookie();
397    testHarness.set_ewouldblock_handling(cookie, false);
398    item *i = NULL;
399    check(set_vbucket_state(h, h1, 1, vbucket_state_pending), "Failed to set vbucket state.");
400    check(verify_vbucket_state(h, h1, 1, vbucket_state_pending), "Bucket state was not set to pending.");
401    uint64_t cas = 11;
402    if (op == OPERATION_ADD) {
403        // Add operation with cas != 0 doesn't make sense..
404        cas = 0;
405    }
406    check(store(h, h1, cookie, op,
407                "key", "somevalue", &i, cas, 1) == ENGINE_EWOULDBLOCK,
408        "Expected woodblock");
409    h1->release(h, NULL, i);
410    testHarness.destroy_cookie(cookie);
411    return SUCCESS;
412}
413
414static enum test_result test_replica_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
415                                                 ENGINE_STORE_OPERATION op) {
416    item *i = NULL;
417    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set vbucket state.");
418    check(verify_vbucket_state(h, h1, 1, vbucket_state_replica), "Bucket state was not set to replica.");
419    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
420
421    uint64_t cas = 11;
422    if (op == OPERATION_ADD) {
423        // performing add with a CAS != 0 doesn't make sense...
424        cas = 0;
425    }
426    check(store(h, h1, NULL, op,
427                "key", "somevalue", &i, cas, 1) == ENGINE_NOT_MY_VBUCKET,
428        "Expected not my vbucket");
429    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
430    h1->release(h, NULL, i);
431    return SUCCESS;
432}
433
434//
435// ----------------------------------------------------------------------
436// The actual tests are below.
437// ----------------------------------------------------------------------
438//
439
440static enum test_result test_get_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
441    check(verify_key(h, h1, "k") == ENGINE_KEY_ENOENT, "Expected miss.");
442    return SUCCESS;
443}
444
445static enum test_result test_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
446    item *i = NULL;
447    check(ENGINE_SUCCESS ==
448          store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
449          "Error setting.");
450    h1->release(h, NULL, i);
451    return SUCCESS;
452}
453
454struct handle_pair {
455    ENGINE_HANDLE *h;
456    ENGINE_HANDLE_V1 *h1;
457};
458
459extern "C" {
460    static void conc_del_set_thread(void *arg) {
461        struct handle_pair *hp = static_cast<handle_pair *>(arg);
462        item *it = NULL;
463
464        for (int i = 0; i < 5000; ++i) {
465            store(hp->h, hp->h1, NULL, OPERATION_ADD,
466                  "key", "somevalue", &it);
467            hp->h1->release(hp->h, NULL, it);
468            usleep(10);
469            checkeq(ENGINE_SUCCESS,
470                    store(hp->h, hp->h1, NULL, OPERATION_SET,
471                          "key", "somevalue", &it),
472                    "Error setting.");
473            hp->h1->release(hp->h, NULL, it);
474            usleep(10);
475            // Ignoring the result here -- we're racing.
476            del(hp->h, hp->h1, "key", 0, 0);
477            usleep(10);
478        }
479    }
480}
481
482static enum test_result test_conc_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
483
484    const int n_threads = 8;
485    cb_thread_t threads[n_threads];
486    struct handle_pair hp = {h, h1};
487
488    wait_for_persisted_value(h, h1, "key", "value1");
489
490    for (int i = 0; i < n_threads; i++) {
491        int r = cb_create_thread(&threads[i], conc_del_set_thread, &hp, 0);
492        cb_assert(r == 0);
493    }
494
495    for (int i = 0; i < n_threads; i++) {
496        int r = cb_join_thread(threads[i]);
497        cb_assert(r == 0);
498    }
499
500    wait_for_flusher_to_settle(h, h1);
501
502    testHarness.reload_engine(&h, &h1,
503                              testHarness.engine_path,
504                              testHarness.get_current_testcase()->cfg,
505                              true, false);
506    wait_for_warmup_complete(h, h1);
507
508    cb_assert(0 == get_int_stat(h, h1, "ep_warmed_dups"));
509
510    return SUCCESS;
511}
512
513static enum test_result test_set_get_hit(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
514    item *i = NULL;
515    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
516          "store failure");
517    check_key_value(h, h1, "key", "somevalue", 9);
518    h1->release(h, NULL, i);
519    return SUCCESS;
520}
521
522static enum test_result test_set_get_hit_bin(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
523    char binaryData[] = "abcdefg\0gfedcba";
524    cb_assert(sizeof(binaryData) != strlen(binaryData));
525
526    item *i = NULL;
527    check(ENGINE_SUCCESS ==
528          storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
529                       binaryData, sizeof(binaryData), 82758, &i, 0, 0),
530          "Failed to set.");
531    h1->release(h, NULL, i);
532    check_key_value(h, h1, "key", binaryData, sizeof(binaryData));
533    return SUCCESS;
534}
535
536static enum test_result test_set_with_cas_non_existent(ENGINE_HANDLE *h,
537                                                       ENGINE_HANDLE_V1 *h1) {
538    const char *key = "test_expiry_flush";
539    item *i = NULL;
540
541    check(h1->allocate(h, NULL, &i, key, strlen(key), 10, 0, 0,
542          PROTOCOL_BINARY_RAW_BYTES) == ENGINE_SUCCESS, "Allocation failed.");
543
544    Item *it = reinterpret_cast<Item*>(i);
545    it->setCas(1234);
546
547    uint64_t cas = 0;
548    check(h1->store(h, NULL, i, &cas, OPERATION_SET, 0) == ENGINE_KEY_ENOENT,
549          "Expected not found");
550    h1->release(h, NULL, i);
551
552    return SUCCESS;
553}
554
555static enum test_result test_set_change_flags(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
556    item *i = NULL;
557    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
558          "Failed to set.");
559    h1->release(h, NULL, i);
560
561    item_info info;
562    uint32_t flags = 828258;
563    check(get_item_info(h, h1, &info, "key"), "Failed to get value.");
564    cb_assert(info.flags != flags);
565
566    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
567                       "newvalue", strlen("newvalue"), flags, &i, 0, 0) == ENGINE_SUCCESS,
568          "Failed to set again.");
569    h1->release(h, NULL, i);
570
571    check(get_item_info(h, h1, &info, "key"), "Failed to get value.");
572
573    return info.flags == flags ? SUCCESS : FAIL;
574}
575
576static enum test_result test_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
577    item *i = NULL;
578    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
579          "Failed to do initial set.");
580    h1->release(h, NULL, i);
581    check(store(h, h1, NULL, OPERATION_CAS, "key", "failcas", &i) != ENGINE_SUCCESS,
582          "Failed to fail initial CAS.");
583    h1->release(h, NULL, i);
584    check_key_value(h, h1, "key", "somevalue", 9);
585
586    check(h1->get(h, NULL, &i, "key", 3, 0) == ENGINE_SUCCESS,
587          "Failed to get value.");
588
589    item_info info;
590    info.nvalue = 1;
591    check(h1->get_item_info(h, NULL, i, &info), "Failed to get item info.");
592    h1->release(h, NULL, i);
593
594    check(store(h, h1, NULL, OPERATION_CAS, "key", "winCas", &i,
595                info.cas) == ENGINE_SUCCESS,
596          "Failed to store CAS");
597    h1->release(h, NULL, i);
598    check_key_value(h, h1, "key", "winCas", 6);
599
600    uint64_t cval = 99999;
601    check(store(h, h1, NULL, OPERATION_CAS, "non-existing", "winCas", &i,
602                cval) == ENGINE_KEY_ENOENT,
603          "CAS for non-existing key returned the wrong error code");
604    h1->release(h, NULL, i);
605    return SUCCESS;
606}
607
608static enum test_result test_add(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
609    item *i = NULL;
610    check(store(h, h1, NULL, OPERATION_ADD,"key", "somevalue", &i) == ENGINE_SUCCESS,
611          "Failed to add value.");
612    h1->release(h, NULL, i);
613    check(store(h, h1, NULL, OPERATION_ADD,"key", "somevalue", &i) == ENGINE_NOT_STORED,
614          "Failed to fail to re-add value.");
615    h1->release(h, NULL, i);
616
617    // This aborts on failure.
618    check_key_value(h, h1, "key", "somevalue", 9);
619
620    // Expiration above was an hour, so let's go to The Future
621    testHarness.time_travel(3800);
622
623    check(store(h, h1, NULL, OPERATION_ADD,"key", "newvalue", &i) == ENGINE_SUCCESS,
624          "Failed to add value again.");
625    h1->release(h, NULL, i);
626    check_key_value(h, h1, "key", "newvalue", 8);
627    return SUCCESS;
628}
629
630static enum test_result test_add_add_with_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
631    item *i = NULL;
632    check(store(h, h1, NULL, OPERATION_ADD, "key",
633                "somevalue", &i) == ENGINE_SUCCESS,
634          "Failed set.");
635    check_key_value(h, h1, "key", "somevalue", 9);
636    item_info info;
637    info.nvalue = 1;
638    info.nvalue = 1;
639    check(h1->get_item_info(h, NULL, i, &info) == true,
640          "Should be able to get info");
641
642    item *i2 = NULL;
643    ENGINE_ERROR_CODE ret;
644    check((ret = store(h, h1, NULL, OPERATION_ADD, "key",
645                       "somevalue", &i2, info.cas)) == ENGINE_KEY_EEXISTS,
646          "Should not be able to add the key two times");
647
648    h1->release(h, NULL, i);
649    h1->release(h, NULL, i2);
650    return SUCCESS;
651}
652
653static enum test_result test_replace(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
654    item *i = NULL;
655    check(store(h, h1, NULL, OPERATION_REPLACE,"key", "somevalue", &i) != ENGINE_SUCCESS,
656          "Failed to fail to replace non-existing value.");
657    h1->release(h, NULL, i);
658    check(store(h, h1, NULL, OPERATION_SET,"key", "somevalue", &i) == ENGINE_SUCCESS,
659          "Failed to set value.");
660    h1->release(h, NULL, i);
661    check(store(h, h1, NULL, OPERATION_REPLACE,"key", "somevalue", &i) == ENGINE_SUCCESS,
662          "Failed to replace existing value.");
663    h1->release(h, NULL, i);
664    check_key_value(h, h1, "key", "somevalue", 9);
665    return SUCCESS;
666}
667
668static enum test_result test_replace_with_eviction(ENGINE_HANDLE *h,
669                                                   ENGINE_HANDLE_V1 *h1) {
670    item *i = NULL;
671    check(store(h, h1, NULL, OPERATION_SET,"key", "somevalue", &i) == ENGINE_SUCCESS,
672          "Failed to set value.");
673    h1->release(h, NULL, i);
674    wait_for_flusher_to_settle(h, h1);
675    evict_key(h, h1, "key");
676    int numBgFetched = get_int_stat(h, h1, "ep_bg_fetched");
677
678    check(store(h, h1, NULL, OPERATION_REPLACE,"key", "somevalue1", &i) == ENGINE_SUCCESS,
679          "Failed to replace existing value.");
680
681    check(h1->get_stats(h, NULL, NULL, 0, add_stats) == ENGINE_SUCCESS,
682          "Failed to get stats.");
683    std::string eviction_policy = vals.find("ep_item_eviction_policy")->second;
684    if (eviction_policy == "full_eviction") {
685        numBgFetched++;
686    }
687
688    check(get_int_stat(h, h1, "ep_bg_fetched") == numBgFetched,
689          "Bg fetched value didn't match");
690
691    h1->release(h, NULL, i);
692    check_key_value(h, h1, "key", "somevalue1", 10);
693    return SUCCESS;
694}
695
696static enum test_result test_incr_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
697    uint64_t cas = 0, result = 0;
698    h1->arithmetic(h, NULL, "key", 3, true, false, 1, 0, 0,
699                   &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
700                   0);
701    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected to not find key");
702    return SUCCESS;
703}
704
705static enum test_result test_incr_default(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
706    uint64_t cas = 0, result = 0;
707    check(h1->arithmetic(h, NULL, "key", 3, true, true, 1, 1, 0,
708                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
709                         0) == ENGINE_SUCCESS,
710          "Failed first arith");
711    check(result == 1, "Failed result verification.");
712
713    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
714                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
715                         0) == ENGINE_SUCCESS,
716          "Failed second arith.");
717    check(result == 2, "Failed second result verification.");
718
719    check(h1->arithmetic(h, NULL, "key", 3, true, true, 1, 1, 0,
720                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
721                         0) == ENGINE_SUCCESS,
722          "Failed third arith.");
723    check(result == 3, "Failed third result verification.");
724
725    check_key_value(h, h1, "key", "3", 1);
726    return SUCCESS;
727}
728
729static enum test_result test_append(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
730    item *i = NULL;
731
732    // MB-11332: append on non-existing key should return NOT_STORED
733    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key",
734                       "foo\r\n", 5, 82758, &i, 0, 0)
735          == ENGINE_NOT_STORED,
736          "MB-11332: Failed append.");
737
738    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
739                       "\r\n", 2, 82758, &i, 0, 0)
740          == ENGINE_SUCCESS,
741          "Failed set.");
742    h1->release(h, NULL, i);
743
744    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key",
745                       "foo\r\n", 5, 82758, &i, 0, 0)
746          == ENGINE_SUCCESS,
747          "Failed append.");
748    h1->release(h, NULL, i);
749
750    check_key_value(h, h1, "key", "\r\nfoo\r\n", 7);
751
752    char binaryData1[] = "abcdefg\0gfedcba\r\n";
753    char binaryData2[] = "abzdefg\0gfedcba\r\n";
754    size_t dataSize = 20*1024*1024;
755    char *bigBinaryData3 = new char[dataSize];
756    memset(bigBinaryData3, '\0', dataSize);
757
758    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
759                       binaryData1, sizeof(binaryData1) - 1, 82758, &i, 0, 0)
760          == ENGINE_SUCCESS,
761          "Failed set.");
762    h1->release(h, NULL, i);
763
764    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key",
765                       bigBinaryData3, dataSize, 82758, &i, 0, 0)
766          == ENGINE_E2BIG,
767          "Expected append failure.");
768    h1->release(h, NULL, i);
769    delete[] bigBinaryData3;
770
771    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key",
772                       binaryData2, sizeof(binaryData2) - 1, 82758, &i, 0, 0)
773          == ENGINE_SUCCESS,
774          "Failed append.");
775    h1->release(h, NULL, i);
776
777    std::string expected;
778    expected.append(binaryData1, sizeof(binaryData1) - 1);
779    expected.append(binaryData2, sizeof(binaryData2) - 1);
780
781    check_key_value(h, h1, "key", expected.data(), expected.length());
782    return SUCCESS;
783}
784
785static enum test_result test_prepend(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
786    item *i = NULL;
787
788    // MB-11332: prepend on non-existing key should return NOT_STORED
789    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key",
790                       "foo\r\n", 5, 82758, &i, 0, 0)
791          == ENGINE_NOT_STORED,
792          "MB-11332: Failed prepend.");
793
794
795    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
796                       "\r\n", 2, 82758, &i, 0, 0)
797          == ENGINE_SUCCESS,
798          "Failed set.");
799    h1->release(h, NULL, i);
800
801    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key",
802                       "foo\r\n", 5, 82758, &i, 0, 0)
803          == ENGINE_SUCCESS,
804          "Failed append.");
805    h1->release(h, NULL, i);
806
807    check_key_value(h, h1, "key", "foo\r\n\r\n", 7);
808
809    char binaryData1[] = "abcdefg\0gfedcba\r\n";
810    char binaryData2[] = "abzdefg\0gfedcba\r\n";
811    size_t dataSize = 20*1024*1024;
812    char *bigBinaryData3 = new char[dataSize];
813    memset(bigBinaryData3, '\0', dataSize);
814
815    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
816                       binaryData1, sizeof(binaryData1) - 1, 82758, &i, 0, 0)
817          == ENGINE_SUCCESS,
818          "Failed set.");
819    h1->release(h, NULL, i);
820
821    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key",
822                       bigBinaryData3, dataSize, 82758, &i, 0, 0)
823          == ENGINE_E2BIG,
824          "Expected prepend failure.");
825    h1->release(h, NULL, i);
826    delete[] bigBinaryData3;
827
828    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key",
829                       binaryData2, sizeof(binaryData2) - 1, 82758, &i, 0, 0)
830          == ENGINE_SUCCESS,
831          "Failed append.");
832    h1->release(h, NULL, i);
833
834    std::string expected;
835    expected.append(binaryData2, sizeof(binaryData2) - 1);
836    expected.append(binaryData1, sizeof(binaryData1) - 1);
837
838    check_key_value(h, h1, "key", expected.data(), expected.length());
839    return SUCCESS;
840}
841
842static enum test_result test_append_compressed(ENGINE_HANDLE *h,
843                                               ENGINE_HANDLE_V1 *h1) {
844
845    item *i = NULL;
846
847    size_t len = snappy_max_compressed_length(2);
848    char *newBuf = (char *) malloc (len);
849    snappy_compress("\r\n", 2, newBuf, &len);
850    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key1",
851                       (const char *)newBuf, len, 82758, &i, 0, 0, 3600,
852                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
853          == ENGINE_SUCCESS, "Failed set.");
854    h1->release(h, NULL, i);
855    free (newBuf);
856
857    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key1",
858                       "foo\r\n", 5, 82758, &i, 0, 0)
859          == ENGINE_SUCCESS,
860          "Failed append uncompressed to compressed.");
861    h1->release(h, NULL, i);
862
863    size_t len1 = snappy_max_compressed_length(7);
864    char *newBuf1 = (char *) malloc (len1);
865    snappy_compress("\r\nfoo\r\n", 7, newBuf1, &len1);
866
867    item_info info;
868    check(get_item_info(h, h1, &info, "key1", 0), "checking key and value");
869    check(info.nvalue == 1, "info.nvalue != 1");
870    check(len1 == info.value[0].iov_len, "Value length mismatch");
871    check(memcmp(info.value[0].iov_base, newBuf1, len1) == 0, "Data mismatch");
872    check(info.datatype == 0x02, "Datatype mismatch");
873    free (newBuf1);
874
875    len = snappy_max_compressed_length(3);
876    newBuf = (char *) malloc (len);
877    snappy_compress("bar", 3, newBuf, &len);
878    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key1",
879                       (const char*)newBuf, len, 82758, &i, 0, 0, 3600,
880                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
881            == ENGINE_SUCCESS,
882            "Failed append compressed to compressed.");
883    h1->release(h, NULL, i);
884    free (newBuf);
885
886    len1 = snappy_max_compressed_length(10);
887    newBuf1 = (char *) malloc (len1);
888    snappy_compress("\r\nfoo\r\nbar", 10, newBuf1, &len1);
889
890    check(get_item_info(h, h1, &info, "key1", 0), "checking key and value");
891    check(info.nvalue == 1, "info.nvalue != 1");
892    check(len1 == info.value[0].iov_len, "Value length mismatch");
893    check(memcmp(info.value[0].iov_base, newBuf1, len1) == 0, "Data mismatch");
894    check(info.datatype == 0x02, "Datatype mismatch");
895    free (newBuf1);
896
897    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key2",
898                       "foo", 3, 82758, &i, 0, 0, 3600,
899                       PROTOCOL_BINARY_RAW_BYTES)
900          == ENGINE_SUCCESS, "Failed set.");
901    h1->release(h, NULL, i);
902
903    len = snappy_max_compressed_length(3);
904    newBuf = (char *) malloc (len);
905    snappy_compress("bar", 3, newBuf, &len);
906    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, "key2",
907                       newBuf, len, 82758, &i, 0, 0, 3600,
908                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
909            == ENGINE_SUCCESS,
910            "Failed append compressed to uncompressed.");
911    h1->release(h, NULL, i);
912    free (newBuf);
913
914    check(get_item_info(h, h1, &info, "key2", 0), "checking key and value");
915    check(info.nvalue == 1, "info.nvalue != 1");
916    check(info.value[0].iov_len == 6, "Value length mismatch");
917    check(memcmp(info.value[0].iov_base, "foobar", 6) == 0, "Data mismatch");
918    check(info.datatype == 0x00, "Datatype mismatch");
919
920    return SUCCESS;
921}
922
923static enum test_result test_prepend_compressed(ENGINE_HANDLE *h,
924                                               ENGINE_HANDLE_V1 *h1) {
925
926    item *i = NULL;
927
928    size_t len = snappy_max_compressed_length(2);
929    char *newBuf = (char *) malloc (len);
930    snappy_compress("\r\n", 2, newBuf, &len);
931    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key1",
932                       (const char *)newBuf, len, 82758, &i, 0, 0, 3600,
933                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
934          == ENGINE_SUCCESS, "Failed set.");
935    h1->release(h, NULL, i);
936    free (newBuf);
937
938    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key1",
939                       "foo\r\n", 5, 82758, &i, 0, 0)
940          == ENGINE_SUCCESS,
941          "Failed prepend uncompressed to compressed.");
942    h1->release(h, NULL, i);
943
944    size_t len1 = snappy_max_compressed_length(7);
945    char *newBuf1 = (char *) malloc (len1);
946    snappy_compress("foo\r\n\r\n", 7, newBuf1, &len1);
947
948    item_info info;
949    check(get_item_info(h, h1, &info, "key1", 0), "checking key and value");
950    check(info.nvalue == 1, "info.nvalue != 1");
951    check(len1 == info.value[0].iov_len, "Value length mismatch");
952    check(memcmp(info.value[0].iov_base, newBuf1, len1) == 0, "Data mismatch");
953    check(info.datatype == 0x02, "Datatype mismatch");
954    free (newBuf1);
955
956    len = snappy_max_compressed_length(3);
957    newBuf = (char *) malloc (len);
958    snappy_compress("bar", 3, newBuf, &len);
959    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key1",
960                       (const char*)newBuf, len, 82758, &i, 0, 0, 3600,
961                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
962            == ENGINE_SUCCESS,
963            "Failed prepend compressed to compressed.");
964    h1->release(h, NULL, i);
965    free (newBuf);
966
967    len1 = snappy_max_compressed_length(10);
968    newBuf1 = (char *) malloc (len1);
969    snappy_compress("barfoo\r\n\r\n", 10, newBuf1, &len1);
970
971    check(get_item_info(h, h1, &info, "key1", 0), "checking key and value");
972    check(info.nvalue == 1, "info.nvalue != 1");
973    check(len1 == info.value[0].iov_len, "Value length mismatch");
974    check(memcmp(info.value[0].iov_base, newBuf1, len1) == 0, "Data mismatch");
975    check(info.datatype == 0x02, "Datatype mismatch");
976    free (newBuf1);
977
978    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key2",
979                       "foo", 3, 82758, &i, 0, 0, 3600,
980                       PROTOCOL_BINARY_RAW_BYTES)
981          == ENGINE_SUCCESS, "Failed set.");
982    h1->release(h, NULL, i);
983
984    len = snappy_max_compressed_length(3);
985    newBuf = (char *) malloc (len);
986    snappy_compress("bar", 3, newBuf, &len);
987    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, "key2",
988                       newBuf, len, 82758, &i, 0, 0, 3600,
989                       PROTOCOL_BINARY_DATATYPE_COMPRESSED)
990            == ENGINE_SUCCESS,
991            "Failed prepend compressed to uncompressed.");
992    h1->release(h, NULL, i);
993    free (newBuf);
994
995    check(get_item_info(h, h1, &info, "key2", 0), "checking key and value");
996    check(info.nvalue == 1, "info.nvalue != 1");
997    check(info.value[0].iov_len == 6, "Value length mismatch");
998    check(memcmp(info.value[0].iov_base, "barfoo", 6) == 0, "Data mismatch");
999    check(info.datatype == 0x00, "Datatype mismatch");
1000
1001    return SUCCESS;
1002}
1003
1004static enum test_result test_append_prepend_to_json(ENGINE_HANDLE *h,
1005                                                    ENGINE_HANDLE_V1 *h1) {
1006    item *i = NULL;
1007    item_info info;
1008
1009    const char* key1 = "foo1";
1010    const char* key2 = "foo2";
1011    const char* value1 = "{\"foo1\":\"bar1\"}";
1012    const char* value2 = "{\"foo2\":\"bar2\"}";
1013
1014    // APPEND
1015    check(storeCasVb11(h, h1, NULL, OPERATION_SET, key1,
1016                       value1, strlen(value1), 82758, &i, 0, 0,
1017                       3600, PROTOCOL_BINARY_DATATYPE_JSON)
1018          == ENGINE_SUCCESS, "Failed set.");
1019    h1->release(h, NULL, i);
1020
1021    check(h1->get(h, NULL, &i, key1, strlen(key1), 0) == ENGINE_SUCCESS,
1022            "Unable to get stored item");
1023    h1->release(h, NULL, i);
1024    info.nvalue = 1;
1025    h1->get_item_info(h, NULL, i, &info);
1026    check(checkUTF8JSON((const unsigned char*)info.value[0].iov_base,
1027                        (int)info.value[0].iov_len) == 1, "Expected JSON");
1028    check(info.datatype == PROTOCOL_BINARY_DATATYPE_JSON, "Invalid datatype");
1029
1030    check(storeCasVb11(h, h1, NULL, OPERATION_APPEND, key1,
1031                       value2, strlen(value2), 82758, &i, 0, 0)
1032          == ENGINE_SUCCESS,
1033          "Failed append.");
1034    h1->release(h, NULL, i);
1035
1036    check(h1->get(h, NULL, &i, key1, strlen(key1), 0) == ENGINE_SUCCESS,
1037            "Unable to get stored item");
1038    info.nvalue = 1;
1039    h1->get_item_info(h, NULL, i, &info);
1040    h1->release(h, NULL, i);
1041    check(checkUTF8JSON((const unsigned char*)info.value[0].iov_base,
1042                        (int)info.value[0].iov_len) == 0, "Expected Binary");
1043    check(info.datatype == PROTOCOL_BINARY_RAW_BYTES,
1044                "Invalid datatype after append");
1045
1046    // PREPEND
1047    check(storeCasVb11(h, h1, NULL, OPERATION_SET, key2,
1048                       value1, strlen(value1), 82758, &i, 0, 0,
1049                       3600, PROTOCOL_BINARY_DATATYPE_JSON)
1050          == ENGINE_SUCCESS, "Failed set.");
1051    h1->release(h, NULL, i);
1052
1053    check(h1->get(h, NULL, &i, key2, strlen(key2), 0) == ENGINE_SUCCESS,
1054            "Unable to get stored item");
1055    info.nvalue = 1;
1056    h1->get_item_info(h, NULL, i, &info);
1057    h1->release(h, NULL, i);
1058    check(checkUTF8JSON((const unsigned char*)info.value[0].iov_base,
1059                        (int)info.value[0].iov_len) == 1, "Expected JSON");
1060    check(info.datatype == PROTOCOL_BINARY_DATATYPE_JSON, "Invalid datatype");
1061
1062    check(storeCasVb11(h, h1, NULL, OPERATION_PREPEND, key2,
1063                       value2, strlen(value2), 82758, &i, 0, 0)
1064          == ENGINE_SUCCESS,
1065          "Failed prepend.");
1066    h1->release(h, NULL, i);
1067
1068    check(h1->get(h, NULL, &i, key2, strlen(key2), 0) == ENGINE_SUCCESS,
1069            "Unable to get stored item");
1070    h1->release(h, NULL, i);
1071    info.nvalue = 1;
1072    h1->get_item_info(h, NULL, i, &info);
1073    check(checkUTF8JSON((const unsigned char*)info.value[0].iov_base,
1074                        (int)info.value[0].iov_len) == 0, "Expected Binary");
1075    check(info.datatype == PROTOCOL_BINARY_RAW_BYTES,
1076                "Invalid datatype after prepend");
1077
1078    return SUCCESS;
1079}
1080
1081static enum test_result test_incr(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1082    uint64_t cas = 0, result = 0;
1083    item *i = NULL;
1084    check(store(h, h1, NULL, OPERATION_ADD,"key", "1", &i) == ENGINE_SUCCESS,
1085          "Failed to add value.");
1086    h1->release(h, NULL, i);
1087
1088    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
1089                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1090                         0) == ENGINE_SUCCESS,
1091          "Failed to incr value.");
1092
1093    check_key_value(h, h1, "key", "2", 1);
1094    return SUCCESS;
1095}
1096
1097static enum test_result test_bug2799(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1098    uint64_t cas = 0, result = 0;
1099    item *i = NULL;
1100    check(store(h, h1, NULL, OPERATION_ADD, "key", "1", &i) == ENGINE_SUCCESS,
1101          "Failed to add value.");
1102    h1->release(h, NULL, i);
1103
1104    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
1105                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1106                         0) == ENGINE_SUCCESS,
1107          "Failed to incr value.");
1108
1109    check_key_value(h, h1, "key", "2", 1);
1110
1111    testHarness.time_travel(3617);
1112
1113    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1114    return SUCCESS;
1115}
1116
1117static enum test_result test_flush(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1118    item *i = NULL;
1119    // First try to delete something we know to not be there.
1120    check(del(h, h1, "key", 0, 0) == ENGINE_KEY_ENOENT, "Failed to fail initial delete.");
1121    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1122          "Failed set.");
1123    h1->release(h, NULL, i);
1124    check_key_value(h, h1, "key", "somevalue", 9);
1125    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS,
1126          "Failed to flush");
1127    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1128
1129    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1130          "Failed post-flush set.");
1131    h1->release(h, NULL, i);
1132    check_key_value(h, h1, "key", "somevalue", 9);
1133
1134    return SUCCESS;
1135}
1136
1137static enum test_result test_flush_disabled(ENGINE_HANDLE *h,
1138                                            ENGINE_HANDLE_V1 *h1) {
1139    item *i = NULL;
1140    // start an engine with disabled flush, the flush() should be noop and
1141    // we expect to see the key after flush()
1142
1143    // store a key and check its existence
1144    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1145          "Failed set.");
1146    h1->release(h, NULL, i);
1147    check_key_value(h, h1, "key", "somevalue", 9);
1148    // expect error msg engine does not support operation
1149    check(h1->flush(h, NULL, 0) == ENGINE_ENOTSUP, "Flush should be disabled");
1150    //check the key
1151    check(ENGINE_SUCCESS == verify_key(h, h1, "key"), "Expected key");
1152
1153    // restart engine with flush enabled and redo the test, we expect flush to succeed
1154    std::string param = "flushall_enabled=false";
1155    std::string config = testHarness.get_current_testcase()->cfg;
1156    size_t found = config.find(param);
1157    if(found != config.npos) {
1158        config.replace(found, param.size(), "flushall_enabled=true");
1159    }
1160    testHarness.reload_engine(&h, &h1,
1161                              testHarness.engine_path,
1162                              config.c_str(),
1163                              true, false);
1164    wait_for_warmup_complete(h, h1);
1165
1166    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS, "Flush should be enabled");
1167
1168    //expect missing key
1169    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1170
1171    return SUCCESS;
1172}
1173
1174static enum test_result test_flush_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1175    item *i = NULL;
1176    int mem_used = get_int_stat(h, h1, "mem_used");
1177    int overhead = get_int_stat(h, h1, "ep_overhead");
1178    int cacheSize = get_int_stat(h, h1, "ep_total_cache_size");
1179    int nonResident = get_int_stat(h, h1, "ep_num_non_resident");
1180
1181    int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
1182    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1183          "Failed set.");
1184    h1->release(h, NULL, i);
1185    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i) == ENGINE_SUCCESS,
1186          "Failed set.");
1187    h1->release(h, NULL, i);
1188    testHarness.time_travel(65);
1189    wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
1190
1191    check(ENGINE_SUCCESS == verify_key(h, h1, "key"), "Expected key");
1192    check(ENGINE_SUCCESS == verify_key(h, h1, "key2"), "Expected key2");
1193
1194    check_key_value(h, h1, "key", "somevalue", 9);
1195    check_key_value(h, h1, "key2", "somevalue", 9);
1196
1197    int mem_used2 = get_int_stat(h, h1, "mem_used");
1198    int overhead2 = get_int_stat(h, h1, "ep_overhead");
1199    int cacheSize2 = get_int_stat(h, h1, "ep_total_cache_size");
1200
1201    cb_assert(mem_used2 > mem_used);
1202    // "mem_used2 - overhead2" (i.e., ep_kv_size) should be greater than the hashtable cache size
1203    // due to the checkpoint overhead
1204    cb_assert(mem_used2 - overhead2 > cacheSize2);
1205
1206    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS, "Failed to flush");
1207    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1208    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key2"), "Expected missing key");
1209
1210    wait_for_flusher_to_settle(h, h1);
1211
1212    mem_used2 = get_int_stat(h, h1, "mem_used");
1213    overhead2 = get_int_stat(h, h1, "ep_overhead");
1214    cacheSize2 = get_int_stat(h, h1, "ep_total_cache_size");
1215    int nonResident2 = get_int_stat(h, h1, "ep_num_non_resident");
1216
1217    cb_assert(mem_used2 == mem_used);
1218    cb_assert(overhead2 == overhead);
1219    cb_assert(nonResident2 == nonResident);
1220    cb_assert(cacheSize2 == cacheSize);
1221
1222    return SUCCESS;
1223}
1224
1225static enum test_result test_flush_multiv(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1226    item *i = NULL;
1227    check(set_vbucket_state(h, h1, 2, vbucket_state_active), "Failed to set vbucket state.");
1228    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1229          "Failed set.");
1230    h1->release(h, NULL, i);
1231    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i,
1232                0, 2) == ENGINE_SUCCESS,
1233          "Failed set in vb2.");
1234    h1->release(h, NULL, i);
1235
1236    check(ENGINE_SUCCESS == verify_key(h, h1, "key"), "Expected key");
1237    check(ENGINE_SUCCESS == verify_key(h, h1, "key2", 2), "Expected key2");
1238
1239    check_key_value(h, h1, "key", "somevalue", 9);
1240    check_key_value(h, h1, "key2", "somevalue", 9, 2);
1241
1242    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS, "Failed to flush");
1243
1244    vals.clear();
1245    check(h1->get_stats(h, NULL, NULL, 0, add_stats) == ENGINE_SUCCESS,
1246          "Failed to get stats.");
1247    check(vals.find("ep_flush_all") != vals.end(), "Failed to get the status of flush_all");
1248
1249    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1250    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key2", 2), "Expected missing key");
1251
1252    return SUCCESS;
1253}
1254
1255static int checkCurrItemsAfterShutdown(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1256                                       int numItems2Load, bool shutdownForce) {
1257    std::vector<std::string> keys;
1258    for (int index = 0; index < numItems2Load; ++index) {
1259        std::stringstream s;
1260        s << "keys_2_load-" << index;
1261        std::string key(s.str());
1262        keys.push_back(key);
1263    }
1264
1265    check(get_int_stat(h, h1, "ep_total_persisted") == 0,
1266          "Expected ep_total_persisted equals 0");
1267    check(get_int_stat(h, h1, "curr_items") == 0,
1268          "Expected curr_items equals 0");
1269
1270    // stop flusher before loading new items
1271    protocol_binary_request_header *pkt = createPacket(PROTOCOL_BINARY_CMD_STOP_PERSISTENCE);
1272    check(h1->unknown_command(h, NULL, pkt, add_response) == ENGINE_SUCCESS,
1273          "CMD_STOP_PERSISTENCE failed!");
1274    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
1275          "Failed to stop persistence!");
1276    free(pkt);
1277
1278    std::vector<std::string>::iterator itr;
1279    for (itr = keys.begin(); itr != keys.end(); ++itr) {
1280        item *i;
1281        check(store(h, h1, NULL, OPERATION_SET, itr->c_str(), "oracle", &i, 0, 0)
1282              == ENGINE_SUCCESS, "Failed to store a value");
1283        h1->release(h, NULL, i);
1284    }
1285
1286    check(get_int_stat(h, h1, "ep_total_persisted") == 0,
1287          "Incorrect ep_total_persisted, expected 0");
1288    std::stringstream ss;
1289    ss << "Incorrect curr_items, expected " << numItems2Load;
1290    std::string errmsg(ss.str());
1291    check(get_int_stat(h, h1, "curr_items") == numItems2Load,
1292          errmsg.c_str());
1293
1294    // resume flusher before shutdown + warmup
1295    pkt = createPacket(PROTOCOL_BINARY_CMD_START_PERSISTENCE);
1296    check(h1->unknown_command(h, NULL, pkt, add_response) == ENGINE_SUCCESS,
1297          "CMD_START_PERSISTENCE failed!");
1298    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
1299          "Failed to start persistence!");
1300    free(pkt);
1301
1302    // shutdown engine force and restart
1303    testHarness.reload_engine(&h, &h1,
1304                              testHarness.engine_path,
1305                              testHarness.get_current_testcase()->cfg,
1306                              true, shutdownForce);
1307    wait_for_warmup_complete(h, h1);
1308    return get_int_stat(h, h1, "curr_items");
1309}
1310
1311static enum test_result test_flush_shutdown_force(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1312    int numItems2load = 3000;
1313    bool shutdownForce = true;
1314    int currItems = checkCurrItemsAfterShutdown(h, h1, numItems2load, shutdownForce);
1315    check (currItems <= numItems2load,
1316           "Number of curr items should be <= 3000, unless previous "
1317           "shutdown force had to wait for the flusher");
1318    return SUCCESS;
1319}
1320
1321static enum test_result test_flush_shutdown_noforce(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1322    int numItems2load = 3000;
1323    bool shutdownForce = false;
1324    int currItems = checkCurrItemsAfterShutdown(h, h1, numItems2load, shutdownForce);
1325    check (currItems == numItems2load,
1326           "Number of curr items should be equal to 3000, unless previous "
1327           "shutdown did not wait for the flusher");
1328    return SUCCESS;
1329}
1330
1331static enum test_result test_flush_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1332    item *i = NULL;
1333    // First try to delete something we know to not be there.
1334    check(del(h, h1, "key", 0, 0) == ENGINE_KEY_ENOENT, "Failed to fail initial delete.");
1335    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1336          "Failed set.");
1337    h1->release(h, NULL, i);
1338    check_key_value(h, h1, "key", "somevalue", 9);
1339
1340    // Restart once to ensure written to disk.
1341    testHarness.reload_engine(&h, &h1,
1342                              testHarness.engine_path,
1343                              testHarness.get_current_testcase()->cfg,
1344                              true, false);
1345    wait_for_warmup_complete(h, h1);
1346
1347    // Read value from disk.
1348    check_key_value(h, h1, "key", "somevalue", 9);
1349
1350    // Flush
1351    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS,
1352          "Failed to flush");
1353
1354    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i) == ENGINE_SUCCESS,
1355          "Failed post-flush set.");
1356    h1->release(h, NULL, i);
1357    check_key_value(h, h1, "key2", "somevalue", 9);
1358
1359    // Restart again, ensure written to disk.
1360    testHarness.reload_engine(&h, &h1,
1361                              testHarness.engine_path,
1362                              testHarness.get_current_testcase()->cfg,
1363                              true, false);
1364    wait_for_warmup_complete(h, h1);
1365
1366    check(store(h, h1, NULL, OPERATION_SET, "key3", "somevalue", &i) == ENGINE_SUCCESS,
1367          "Failed post-flush, post-restart set.");
1368    h1->release(h, NULL, i);
1369    check_key_value(h, h1, "key3", "somevalue", 9);
1370
1371    // Read value again, should not be there.
1372    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1373    return SUCCESS;
1374}
1375
1376static enum test_result test_flush_multiv_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1377    item *i = NULL;
1378    check(set_vbucket_state(h, h1, 2, vbucket_state_active), "Failed to set vbucket state.");
1379    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1380          "Failed set.");
1381    h1->release(h, NULL, i);
1382    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i,
1383                0, 2) == ENGINE_SUCCESS,
1384          "Failed set in vb2.");
1385    h1->release(h, NULL, i);
1386
1387    // Restart once to ensure written to disk.
1388    testHarness.reload_engine(&h, &h1,
1389                              testHarness.engine_path,
1390                              testHarness.get_current_testcase()->cfg,
1391                              true, false);
1392    wait_for_warmup_complete(h, h1);
1393
1394    // Read value from disk.
1395    check_key_value(h, h1, "key", "somevalue", 9);
1396
1397    // Flush
1398    check(h1->flush(h, NULL, 0) == ENGINE_SUCCESS,
1399          "Failed to flush");
1400
1401    // Restart again, ensure written to disk.
1402    testHarness.reload_engine(&h, &h1,
1403                              testHarness.engine_path,
1404                              testHarness.get_current_testcase()->cfg,
1405                              true, false);
1406    wait_for_warmup_complete(h, h1);
1407
1408    // Read value again, should not be there.
1409    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1410    check(verify_vbucket_missing(h, h1, 2), "Bucket 2 came back.");
1411    return SUCCESS;
1412}
1413
1414static enum test_result test_delete(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1415    item *i = NULL;
1416    // First try to delete something we know to not be there.
1417    check(del(h, h1, "key", 0, 0) == ENGINE_KEY_ENOENT, "Failed to fail initial delete.");
1418    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i) == ENGINE_SUCCESS,
1419          "Failed set.");
1420    Item *it = reinterpret_cast<Item*>(i);
1421    uint64_t orig_cas = it->getCas();
1422    h1->release(h, NULL, i);
1423    check_key_value(h, h1, "key", "somevalue", 9);
1424
1425    uint64_t cas = 0;
1426    check(h1->remove(h, NULL, "key", 3, &cas, 0) == ENGINE_SUCCESS,
1427          "Failed remove with value.");
1428    check(orig_cas + 1 == cas, "Cas mismatch on delete");
1429    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1430
1431    // Can I time travel to an expired object and delete it?
1432    checkeq(ENGINE_SUCCESS, store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1433            "Failed set.");
1434    h1->release(h, NULL, i);
1435    testHarness.time_travel(3617);
1436    checkeq(ENGINE_KEY_ENOENT, del(h, h1, "key", 0, 0),
1437            "Did not get ENOENT removing an expired object.");
1438    checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"), "Expected missing key");
1439
1440    return SUCCESS;
1441}
1442
1443static enum test_result test_set_delete(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1444    item *i = NULL;
1445    checkeq(ENGINE_SUCCESS, store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1446            "Failed set.");
1447    h1->release(h, NULL, i);
1448    check_key_value(h, h1, "key", "somevalue", 9);
1449    checkeq(ENGINE_SUCCESS, del(h, h1, "key", 0, 0),
1450            "Failed remove with value.");
1451    checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"), "Expected missing key");
1452    wait_for_flusher_to_settle(h, h1);
1453    wait_for_stat_to_be(h, h1, "curr_items", 0);
1454    return SUCCESS;
1455}
1456
1457static enum test_result test_set_delete_invalid_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1458    item *i = NULL;
1459    check(store(h, h1, NULL, OPERATION_SET, "key",
1460                "somevalue", &i) == ENGINE_SUCCESS,
1461          "Failed set.");
1462    check_key_value(h, h1, "key", "somevalue", 9);
1463    item_info info;
1464    info.nvalue = 1;
1465    check(h1->get_item_info(h, NULL, i, &info) == true,
1466          "Should be able to get info");
1467    h1->release(h, NULL, i);
1468
1469    check(del(h, h1, "key", info.cas + 1, 0) == ENGINE_KEY_EEXISTS,
1470          "Didn't expect to be able to remove the item with wrong cas");
1471    return SUCCESS;
1472}
1473
1474static enum test_result test_bug2509(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1475    for (int j = 0; j < 10000; ++j) {
1476        item *itm = NULL;
1477        checkeq(ENGINE_SUCCESS,
1478                store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &itm),
1479                "Failed set.");
1480        h1->release(h, NULL, itm);
1481        usleep(10);
1482        checkeq(ENGINE_SUCCESS, del(h, h1, "key", 0, 0), "Failed remove with value.");
1483        usleep(10);
1484    }
1485
1486    // Restart again, to verify we don't have any duplicates.
1487    testHarness.reload_engine(&h, &h1,
1488                              testHarness.engine_path,
1489                              testHarness.get_current_testcase()->cfg,
1490                              true, false);
1491    wait_for_warmup_complete(h, h1);
1492
1493    return get_int_stat(h, h1, "ep_warmup_dups") == 0 ? SUCCESS : FAIL;
1494}
1495
1496static enum test_result test_bug7023(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1497    std::vector<std::string> keys;
1498    // Make a vbucket mess.
1499    for (int j = 0; j < 10000; ++j) {
1500        std::stringstream ss;
1501        ss << "key" << j;
1502        std::string key(ss.str());
1503        keys.push_back(key);
1504    }
1505
1506    std::vector<std::string>::iterator it;
1507    for (int j = 0; j < 5; ++j) {
1508        check(set_vbucket_state(h, h1, 0, vbucket_state_dead), "Failed set set vbucket 0 dead.");
1509        vbucketDelete(h, h1, 0);
1510        check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
1511              "Expected vbucket deletion to work.");
1512        check(set_vbucket_state(h, h1, 0, vbucket_state_active), "Failed set set vbucket 0 active.");
1513        for (it = keys.begin(); it != keys.end(); ++it) {
1514            item *i;
1515            check(store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str(), &i)
1516                  == ENGINE_SUCCESS, "Failed to store a value");
1517        }
1518    }
1519    wait_for_flusher_to_settle(h, h1);
1520
1521    // Restart again, to verify no data loss.
1522    testHarness.reload_engine(&h, &h1,
1523                              testHarness.engine_path,
1524                              testHarness.get_current_testcase()->cfg,
1525                              true, false);
1526    wait_for_warmup_complete(h, h1);
1527    return get_int_stat(h, h1, "ep_warmup_value_count", "warmup") == 10000 ? SUCCESS : FAIL;
1528}
1529
1530static enum test_result test_delete_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1531    wait_for_persisted_value(h, h1, "key", "value1");
1532
1533    check(del(h, h1, "key", 0, 0) == ENGINE_SUCCESS, "Failed remove with value.");
1534
1535    wait_for_persisted_value(h, h1, "key", "value2");
1536
1537    testHarness.reload_engine(&h, &h1,
1538                              testHarness.engine_path,
1539                              testHarness.get_current_testcase()->cfg,
1540                              true, false);
1541    wait_for_warmup_complete(h, h1);
1542
1543    check_key_value(h, h1, "key", "value2", 6);
1544    check(del(h, h1, "key", 0, 0) == ENGINE_SUCCESS, "Failed remove with value.");
1545    wait_for_flusher_to_settle(h, h1);
1546
1547    testHarness.reload_engine(&h, &h1,
1548                              testHarness.engine_path,
1549                              testHarness.get_current_testcase()->cfg,
1550                              true, false);
1551    wait_for_warmup_complete(h, h1);
1552
1553    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1554
1555    return SUCCESS;
1556}
1557
1558static enum test_result test_get_delete_missing_file(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1559    const char *key = "key";
1560    wait_for_persisted_value(h, h1, key, "value2delete");
1561
1562    // whack the db file and directory where the key is stored
1563    rmdb();
1564
1565    item *i = NULL;
1566    int errorCode = h1->get(h, NULL, &i, key, strlen(key), 0);
1567    h1->release(h, NULL, i);
1568
1569    // ep engine must be unaware of well-being of the db file as long as
1570    // the item is still in the memory
1571    check(errorCode == ENGINE_SUCCESS, "Expected success for get");
1572
1573    i = NULL;
1574    evict_key(h, h1, key);
1575    errorCode = h1->get(h, NULL, &i, key, strlen(key), 0);
1576    h1->release(h, NULL, i);
1577
1578    // ep engine must be now aware of the ill-fated db file where
1579    // the item is supposedly stored
1580    check(errorCode == ENGINE_TMPFAIL, "Expected tmp fail for get");
1581
1582    return SUCCESS;
1583}
1584
1585
1586static enum test_result test_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1587    item *i = NULL;
1588    static const char val[] = "somevalue";
1589    ENGINE_ERROR_CODE ret;
1590    check((ret = store(h, h1, NULL, OPERATION_SET, "key", val, &i)) == ENGINE_SUCCESS,
1591          "Failed set.");
1592    h1->release(h, NULL, i);
1593
1594    testHarness.reload_engine(&h, &h1,
1595                              testHarness.engine_path,
1596                              testHarness.get_current_testcase()->cfg,
1597                              true, false);
1598    wait_for_warmup_complete(h, h1);
1599    check_key_value(h, h1, "key", val, strlen(val));
1600    return SUCCESS;
1601}
1602
1603static enum test_result test_restart_session_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1604    createTapConn(h, h1, "tap_client_thread");
1605
1606    testHarness.reload_engine(&h, &h1,
1607                              testHarness.engine_path,
1608                              testHarness.get_current_testcase()->cfg,
1609                              true, false);
1610    wait_for_warmup_complete(h, h1);
1611    createTapConn(h, h1, "tap_client_thread");
1612
1613    check(h1->get_stats(h, NULL, "tap", 3, add_stats) == ENGINE_SUCCESS,
1614          "Failed to get stats.");
1615    std::string val = vals["eq_tapq:tap_client_thread:backfill_completed"];
1616    check(strcmp(val.c_str(), "true") == 0, "Don't expect the backfill upon restart");
1617    return SUCCESS;
1618}
1619
1620static enum test_result test_specialKeys(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1621    item *i = NULL;
1622    ENGINE_ERROR_CODE ret;
1623
1624    // Simplified Chinese "Couchbase"
1625    static const char key0[] = "沙发数据库";
1626    static const char val0[] = "some Chinese value";
1627    check((ret = store(h, h1, NULL, OPERATION_SET, key0, val0, &i)) == ENGINE_SUCCESS,
1628          "Failed set Chinese key");
1629    check_key_value(h, h1, key0, val0, strlen(val0));
1630    h1->release(h, NULL, i);
1631    // Traditional Chinese "Couchbase"
1632    static const char key1[] = "沙發數據庫";
1633    static const char val1[] = "some Traditional Chinese value";
1634    check((ret = store(h, h1, NULL, OPERATION_SET, key1, val1, &i)) == ENGINE_SUCCESS,
1635          "Failed set Traditional Chinese key");
1636    h1->release(h, NULL, i);
1637    // Korean "couch potato"
1638    static const char key2[] = "쇼파감자";
1639    static const char val2[] = "some Korean value";
1640    check((ret = store(h, h1, NULL, OPERATION_SET, key2, val2, &i)) == ENGINE_SUCCESS,
1641          "Failed set Korean key");
1642    h1->release(h, NULL, i);
1643    // Russian "couch potato"
1644    static const char key3[] = "лодырь, лентяй";
1645    static const char val3[] = "some Russian value";
1646    check((ret = store(h, h1, NULL, OPERATION_SET, key3, val3, &i)) == ENGINE_SUCCESS,
1647          "Failed set Russian key");
1648    h1->release(h, NULL, i);
1649    // Japanese "couch potato"
1650    static const char key4[] = "カウチポテト";
1651    static const char val4[] = "some Japanese value";
1652    check((ret = store(h, h1, NULL, OPERATION_SET, key4, val4, &i)) == ENGINE_SUCCESS,
1653          "Failed set Japanese key");
1654    h1->release(h, NULL, i);
1655    // Indian char key, and no idea what it is
1656    static const char key5[] = "हरियानवी";
1657    static const char val5[] = "some Indian value";
1658    check((ret = store(h, h1, NULL, OPERATION_SET, key5, val5, &i)) == ENGINE_SUCCESS,
1659          "Failed set Indian key");
1660    h1->release(h, NULL, i);
1661    // Portuguese translation "couch potato"
1662    static const char key6[] = "sedentário";
1663    static const char val6[] = "some Portuguese value";
1664    check((ret = store(h, h1, NULL, OPERATION_SET, key6, val6, &i)) == ENGINE_SUCCESS,
1665          "Failed set Portuguese key");
1666    h1->release(h, NULL, i);
1667    // Arabic translation "couch potato"
1668    static const char key7[] = "الحافلةالبطاطة";
1669    static const char val7[] = "some Arabic value";
1670    check((ret = store(h, h1, NULL, OPERATION_SET, key7, val7, &i)) == ENGINE_SUCCESS,
1671          "Failed set Arabic key");
1672    h1->release(h, NULL, i);
1673
1674    testHarness.reload_engine(&h, &h1,
1675                              testHarness.engine_path,
1676                              testHarness.get_current_testcase()->cfg,
1677                              true, false);
1678    wait_for_warmup_complete(h, h1);
1679    check_key_value(h, h1, key0, val0, strlen(val0));
1680    check_key_value(h, h1, key1, val1, strlen(val1));
1681    check_key_value(h, h1, key2, val2, strlen(val2));
1682    check_key_value(h, h1, key3, val3, strlen(val3));
1683    check_key_value(h, h1, key4, val4, strlen(val4));
1684    check_key_value(h, h1, key5, val5, strlen(val5));
1685    check_key_value(h, h1, key6, val6, strlen(val6));
1686    check_key_value(h, h1, key7, val7, strlen(val7));
1687    return SUCCESS;
1688}
1689
1690static enum test_result test_binKeys(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1691    item *i = NULL;
1692    ENGINE_ERROR_CODE ret;
1693
1694    // binary key with char values beyond 0x7F
1695    static const char key0[] = "\xe0\xed\xf1\x6f\x7f\xf8\xfa";
1696    static const char val0[] = "some value val8";
1697    check((ret = store(h, h1, NULL, OPERATION_SET, key0, val0, &i)) == ENGINE_SUCCESS,
1698          "Failed set binary key0");
1699    check_key_value(h, h1, key0, val0, strlen(val0));
1700    h1->release(h, NULL, i);
1701    // binary keys with char values beyond 0x7F
1702    static const char key1[] = "\xf1\xfd\xfe\xff\xf0\xf8\xef";
1703    static const char val1[] = "some value val9";
1704    check((ret = store(h, h1, NULL, OPERATION_SET, key1, val1, &i)) == ENGINE_SUCCESS,
1705          "Failed set binary key1");
1706    check_key_value(h, h1, key1, val1, strlen(val1));
1707    h1->release(h, NULL, i);
1708    // binary keys with special utf-8 BOM (Byte Order Mark) values 0xBB 0xBF 0xEF
1709    static const char key2[] = "\xff\xfe\xbb\xbf\xef";
1710    static const char val2[] = "some utf-8 bom value";
1711    check((ret = store(h, h1, NULL, OPERATION_SET, key2, val2, &i)) == ENGINE_SUCCESS,
1712          "Failed set binary utf-8 bom key");
1713    check_key_value(h, h1, key2, val2, strlen(val2));
1714    h1->release(h, NULL, i);
1715    // binary keys with special utf-16BE BOM values "U+FEFF"
1716    static const char key3[] = "U+\xfe\xff\xefU+\xff\xfe";
1717    static const char val3[] = "some utf-16 bom value";
1718    check((ret = store(h, h1, NULL, OPERATION_SET, key3, val3, &i)) == ENGINE_SUCCESS,
1719          "Failed set binary utf-16 bom key");
1720    check_key_value(h, h1, key3, val3, strlen(val3));
1721    h1->release(h, NULL, i);
1722
1723    testHarness.reload_engine(&h, &h1,
1724                              testHarness.engine_path,
1725                              testHarness.get_current_testcase()->cfg,
1726                              true, false);
1727    wait_for_warmup_complete(h, h1);
1728    check_key_value(h, h1, key0, val0, strlen(val0));
1729    check_key_value(h, h1, key1, val1, strlen(val1));
1730    check_key_value(h, h1, key2, val2, strlen(val2));
1731    check_key_value(h, h1, key3, val3, strlen(val3));
1732    return SUCCESS;
1733}
1734
1735static enum test_result test_restart_bin_val(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1736
1737
1738
1739    char binaryData[] = "abcdefg\0gfedcba";
1740    cb_assert(sizeof(binaryData) != strlen(binaryData));
1741
1742    item *i = NULL;
1743    check(storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
1744                       binaryData, sizeof(binaryData), 82758, &i, 0, 0)
1745          == ENGINE_SUCCESS,
1746          "Failed set.");
1747    h1->release(h, NULL, i);
1748
1749    testHarness.reload_engine(&h, &h1,
1750                              testHarness.engine_path,
1751                              testHarness.get_current_testcase()->cfg,
1752                              true, false);
1753    wait_for_warmup_complete(h, h1);
1754
1755    check_key_value(h, h1, "key", binaryData, sizeof(binaryData));
1756    return SUCCESS;
1757}
1758
1759static enum test_result test_wrong_vb_get(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1760    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1761    check(ENGINE_NOT_MY_VBUCKET == verify_key(h, h1, "key", 1),
1762          "Expected wrong bucket.");
1763    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1764    return SUCCESS;
1765}
1766
1767static enum test_result test_vb_get_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1768    check(set_vbucket_state(h, h1, 1, vbucket_state_pending), "Failed to set vbucket state.");
1769    const void *cookie = testHarness.create_cookie();
1770    testHarness.set_ewouldblock_handling(cookie, false);
1771
1772    item *i = NULL;
1773    check(ENGINE_EWOULDBLOCK == h1->get(h, cookie, &i, "key", strlen("key"), 1),
1774          "Expected woodblock.");
1775    h1->release(h, NULL, i);
1776
1777    testHarness.destroy_cookie(cookie);
1778    return SUCCESS;
1779}
1780
1781static enum test_result test_vb_get_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1782    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set vbucket state.");
1783    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1784    check(ENGINE_NOT_MY_VBUCKET == verify_key(h, h1, "key", 1),
1785          "Expected not my bucket.");
1786    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1787    return SUCCESS;
1788}
1789
1790static enum test_result test_wrong_vb_incr(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1791    uint64_t cas, result;
1792    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1793    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
1794                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1795                         1) == ENGINE_NOT_MY_VBUCKET,
1796          "Expected not my vbucket.");
1797    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1798    return SUCCESS;
1799}
1800
1801static enum test_result test_vb_incr_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1802    const void *cookie = testHarness.create_cookie();
1803    testHarness.set_ewouldblock_handling(cookie, false);
1804    uint64_t cas, result;
1805    check(set_vbucket_state(h, h1, 1, vbucket_state_pending), "Failed to set vbucket state.");
1806    check(h1->arithmetic(h, cookie, "key", 3, true, false, 1, 1, 0,
1807                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1808                         1) == ENGINE_EWOULDBLOCK,
1809          "Expected woodblock.");
1810    testHarness.destroy_cookie(cookie);
1811    return SUCCESS;
1812}
1813
1814static enum test_result test_vb_incr_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1815    uint64_t cas, result;
1816    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set vbucket state.");
1817    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1818    check(h1->arithmetic(h, NULL, "key", 3, true, false, 1, 1, 0,
1819                         &cas, PROTOCOL_BINARY_RAW_BYTES, &result,
1820                         1) == ENGINE_NOT_MY_VBUCKET,
1821          "Expected not my bucket.");
1822    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1823    return SUCCESS;
1824}
1825
1826static enum test_result test_wrong_vb_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1827    return test_wrong_vb_mutation(h, h1, OPERATION_SET);
1828}
1829
1830static enum test_result test_wrong_vb_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1831    return test_wrong_vb_mutation(h, h1, OPERATION_CAS);
1832}
1833
1834static enum test_result test_wrong_vb_add(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1835    return test_wrong_vb_mutation(h, h1, OPERATION_ADD);
1836}
1837
1838static enum test_result test_wrong_vb_replace(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1839    return test_wrong_vb_mutation(h, h1, OPERATION_REPLACE);
1840}
1841
1842static enum test_result test_wrong_vb_append(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1843    return test_wrong_vb_mutation(h, h1, OPERATION_APPEND);
1844}
1845
1846static enum test_result test_wrong_vb_prepend(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1847    return test_wrong_vb_mutation(h, h1, OPERATION_PREPEND);
1848}
1849
1850static enum test_result test_wrong_vb_del(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1851    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1852    check(ENGINE_NOT_MY_VBUCKET == del(h, h1, "key", 0, 1), "Expected wrong bucket.");
1853    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1854    return SUCCESS;
1855}
1856
1857static enum test_result test_expiry(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1858    const char *key = "test_expiry";
1859    const char *data = "some test data here.";
1860
1861    item *it = NULL;
1862
1863    ENGINE_ERROR_CODE rv;
1864    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 2,
1865                      PROTOCOL_BINARY_RAW_BYTES);
1866    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1867
1868    item_info info;
1869    info.nvalue = 1;
1870    if (!h1->get_item_info(h, NULL, it, &info)) {
1871        abort();
1872    }
1873    memcpy(info.value[0].iov_base, data, strlen(data));
1874
1875    uint64_t cas = 0;
1876    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
1877    check(rv == ENGINE_SUCCESS, "Set failed.");
1878    check_key_value(h, h1, key, data, strlen(data));
1879    h1->release(h, NULL, it);
1880
1881    testHarness.time_travel(5);
1882    check(h1->get(h, NULL, &it, key, strlen(key), 0) == ENGINE_KEY_ENOENT,
1883          "Item didn't expire");
1884
1885    int expired_access = get_int_stat(h, h1, "ep_expired_access");
1886    int expired_pager = get_int_stat(h, h1, "ep_expired_pager");
1887    int active_expired = get_int_stat(h, h1, "vb_active_expired");
1888    check(expired_pager == 0, "Expected zero expired item by pager");
1889    check(expired_access == 1, "Expected an expired item on access");
1890    check(active_expired == 1, "Expected an expired active item");
1891    checkeq(ENGINE_SUCCESS, store(h, h1, NULL, OPERATION_SET, key, data, &it),
1892            "Failed set.");
1893    h1->release(h, NULL, it);
1894
1895    std::stringstream ss;
1896    ss << "curr_items stat should be still 1 after ";
1897    ss << "overwriting the key that was expired, but not purged yet";
1898    checkeq(1, get_int_stat(h, h1, "curr_items"), ss.str().c_str());
1899
1900    return SUCCESS;
1901}
1902
1903static enum test_result test_expiry_loader(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1904    const char *key = "test_expiry_loader";
1905    const char *data = "some test data here.";
1906
1907    item *it = NULL;
1908
1909    ENGINE_ERROR_CODE rv;
1910    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 2,
1911                      PROTOCOL_BINARY_RAW_BYTES);
1912    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1913
1914    item_info info;
1915    info.nvalue = 1;
1916    if (!h1->get_item_info(h, NULL, it, &info)) {
1917        abort();
1918    }
1919    memcpy(info.value[0].iov_base, data, strlen(data));
1920
1921    uint64_t cas = 0;
1922    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
1923    check(rv == ENGINE_SUCCESS, "Set failed.");
1924    check_key_value(h, h1, key, data, strlen(data));
1925    h1->release(h, NULL, it);
1926
1927    testHarness.time_travel(3);
1928
1929    check(h1->get(h, NULL, &it, key, strlen(key), 0) == ENGINE_KEY_ENOENT,
1930          "Item didn't expire");
1931
1932    // Restart the engine to ensure the above expired item is not loaded
1933    testHarness.reload_engine(&h, &h1,
1934                              testHarness.engine_path,
1935                              testHarness.get_current_testcase()->cfg,
1936                              true, false);
1937    wait_for_warmup_complete(h, h1);
1938    cb_assert(0 == get_int_stat(h, h1, "ep_warmup_value_count", "warmup"));
1939
1940    return SUCCESS;
1941}
1942
1943static enum test_result test_expiration_on_warmup(ENGINE_HANDLE *h,
1944                                                  ENGINE_HANDLE_V1 *h1) {
1945    const char *key = "KEY";
1946    const char *data = "VALUE";
1947
1948    item *it = NULL;
1949
1950    ENGINE_ERROR_CODE rv;
1951    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 3,
1952                      PROTOCOL_BINARY_RAW_BYTES);
1953    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1954
1955    item_info info;
1956    info.nvalue = 1;
1957    if (!h1->get_item_info(h, NULL, it, &info)) {
1958        abort();
1959    }
1960    memcpy(info.value[0].iov_base, data, strlen(data));
1961
1962    uint64_t cas = 0;
1963    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
1964    check(rv == ENGINE_SUCCESS, "Set failed.");
1965    check_key_value(h, h1, key, data, strlen(data));
1966    h1->release(h, NULL, it);
1967    wait_for_flusher_to_settle(h, h1);
1968
1969    check(get_int_stat(h, h1, "curr_items") == 1, "Failed store item");
1970    testHarness.time_travel(5);
1971
1972    // Restart the engine to ensure the above item is expired
1973    testHarness.reload_engine(&h, &h1,
1974                              testHarness.engine_path,
1975                              testHarness.get_current_testcase()->cfg,
1976                              true, false);
1977    wait_for_warmup_complete(h, h1);
1978    wait_for_flusher_to_settle(h, h1);
1979    check(get_int_stat(h, h1, "curr_items") == 0,
1980            "The item should have been expired.");
1981
1982    return SUCCESS;
1983
1984}
1985
1986static enum test_result test_bug3454(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1987    const char *key = "test_expiry_duplicate_warmup";
1988    const char *data = "some test data here.";
1989
1990    item *it = NULL;
1991
1992    ENGINE_ERROR_CODE rv;
1993    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 5,
1994                      PROTOCOL_BINARY_RAW_BYTES);
1995    check(rv == ENGINE_SUCCESS, "Allocation failed.");
1996
1997    item_info info;
1998    info.nvalue = 1;
1999    if (!h1->get_item_info(h, NULL, it, &info)) {
2000        abort();
2001    }
2002    memcpy(info.value[0].iov_base, data, strlen(data));
2003
2004    uint64_t cas = 0;
2005    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
2006    check(rv == ENGINE_SUCCESS, "Set failed.");
2007    check_key_value(h, h1, key, data, strlen(data));
2008    h1->release(h, NULL, it);
2009    wait_for_flusher_to_settle(h, h1);
2010
2011    // Advance the ep_engine time by 10 sec for the above item to be expired.
2012    testHarness.time_travel(10);
2013    check(h1->get(h, NULL, &it, key, strlen(key), 0) == ENGINE_KEY_ENOENT,
2014          "Item didn't expire");
2015
2016    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 0,
2017                      PROTOCOL_BINARY_RAW_BYTES);
2018    check(rv == ENGINE_SUCCESS, "Allocation failed.");
2019
2020    info.nvalue = 1;
2021    if (!h1->get_item_info(h, NULL, it, &info)) {
2022        abort();
2023    }
2024    memcpy(info.value[0].iov_base, data, strlen(data));
2025
2026    cas = 0;
2027    // Add a new item with the same key.
2028    rv = h1->store(h, NULL, it, &cas, OPERATION_ADD, 0);
2029    check(rv == ENGINE_SUCCESS, "Add failed.");
2030    check_key_value(h, h1, key, data, strlen(data));
2031    h1->release(h, NULL, it);
2032
2033    check(h1->get(h, NULL, &it, key, strlen(key), 0) == ENGINE_SUCCESS,
2034          "Item shouldn't expire");
2035    h1->release(h, NULL, it);
2036
2037    // Restart the engine to ensure the above unexpired new item is loaded
2038    testHarness.reload_engine(&h, &h1,
2039                              testHarness.engine_path,
2040                              testHarness.get_current_testcase()->cfg,
2041                              true, false);
2042    wait_for_warmup_complete(h, h1);
2043    cb_assert(1 == get_int_stat(h, h1, "ep_warmup_value_count", "warmup"));
2044    cb_assert(0 == get_int_stat(h, h1, "ep_warmup_dups", "warmup"));
2045
2046    return SUCCESS;
2047}
2048
2049static enum test_result test_bug3522(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2050    const char *key = "test_expiry_no_items_warmup";
2051    const char *data = "some test data here.";
2052
2053    item *it = NULL;
2054
2055    ENGINE_ERROR_CODE rv;
2056    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(data), 0, 0,
2057                      PROTOCOL_BINARY_RAW_BYTES);
2058    check(rv == ENGINE_SUCCESS, "Allocation failed.");
2059
2060    item_info info;
2061    info.nvalue = 1;
2062    if (!h1->get_item_info(h, NULL, it, &info)) {
2063        abort();
2064    }
2065    memcpy(info.value[0].iov_base, data, strlen(data));
2066
2067    uint64_t cas = 0;
2068    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
2069    check(rv == ENGINE_SUCCESS, "Set failed.");
2070    check_key_value(h, h1, key, data, strlen(data));
2071    h1->release(h, NULL, it);
2072    wait_for_flusher_to_settle(h, h1);
2073
2074    // Add a new item with the same key and 2 sec of expiration.
2075    const char *new_data = "new data here.";
2076    rv = h1->allocate(h, NULL, &it, key, strlen(key), strlen(new_data), 0, 2,
2077                      PROTOCOL_BINARY_RAW_BYTES);
2078    check(rv == ENGINE_SUCCESS, "Allocation failed.");
2079
2080    info.nvalue = 1;
2081    if (!h1->get_item_info(h, NULL, it, &info)) {
2082        abort();
2083    }
2084    memcpy(info.value[0].iov_base, new_data, strlen(new_data));
2085
2086    int pager_runs = get_int_stat(h, h1, "ep_num_expiry_pager_runs");
2087    cas = 0;
2088    rv = h1->store(h, NULL, it, &cas, OPERATION_SET, 0);
2089    check(rv == ENGINE_SUCCESS, "Set failed.");
2090    check_key_value(h, h1, key, new_data, strlen(new_data));
2091    h1->release(h, NULL, it);
2092    testHarness.time_travel(3);
2093    wait_for_stat_change(h, h1, "ep_num_expiry_pager_runs", pager_runs);
2094    wait_for_flusher_to_settle(h, h1);
2095
2096    // Restart the engine.
2097    testHarness.reload_engine(&h, &h1,
2098                              testHarness.engine_path,
2099                              testHarness.get_current_testcase()->cfg,
2100                              true, false);
2101    wait_for_warmup_complete(h, h1);
2102    // TODO: modify this for a better test case
2103    cb_assert(0 == get_int_stat(h, h1, "ep_warmup_dups", "warmup"));
2104
2105    return SUCCESS;
2106}
2107
2108static enum test_result test_get_replica_active_state(ENGINE_HANDLE *h,
2109                                                      ENGINE_HANDLE_V1 *h1) {
2110    protocol_binary_request_header *pkt;
2111    pkt = prepare_get_replica(h, h1, vbucket_state_active);
2112    check(h1->unknown_command(h, NULL, pkt, add_response) ==
2113          ENGINE_SUCCESS, "Get Replica Failed");
2114    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2115          "Expected PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET response.");
2116
2117    return SUCCESS;
2118}
2119
2120static enum test_result test_get_replica_pending_state(ENGINE_HANDLE *h,
2121                                                       ENGINE_HANDLE_V1 *h1) {
2122    protocol_binary_request_header *pkt;
2123
2124    const void *cookie = testHarness.create_cookie();
2125    testHarness.set_ewouldblock_handling(cookie, false);
2126    pkt = prepare_get_replica(h, h1, vbucket_state_pending);
2127    check(h1->unknown_command(h, cookie, pkt, add_response) ==
2128          ENGINE_EWOULDBLOCK, "Should have returned error for pending state");
2129    testHarness.destroy_cookie(cookie);
2130    return SUCCESS;
2131}
2132
2133static enum test_result test_get_replica_dead_state(ENGINE_HANDLE *h,
2134                                                    ENGINE_HANDLE_V1 *h1) {
2135    protocol_binary_request_header *pkt;
2136    pkt = prepare_get_replica(h, h1, vbucket_state_dead);
2137    check(h1->unknown_command(h, NULL, pkt, add_response) ==
2138          ENGINE_SUCCESS, "Get Replica Failed");
2139    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2140          "Expected PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET response.");
2141
2142    return SUCCESS;
2143}
2144
2145static enum test_result test_get_replica(ENGINE_HANDLE *h,
2146                                         ENGINE_HANDLE_V1 *h1) {
2147    protocol_binary_request_header *pkt;
2148    pkt = prepare_get_replica(h, h1, vbucket_state_replica);
2149    check(h1->unknown_command(h, NULL, pkt, add_response) == ENGINE_SUCCESS,
2150                              "Get Replica Failed");
2151    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
2152          "Expected PROTOCOL_BINARY_RESPONSE_SUCCESS response.");
2153    check(strcmp("replicadata", last_body) == 0,
2154                 "Should have returned identical value");
2155
2156    return SUCCESS;
2157}
2158
2159static enum test_result test_get_replica_non_resident(ENGINE_HANDLE *h,
2160                                                      ENGINE_HANDLE_V1 *h1) {
2161
2162    item *i = NULL;
2163    check(store(h, h1, NULL, OPERATION_SET, "key", "value", &i, 0, 0)
2164          == ENGINE_SUCCESS, "Store Failed");
2165    h1->release(h, NULL, i);
2166    wait_for_flusher_to_settle(h, h1);
2167    wait_for_stat_to_be(h, h1, "ep_total_persisted", 1);
2168
2169    evict_key(h, h1, "key", 0, "Ejected.");
2170    check(set_vbucket_state(h, h1, 0, vbucket_state_replica),
2171          "Failed to set vbucket to replica");
2172
2173    get_replica(h, h1, "key", 0);
2174    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "Expected success");
2175
2176    return SUCCESS;
2177}
2178
2179static enum test_result test_get_replica_invalid_key(ENGINE_HANDLE *h,
2180                                                     ENGINE_HANDLE_V1 *h1) {
2181    protocol_binary_request_header *pkt;
2182    bool makeinvalidkey = true;
2183    pkt = prepare_get_replica(h, h1, vbucket_state_replica, makeinvalidkey);
2184    check(h1->unknown_command(h, NULL, pkt, add_response) ==
2185          ENGINE_SUCCESS, "Get Replica Failed");
2186    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2187          "Expected PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET response.");
2188    return SUCCESS;
2189}
2190
2191static enum test_result test_vb_del_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2192    const void *cookie = testHarness.create_cookie();
2193    testHarness.set_ewouldblock_handling(cookie, false);
2194    check(set_vbucket_state(h, h1, 1, vbucket_state_pending), "Failed to set vbucket state.");
2195    check(ENGINE_EWOULDBLOCK == del(h, h1, "key", 0, 1, cookie),
2196          "Expected woodblock.");
2197    testHarness.destroy_cookie(cookie);
2198    return SUCCESS;
2199}
2200
2201static enum test_result test_vb_del_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2202    check(set_vbucket_state(h, h1, 1, vbucket_state_replica), "Failed to set vbucket state.");
2203    int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
2204    check(ENGINE_NOT_MY_VBUCKET == del(h, h1, "key", 0, 1),
2205          "Expected not my vbucket.");
2206    wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
2207    return SUCCESS;
2208}
2209
2210static enum test_result test_touch(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2211    // key is a mandatory field!
2212    touch(h, h1, NULL, 0, (time(NULL) + 10));
2213    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2214
2215    // extlen is a mandatory field!
2216    protocol_binary_request_header *request;
2217    request = createPacket(PROTOCOL_BINARY_CMD_TOUCH, 0, 0, NULL, 0, "akey", 4);
2218    check(h1->unknown_command(h, NULL, request, add_response) == ENGINE_SUCCESS,
2219          "Failed to call touch");
2220    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2221    free(request);
2222
2223    // Try to touch an unknown item...
2224    touch(h, h1, "mykey", 0, (time(NULL) + 10));
2225    check(last_status == PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, "Testing unknown key");
2226
2227    // illegal vbucket
2228    touch(h, h1, "mykey", 5, (time(NULL) + 10));
2229    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET, "Testing illegal vbucket");
2230
2231    // Store the item!
2232    item *itm = NULL;
2233    check(store(h, h1, NULL, OPERATION_SET, "mykey", "somevalue", &itm) == ENGINE_SUCCESS,
2234          "Failed set.");
2235    h1->release(h, NULL, itm);
2236
2237    check_key_value(h, h1, "mykey", "somevalue", strlen("somevalue"));
2238
2239    touch(h, h1, "mykey", 0, (time(NULL) + 10));
2240    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch mykey");
2241
2242    // time-travel 9 secs..
2243    testHarness.time_travel(9);
2244
2245    // The item should still exist
2246    check_key_value(h, h1, "mykey", "somevalue", 9);
2247
2248    // time-travel 2 secs..
2249    testHarness.time_travel(2);
2250
2251    // The item should have expired now...
2252    check(h1->get(h, NULL, &itm, "mykey", 5, 0) == ENGINE_KEY_ENOENT, "Item should be gone");
2253    return SUCCESS;
2254}
2255
2256static enum test_result test_touch_mb7342(ENGINE_HANDLE *h,
2257                                          ENGINE_HANDLE_V1 *h1) {
2258    const char *key = "MB-7342";
2259    // Store the item!
2260    item *itm = NULL;
2261    check(store(h, h1, NULL, OPERATION_SET, key, "v", &itm) == ENGINE_SUCCESS,
2262          "Failed set.");
2263    h1->release(h, NULL, itm);
2264
2265    touch(h, h1, key, 0, 0);
2266    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch key");
2267
2268    check_key_value(h, h1, key, "v", 1);
2269
2270    // Travel a loong time to see if the object is still there (the default
2271    // store sets an exp time of 3600
2272    testHarness.time_travel(3700);
2273
2274    check_key_value(h, h1, key, "v", 1);
2275
2276    return SUCCESS;
2277}
2278
2279static enum test_result test_touch_mb10277(ENGINE_HANDLE *h,
2280                                            ENGINE_HANDLE_V1 *h1) {
2281    const char *key = "MB-10277";
2282    // Store the item!
2283    item *itm = NULL;
2284    check(store(h, h1, NULL, OPERATION_SET, key, "v", &itm) == ENGINE_SUCCESS,
2285          "Failed set.");
2286    h1->release(h, NULL, itm);
2287    wait_for_flusher_to_settle(h, h1);
2288    evict_key(h, h1, key, 0, "Ejected.");
2289
2290    touch(h, h1, key, 0, 3600); // A new expiration time remains in the same.
2291    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch key");
2292
2293    return SUCCESS;
2294}
2295
2296static enum test_result test_gat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2297    // key is a mandatory field!
2298    gat(h, h1, NULL, 0, 10);
2299    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2300
2301    // extlen is a mandatory field!
2302    protocol_binary_request_header *request;
2303    request = createPacket(PROTOCOL_BINARY_CMD_GAT, 0, 0, NULL, 0, "akey", 4);
2304    check(h1->unknown_command(h, NULL, request, add_response) == ENGINE_SUCCESS,
2305          "Failed to call gat");
2306    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2307    free(request);
2308
2309    // Try to gat an unknown item...
2310    gat(h, h1, "mykey", 0, 10);
2311    check(last_status == PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, "Testing unknown key");
2312
2313    // illegal vbucket
2314    gat(h, h1, "mykey", 5, 10);
2315    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET, "Testing illegal vbucket");
2316
2317    // Store the item!
2318    item *itm = NULL;
2319    check(store(h, h1, NULL, OPERATION_SET, "mykey", "{\"some\":\"value\"}",
2320                &itm, 0, 0, 3600, PROTOCOL_BINARY_DATATYPE_JSON) == ENGINE_SUCCESS,
2321          "Failed set.");
2322    h1->release(h, NULL, itm);
2323
2324    check_key_value(h, h1, "mykey", "{\"some\":\"value\"}",
2325            strlen("{\"some\":\"value\"}"));
2326
2327    gat(h, h1, "mykey", 0, 10);
2328    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "gat mykey");
2329    check(last_datatype == PROTOCOL_BINARY_DATATYPE_JSON, "Expected datatype to be JSON");
2330    check(memcmp(last_body, "{\"some\":\"value\"}", sizeof("{\"some\":\"value\"}")) == 0,
2331          "Invalid data returned");
2332
2333    // time-travel 9 secs..
2334    testHarness.time_travel(9);
2335
2336    // The item should still exist
2337    check_key_value(h, h1, "mykey", "{\"some\":\"value\"}",
2338                    strlen("{\"some\":\"value\"}"));
2339
2340    // time-travel 2 secs..
2341    testHarness.time_travel(2);
2342
2343    // The item should have expired now...
2344    check(h1->get(h, NULL, &itm, "mykey", 5, 0) == ENGINE_KEY_ENOENT, "Item should be gone");
2345    return SUCCESS;
2346}
2347
2348static enum test_result test_gatq(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2349    // key is a mandatory field!
2350    gat(h, h1, NULL, 0, 10, true);
2351    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2352
2353    // extlen is a mandatory field!
2354    protocol_binary_request_header *request;
2355    request = createPacket(PROTOCOL_BINARY_CMD_GATQ, 0, 0, NULL, 0, "akey", 4);
2356    check(h1->unknown_command(h, NULL, request, add_response) == ENGINE_SUCCESS,
2357          "Failed to call gatq");
2358    check(last_status == PROTOCOL_BINARY_RESPONSE_EINVAL, "Testing invalid arguments");
2359    free(request);
2360
2361    // Try to gatq an unknown item...
2362    last_status = static_cast<protocol_binary_response_status>(0xffff);
2363    gat(h, h1, "mykey", 0, 10, true);
2364
2365    // We should not have sent any response!
2366    check(last_status == (protocol_binary_response_status)0xffff, "Testing unknown key");
2367
2368    // illegal vbucket
2369    gat(h, h1, "mykey", 5, 10, true);
2370    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2371          "Testing illegal vbucket");
2372
2373    // Store the item!
2374    item *itm = NULL;
2375    check(store(h, h1, NULL, OPERATION_SET, "mykey", "{\"some\":\"value\"}",
2376                &itm, 0, 0, 3600, PROTOCOL_BINARY_DATATYPE_JSON) == ENGINE_SUCCESS,
2377          "Failed set.");
2378    h1->release(h, NULL, itm);
2379
2380    check_key_value(h, h1, "mykey", "{\"some\":\"value\"}",
2381                    strlen("{\"some\":\"value\"}"));
2382
2383    gat(h, h1, "mykey", 0, 10, true);
2384    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "gat mykey");
2385    check(last_datatype == PROTOCOL_BINARY_DATATYPE_JSON, "Expected datatype to be JSON");
2386    check(memcmp(last_body, "{\"some\":\"value\"}", sizeof("{\"some\":\"value\"}")) == 0,
2387          "Invalid data returned");
2388
2389    // time-travel 9 secs..
2390    testHarness.time_travel(9);
2391
2392    // The item should still exist
2393    check_key_value(h, h1, "mykey", "{\"some\":\"value\"}",
2394                    strlen("{\"some\":\"value\"}"));
2395
2396    // time-travel 2 secs..
2397    testHarness.time_travel(2);
2398
2399    // The item should have expired now...
2400    check(h1->get(h, NULL, &itm, "mykey", 5, 0) == ENGINE_KEY_ENOENT, "Item should be gone");
2401    return SUCCESS;
2402}
2403
2404static enum test_result test_mb5215(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2405    item *itm = NULL;
2406    check(store(h, h1, NULL, OPERATION_SET, "coolkey", "cooler", &itm)
2407          == ENGINE_SUCCESS, "Failed set.");
2408    h1->release(h, NULL, itm);
2409
2410    check_key_value(h, h1, "coolkey", "cooler", strlen("cooler"));
2411
2412    // set new exptime to 111
2413    int expTime = time(NULL) + 111;
2414
2415    touch(h, h1, "coolkey", 0, expTime);
2416    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch coolkey");
2417
2418    //reload engine
2419    testHarness.reload_engine(&h, &h1,
2420                              testHarness.engine_path,
2421                              testHarness.get_current_testcase()->cfg,
2422                              true, false);
2423    wait_for_warmup_complete(h, h1);
2424
2425    //verify persisted expiration time
2426    const char *statkey = "key coolkey 0";
2427    int newExpTime;
2428    check(h1->get(h, NULL, &itm, "coolkey", 7, 0) == ENGINE_SUCCESS,
2429          "Missing key");
2430    h1->release(h, NULL, itm);
2431    newExpTime = get_int_stat(h, h1, "key_exptime", statkey);
2432    check(newExpTime == expTime, "Failed to persist new exptime");
2433
2434    // evict key, touch expiration time, and verify
2435    evict_key(h, h1, "coolkey", 0, "Ejected.");
2436
2437    expTime = time(NULL) + 222;
2438    touch(h, h1, "coolkey", 0, expTime);
2439    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch coolkey");
2440
2441    testHarness.reload_engine(&h, &h1,
2442                              testHarness.engine_path,
2443                              testHarness.get_current_testcase()->cfg,
2444                              true, false);
2445    wait_for_warmup_complete(h, h1);
2446
2447    check(h1->get(h, NULL, &itm, "coolkey", 7, 0) == ENGINE_SUCCESS,
2448          "Missing key");
2449    h1->release(h, NULL, itm);
2450    newExpTime = get_int_stat(h, h1, "key_exptime", statkey);
2451    check(newExpTime == expTime, "Failed to persist new exptime");
2452
2453    return SUCCESS;
2454}
2455
2456static enum test_result test_alloc_limit(ENGINE_HANDLE *h,
2457                                         ENGINE_HANDLE_V1 *h1) {
2458    item *it = NULL;
2459    ENGINE_ERROR_CODE rv;
2460
2461    rv = h1->allocate(h, NULL, &it, "key", 3, 20 * 1024 * 1024, 0, 0,
2462                      PROTOCOL_BINARY_RAW_BYTES);
2463    check(rv == ENGINE_SUCCESS, "Allocated 20MB item");
2464    h1->release(h, NULL, it);
2465
2466    rv = h1->allocate(h, NULL, &it, "key", 3, (20 * 1024 * 1024) + 1, 0, 0,
2467                      PROTOCOL_BINARY_RAW_BYTES);
2468    check(rv == ENGINE_E2BIG, "Object too big");
2469
2470    return SUCCESS;
2471}
2472
2473static enum test_result test_whitespace_db(ENGINE_HANDLE *h,
2474                                           ENGINE_HANDLE_V1 *h1) {
2475    vals.clear();
2476    check(h1->get_stats(h, NULL, NULL, 0, add_stats) == ENGINE_SUCCESS,
2477          "Failed to get stats.");
2478    if (vals["ep_dbname"] != std::string(WHITESPACE_DB)) {
2479        std::cerr << "Expected dbname = ``" << WHITESPACE_DB << "''"
2480                  << ", got ``" << vals["ep_dbname"] << "''" << std::endl;
2481        return FAIL;
2482    }
2483
2484    check(access(WHITESPACE_DB, F_OK) != -1, "I expected the whitespace db to exist");
2485    return SUCCESS;
2486}
2487
2488static enum test_result test_memory_limit(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2489    set_param(h, h1, protocol_binary_engine_param_flush, "mutation_mem_threshold", "95");
2490    int used = get_int_stat(h, h1, "mem_used");
2491    double mem_threshold =
2492        static_cast<double>(get_int_stat(h, h1, "ep_mutation_mem_threshold")) / 100;
2493    int max = static_cast<int>(get_int_stat(h, h1, "ep_max_size") * mem_threshold);
2494    check(get_int_stat(h, h1, "ep_oom_errors") == 0 &&
2495          get_int_stat(h, h1, "ep_tmp_oom_errors") == 0, "Expected no OOM errors.");
2496    cb_assert(used < max);
2497
2498    char data[8192];
2499    memset(data, 'x', sizeof(data));
2500    size_t vlen = max - used - 192;
2501    data[vlen] = 0x00;
2502
2503    item *i = NULL;
2504    // So if we add an item,
2505    check(store(h, h1, NULL, OPERATION_SET, "key", data, &i) == ENGINE_SUCCESS,
2506          "store failure");
2507    check_key_value(h, h1, "key", data, vlen);
2508    h1->release(h, NULL, i);
2509
2510    // There should be no room for another.
2511    ENGINE_ERROR_CODE second = store(h, h1, NULL, OPERATION_SET, "key2", data, &i);
2512    check(second == ENGINE_ENOMEM || second == ENGINE_TMPFAIL,
2513          "should have failed second set");
2514    h1->release(h, NULL, i);
2515    check(get_int_stat(h, h1, "ep_oom_errors") == 1 ||
2516          get_int_stat(h, h1, "ep_tmp_oom_errors") == 1, "Expected an OOM error.");
2517
2518    ENGINE_ERROR_CODE overwrite = store(h, h1, NULL, OPERATION_SET, "key", data, &i);
2519    check(overwrite == ENGINE_ENOMEM || overwrite == ENGINE_TMPFAIL,
2520          "should have failed second override");
2521    h1->release(h, NULL, i);
2522    check(get_int_stat(h, h1, "ep_oom_errors") == 2 ||
2523          get_int_stat(h, h1, "ep_tmp_oom_errors") == 2, "Expected another OOM error.");
2524    check_key_value(h, h1, "key", data, vlen);
2525    check(ENGINE_SUCCESS != verify_key(h, h1, "key2"), "Expected a failure in GET");
2526    int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
2527    // Until we remove that item
2528    check(del(h, h1, "key", 0, 0) == ENGINE_SUCCESS, "Failed remove with value.");
2529    check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
2530    testHarness.time_travel(65);
2531    wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
2532
2533    check(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue2", &i) == ENGINE_SUCCESS,
2534          "should have succeded on the last set");
2535    check_key_value(h, h1, "key2", "somevalue2", 10);
2536    h1->release(h, NULL, i);
2537    return SUCCESS;
2538}
2539
2540static enum test_result test_vbucket_get_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2541    return verify_vbucket_missing(h, h1, 1) ? SUCCESS : FAIL;
2542}
2543
2544static enum test_result test_vbucket_get(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2545    return verify_vbucket_state(h, h1, 0, vbucket_state_active) ? SUCCESS : FAIL;
2546}
2547
2548static enum test_result test_vbucket_create(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2549    if (!verify_vbucket_missing(h, h1, 1)) {
2550        fprintf(stderr, "vbucket wasn't missing.\n");
2551        return FAIL;
2552    }
2553
2554    if (!set_vbucket_state(h, h1, 1, vbucket_state_active)) {
2555        fprintf(stderr, "set state failed.\n");
2556        return FAIL;
2557    }
2558
2559    return verify_vbucket_state(h, h1, 1, vbucket_state_active) ? SUCCESS : FAIL;
2560}
2561
2562static enum test_result test_vbucket_compact(ENGINE_HANDLE *h,
2563                                             ENGINE_HANDLE_V1 *h1) {
2564    const char *key = "Carss";
2565    const char *value = "pollute";
2566    if (!verify_vbucket_missing(h, h1, 0)) {
2567        fprintf(stderr, "vbucket wasn't missing.\n");
2568        return FAIL;
2569    }
2570
2571    if (!set_vbucket_state(h, h1, 0, vbucket_state_active)) {
2572        fprintf(stderr, "set state failed.\n");
2573        return FAIL;
2574    }
2575
2576    check(verify_vbucket_state(h, h1, 0, vbucket_state_active),
2577            "VBucket state not active");
2578
2579    // Set two keys - one to be expired and other to remain...
2580    item *itm = NULL;
2581    check(store(h, h1, NULL, OPERATION_SET, key, value, &itm)
2582          == ENGINE_SUCCESS, "Failed set.");
2583    h1->release(h, NULL, itm);
2584
2585    check_key_value(h, h1, key, value, strlen(value));
2586
2587    // Set a non-expiring key...
2588    check(store(h, h1, NULL, OPERATION_SET, "trees", "cleanse", &itm)
2589          == ENGINE_SUCCESS, "Failed set.");
2590    h1->release(h, NULL, itm);
2591
2592    check_key_value(h, h1, "trees", "cleanse", strlen("cleanse"));
2593
2594    touch(h, h1, key, 0, 11);
2595    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "touch Carss");
2596
2597    testHarness.time_travel(12);
2598    wait_for_flusher_to_settle(h, h1);
2599
2600    check(get_int_stat(h, h1, "vb_0:purge_seqno", "vbucket-seqno") == 0,
2601            "purge_seqno not found to be zero before compaction");
2602
2603    // Compaction on VBucket
2604    compact_db(h, h1, 0, 2, 3, 1);
2605
2606    check(get_int_stat(h, h1, "ep_pending_compactions") == 0,
2607    "ep_pending_compactions stat did not tick down after compaction command");
2608
2609    // the key tree and its value should be intact...
2610    check(verify_key(h, h1, "trees") == ENGINE_SUCCESS,
2611          "key trees should be found.");
2612    // the key Carrs should have disappeared...
2613    int val = verify_key(h, h1, "Carss");
2614    check(val == ENGINE_KEY_ENOENT, "Key Carss has not expired.");
2615
2616    check(get_int_stat(h, h1, "vb_0:purge_seqno", "vbucket-seqno") == 3,
2617        "purge_seqno didn't match expected value");
2618
2619    return SUCCESS;
2620}
2621
2622struct comp_thread_ctx {
2623    ENGINE_HANDLE *h;
2624    ENGINE_HANDLE_V1 *h1;
2625    uint16_t vbid;
2626};
2627
2628extern "C" {
2629    static void compaction_thread(void *arg) {
2630        struct comp_thread_ctx *ctx = static_cast<comp_thread_ctx *>(arg);
2631        compact_db(ctx->h, ctx->h1, ctx->vbid, 0, 0, 0);
2632    }
2633}
2634
2635static enum test_result test_multiple_vb_compactions(ENGINE_HANDLE *h,
2636                                                     ENGINE_HANDLE_V1 *h1) {
2637    for (uint16_t i = 0; i < 4; ++i) {
2638        if (!set_vbucket_state(h, h1, i, vbucket_state_active)) {
2639            fprintf(stderr, "set state failed for vbucket %d.\n", i);
2640            return FAIL;
2641        }
2642        check(verify_vbucket_state(h, h1, i, vbucket_state_active),
2643              "VBucket state not active");
2644    }
2645
2646    std::vector<std::string> keys;
2647    for (int j = 0; j < 20000; ++j) {
2648        std::stringstream ss;
2649        ss << "key" << j;
2650        std::string key(ss.str());
2651        keys.push_back(key);
2652    }
2653
2654    int count = 0;
2655    std::vector<std::string>::iterator it;
2656    for (it = keys.begin(); it != keys.end(); ++it) {
2657        uint16_t vbid = count % 4;
2658        item *i;
2659        check(store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str(), &i, 0, vbid)
2660              == ENGINE_SUCCESS, "Failed to store a value");
2661        h1->release(h, NULL, i);
2662        ++count;
2663    }
2664
2665    // Compact multiple vbuckets.
2666    const int n_threads = 4;
2667    cb_thread_t threads[n_threads];
2668    struct comp_thread_ctx ctx[n_threads];
2669
2670    for (int i = 0; i < n_threads; i++) {
2671        ctx[i].h = h;
2672        ctx[i].h1 = h1;
2673        ctx[i].vbid = static_cast<uint16_t>(i);
2674        int r = cb_create_thread(&threads[i], compaction_thread, &ctx[i], 0);
2675        cb_assert(r == 0);
2676    }
2677
2678    for (int i = 0; i < n_threads; i++) {
2679        int r = cb_join_thread(threads[i]);
2680        cb_assert(r == 0);
2681    }
2682
2683    check(get_int_stat(h, h1, "ep_pending_compactions") == 0,
2684    "ep_pending_compactions stat did not tick down after compaction command");
2685
2686    return SUCCESS;
2687}
2688
2689static enum test_result
2690test_multi_vb_compactions_with_workload(ENGINE_HANDLE *h,
2691                                        ENGINE_HANDLE_V1 *h1) {
2692    for (uint16_t i = 0; i < 4; ++i) {
2693        if (!set_vbucket_state(h, h1, i, vbucket_state_active)) {
2694            fprintf(stderr, "set state failed for vbucket %d.\n", i);
2695            return FAIL;
2696        }
2697        check(verify_vbucket_state(h, h1, i, vbucket_state_active),
2698              "VBucket state not active");
2699    }
2700
2701    std::vector<std::string> keys;
2702    for (int j = 0; j < 10000; ++j) {
2703        std::stringstream ss;
2704        ss << "key" << j;
2705        std::string key(ss.str());
2706        keys.push_back(key);
2707    }
2708
2709    int count = 0;
2710    std::vector<std::string>::iterator it;
2711    for (it = keys.begin(); it != keys.end(); ++it) {
2712        uint16_t vbid = count % 4;
2713        item *i;
2714        check(store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str(), &i, 0, vbid)
2715              == ENGINE_SUCCESS, "Failed to store a value");
2716        h1->release(h, NULL, i);
2717        ++count;
2718    }
2719    wait_for_flusher_to_settle(h, h1);
2720
2721    for (int i = 0; i < 2; ++i) {
2722        count = 0;
2723        for (it = keys.begin(); it != keys.end(); ++it) {
2724            uint16_t vbid = count % 4;
2725            item *i = NULL;
2726            check(h1->get(h, NULL, &i, it->c_str(), strlen(it->c_str()), vbid) ==
2727                  ENGINE_SUCCESS, "Unable to get stored item");
2728            h1->release(h, NULL, i);
2729            ++count;
2730        }
2731    }
2732    wait_for_str_stat_to_be(h, h1, "ep_workload_pattern", "read_heavy", NULL);
2733
2734    // Compact multiple vbuckets.
2735    const int n_threads = 4;
2736    cb_thread_t threads[n_threads];
2737    struct comp_thread_ctx ctx[n_threads];
2738
2739    for (int i = 0; i < n_threads; i++) {
2740        ctx[i].h = h;
2741        ctx[i].h1 = h1;
2742        ctx[i].vbid = static_cast<uint16_t>(i);
2743        int r = cb_create_thread(&threads[i], compaction_thread, &ctx[i], 0);
2744        cb_assert(r == 0);
2745    }
2746
2747    for (int i = 0; i < n_threads; i++) {
2748        int r = cb_join_thread(threads[i]);
2749        cb_assert(r == 0);
2750    }
2751
2752    check(get_int_stat(h, h1, "ep_pending_compactions") == 0,
2753    "ep_pending_compactions stat did not tick down after compaction command");
2754
2755    return SUCCESS;
2756}
2757
2758static enum test_result vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
2759                                             const char* value = NULL) {
2760    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed to set vbucket state.");
2761
2762    vbucketDelete(h, h1, 2, value);
2763    check(last_status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
2764          "Expected failure deleting non-existent bucket.");
2765
2766    check(set_vbucket_state(h, h1, 1, vbucket_state_dead), "Failed set set vbucket 1 state.");
2767
2768    vbucketDelete(h, h1, 1, value);
2769    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
2770          "Expected failure deleting non-existent bucket.");
2771
2772    check(verify_vbucket_missing(h, h1, 1),
2773          "vbucket 0 was not missing after deleting it.");
2774
2775    return SUCCESS;
2776}
2777
2778static enum test_result test_vbucket_destroy_stats(ENGINE_HANDLE *h,
2779                                                   ENGINE_HANDLE_V1 *h1) {
2780
2781    int mem_used = get_int_stat(h, h1, "mem_used");
2782    int cacheSize = get_int_stat(h, h1, "ep_total_cache_size");
2783    int overhead = get_int_stat(h, h1, "ep_overhead");
2784    int nonResident = get_int_stat(h, h1, "ep_num_non_resident");
2785
2786    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed to set vbucket state.");
2787
2788    std::vector<std::string> keys;
2789    for (int j = 0; j < 2000; ++j) {
2790        std::stringstream ss;
2791        ss << "key" << j;
2792        std::string key(ss.str());
2793        keys.push_back(key);
2794    }
2795
2796    int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
2797    std::vector<std::string>::iterator it;
2798    for (it = keys.begin(); it != keys.end(); ++it) {
2799        item *i;
2800        check(store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str(), &i, 0, 1)
2801              == ENGINE_SUCCESS, "Failed to store a value");
2802        h1->release(h, NULL, i);
2803    }
2804    wait_for_flusher_to_settle(h, h1);
2805    testHarness.time_travel(65);
2806    wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
2807
2808    check(set_vbucket_state(h, h1, 1, vbucket_state_dead), "Failed set set vbucket 1 state.");
2809
2810    int vbucketDel = get_int_stat(h, h1, "ep_vbucket_del");
2811    vbucketDelete(h, h1, 1);
2812    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
2813          "Expected failure deleting non-existent bucket.");
2814
2815    check(verify_vbucket_missing(h, h1, 1),
2816          "vbucket 1 was not missing after deleting it.");
2817
2818    wait_for_stat_change(h, h1, "ep_vbucket_del", vbucketDel);
2819
2820    wait_for_stat_to_be(h, h1, "mem_used", mem_used);
2821    wait_for_stat_to_be(h, h1, "ep_total_cache_size", cacheSize);
2822    wait_for_stat_to_be(h, h1, "ep_overhead", overhead);
2823    wait_for_stat_to_be(h, h1, "ep_num_non_resident", nonResident);
2824
2825    return SUCCESS;
2826}
2827
2828static enum test_result vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
2829                                                const char* value = NULL) {
2830    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed to set vbucket state.");
2831
2832    // Store a value so the restart will try to resurrect it.
2833    item *i = NULL;
2834    check(store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i, 0, 1)
2835          == ENGINE_SUCCESS, "Failed to set a value");
2836    check_key_value(h, h1, "key", "somevalue", 9, 1);
2837    h1->release(h, NULL, i);
2838
2839    // Reload to get a flush forced.
2840    testHarness.reload_engine(&h, &h1,
2841                              testHarness.engine_path,
2842                              testHarness.get_current_testcase()->cfg,
2843                              true, false);
2844    wait_for_warmup_complete(h, h1);
2845
2846    check(verify_vbucket_state(h, h1, 1, vbucket_state_active),
2847          "Bucket state was what it was initially, after restart.");
2848    check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed to set vbucket state.");
2849    check_key_value(h, h1, "key", "somevalue", 9, 1);
2850
2851    check(set_vbucket_state(h, h1, 1, vbucket_state_dead), "Failed set set vbucket 1 state.");
2852
2853    vbucketDelete(h, h1, 1, value);
2854    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
2855          "Expected failure deleting non-existent bucket.");
2856
2857    check(verify_vbucket_missing(h, h1, 1),
2858          "vbucket 1 was not missing after deleting it.");
2859
2860    testHarness.reload_engine(&h, &h1,
2861                              testHarness.engine_path,
2862                              testHarness.get_current_testcase()->cfg,
2863                              true, false);
2864    wait_for_warmup_complete(h, h1);
2865
2866    if (verify_vbucket_state(h, h1, 1, vbucket_state_pending, true)) {
2867        std::cerr << "Bucket came up in pending state after delete." << std::endl;
2868        abort();
2869    }
2870
2871    check(verify_vbucket_missing(h, h1, 1),
2872          "vbucket 1 was not missing after restart.");
2873
2874    return SUCCESS;
2875}
2876
2877static enum test_result test_async_vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2878    return vbucket_destroy(h, h1);
2879}
2880
2881static enum test_result test_sync_vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2882    return vbucket_destroy(h, h1, "async=0");
2883}
2884
2885static enum test_result test_async_vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2886    return vbucket_destroy_restart(h, h1);
2887}
2888
2889static enum test_result test_sync_vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2890    return vbucket_destroy_restart(h, h1, "async=0");
2891}
2892
2893static enum test_result test_vb_set_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2894    return test_pending_vb_mutation(h, h1, OPERATION_SET);
2895}
2896
2897static enum test_result test_vb_add_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2898    return test_pending_vb_mutation(h, h1, OPERATION_ADD);
2899}
2900
2901static enum test_result test_vb_cas_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2902    return test_pending_vb_mutation(h, h1, OPERATION_CAS);
2903}
2904
2905static enum test_result test_vb_append_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2906    return test_pending_vb_mutation(h, h1, OPERATION_APPEND);
2907}
2908
2909static enum test_result test_vb_prepend_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2910    return test_pending_vb_mutation(h, h1, OPERATION_PREPEND);
2911}
2912
2913static enum test_result test_vb_set_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2914    return test_replica_vb_mutation(h, h1, OPERATION_SET);
2915}
2916
2917static enum test_result test_vb_replace_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2918    return test_replica_vb_mutation(h, h1, OPERATION_REPLACE);
2919}
2920
2921static enum test_result test_vb_replace_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2922    return test_pending_vb_mutation(h, h1, OPERATION_REPLACE);
2923}
2924
2925static enum test_result test_vb_add_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2926    return test_replica_vb_mutation(h, h1, OPERATION_ADD);
2927}
2928
2929static enum test_result test_vb_cas_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2930    return test_replica_vb_mutation(h, h1, OPERATION_CAS);
2931}
2932
2933static enum test_result test_vb_append_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2934    return test_replica_vb_mutation(h, h1, OPERATION_APPEND);
2935}
2936
2937static enum test_result test_vb_prepend_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2938    return test_replica_vb_mutation(h, h1, OPERATION_PREPEND);
2939}
2940
2941static enum test_result test_stats_seqno(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2942    check(set_vbucket_state(h, h1, 1, vbucket_state_active),
2943          "Failed to set vbucket state.");
2944
2945    int num_keys = 100;
2946    for (int ii = 0; ii < num_keys; ++ii) {
2947        std::stringstream ss;
2948        ss << "key" << ii;
2949        check(store(h, h1, NULL, OPERATION_SET, ss.str().c_str(),
2950                    "value", NULL, 0, 0) == ENGINE_SUCCESS,
2951              "Failed to store an item.");
2952    }
2953
2954    check(get_int_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno") == 100,
2955          "Invalid seqno");
2956    check(get_int_stat(h, h1, "vb_1:high_seqno", "vbucket-seqno") == 0,
2957          "Invalid seqno");
2958    check(get_int_stat(h, h1, "vb_1:high_seqno", "vbucket-seqno 1") == 0,
2959          "Invalid seqno");
2960
2961    uint64_t vb_uuid = get_ull_stat(h, h1, "vb_1:0:id", "failovers");
2962    check(get_ull_stat(h, h1, "vb_1:uuid", "vbucket-seqno 1") == vb_uuid,
2963          "Invalid uuid");
2964    check(vals.size() == 4, "Expected four stats");
2965
2966    // Check invalid vbucket
2967    check(h1->get_stats(h, NULL, "vbucket-seqno 2", 15, add_stats)
2968          == ENGINE_NOT_MY_VBUCKET, "Expected not my vbucket");
2969
2970    // Check bad vbucket parameter (not numeric)
2971    check(h1->get_stats(h, NULL, "vbucket-seqno tt2", 17, add_stats)
2972          == ENGINE_EINVAL, "Expected invalid");
2973
2974    // Check extra spaces at the end
2975    check(h1->get_stats(h, NULL, "vbucket-seqno    ", 17, add_stats)
2976          == ENGINE_EINVAL, "Expected invalid");
2977
2978    return SUCCESS;
2979}
2980
2981static enum test_result test_stats_diskinfo(ENGINE_HANDLE *h,
2982                                            ENGINE_HANDLE_V1 *h1) {
2983    check(set_vbucket_state(h, h1, 1, vbucket_state_active),
2984          "Failed to set vbucket state.");
2985
2986    int num_keys = 100;
2987    for (int ii = 0; ii < num_keys; ++ii) {
2988        std::stringstream ss;
2989        ss << "key" << ii;