1 /* -*- MODE: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3  *     Copyright 2016 Couchbase, Inc
4  *
5  *   Licensed under the Apache License, Version 2.0 (the "License");
6  *   you may not use this file except in compliance with the License.
7  *   You may obtain a copy of the License at
8  *
9  *       http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *   Unless required by applicable law or agreed to in writing, software
12  *   distributed under the License is distributed on an "AS IS" BASIS,
13  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *   See the License for the specific language governing permissions and
15  *   limitations under the License.
16  */
17 
18 /*
19  * Testsuite for 'basic' key-value functionality in ep-engine.
20  */
21 
22 #include "config.h"
23 
24 #include "ep_test_apis.h"
25 #include "ep_testsuite_common.h"
26 
27 #include <platform/cb_malloc.h>
28 #include <platform/cbassert.h>
29 #include <JSON_checker.h>
30 
31 #include <array>
32 #include <memcached/types.h>
33 
34 #define WHITESPACE_DB "whitespace sucks.db"
35 
36 // Types //////////////////////////////////////////////////////////////////////
37 
38 
39 // Helper functions ///////////////////////////////////////////////////////////
40 
epsilon(int val, int target, int ep=5)41 static bool epsilon(int val, int target, int ep=5) {
42     return abs(val - target) < ep;
43 }
44 
45 
46 // Testcases //////////////////////////////////////////////////////////////////
47 
test_alloc_limit(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)48 static enum test_result test_alloc_limit(ENGINE_HANDLE *h,
49                                          ENGINE_HANDLE_V1 *h1) {
50     auto rv = allocate(h,
51                        h1,
52                        NULL,
53                        "key",
54                        20 * 1024 * 1024,
55                        0,
56                        0,
57                        PROTOCOL_BINARY_RAW_BYTES,
58                        0);
59     checkeq(cb::engine_errc::success, rv.first, "Allocated 20MB item");
60 
61     rv = allocate(h,
62                   h1,
63                   NULL,
64                   "key",
65                   (20 * 1024 * 1024) + 1,
66                   0,
67                   0,
68                   PROTOCOL_BINARY_RAW_BYTES,
69                   0);
70     checkeq(cb::engine_errc::too_big, rv.first, "Object too big");
71 
72     return SUCCESS;
73 }
74 
test_memory_tracking(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)75 static enum test_result test_memory_tracking(ENGINE_HANDLE *h,
76                                              ENGINE_HANDLE_V1 *h1) {
77     // Need memory tracker to be able to check our memory usage.
78     std::string tracker = get_str_stat(h, h1, "ep_mem_tracker_enabled");
79     if (tracker == "true") {
80         return SUCCESS;
81     } else {
82         std::cerr << "Memory tracker not enabled ...";
83         return SKIPPED;
84     }
85 }
86 
test_max_size_and_water_marks_settings( ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)87 static enum test_result test_max_size_and_water_marks_settings(
88                                         ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
89     checkeq(1000, get_int_stat(h, h1, "ep_max_size"), "Incorrect initial size.");
90     check(epsilon(get_int_stat(h, h1, "ep_mem_low_wat"), 750),
91           "Incorrect initial low wat.");
92     check(epsilon(get_int_stat(h, h1, "ep_mem_high_wat"), 850),
93           "Incorrect initial high wat.");
94     check((get_float_stat(h, h1, "ep_mem_low_wat_percent") == (float)0.75),
95           "Incorrect initial low wat. percent");
96     check((get_float_stat(h, h1, "ep_mem_high_wat_percent") == (float)0.85),
97           "Incorrect initial high wat. percent");
98 
99     set_param(h, h1, protocol_binary_engine_param_flush, "max_size", "1000000");
100 
101     checkeq(1000000, get_int_stat(h, h1, "ep_max_size"),
102             "Incorrect new size.");
103     check(epsilon(get_int_stat(h, h1, "ep_mem_low_wat"), 750000),
104           "Incorrect larger low wat.");
105     check(epsilon(get_int_stat(h, h1, "ep_mem_high_wat"), 850000),
106           "Incorrect larger high wat.");
107     check((get_float_stat(h, h1, "ep_mem_low_wat_percent") == (float)0.75),
108           "Incorrect larger low wat. percent");
109     check((get_float_stat(h, h1, "ep_mem_high_wat_percent") == (float)0.85),
110           "Incorrect larger high wat. percent");
111 
112     set_param(h, h1, protocol_binary_engine_param_flush, "mem_low_wat", "700000");
113     set_param(h, h1, protocol_binary_engine_param_flush, "mem_high_wat", "800000");
114 
115     checkeq(700000, get_int_stat(h, h1, "ep_mem_low_wat"),
116             "Incorrect even larger low wat.");
117     checkeq(800000, get_int_stat(h, h1, "ep_mem_high_wat"),
118             "Incorrect even larger high wat.");
119     check((get_float_stat(h, h1, "ep_mem_low_wat_percent") == (float)0.7),
120           "Incorrect even larger low wat. percent");
121     check((get_float_stat(h, h1, "ep_mem_high_wat_percent") == (float)0.8),
122           "Incorrect even larger high wat. percent");
123 
124     set_param(h, h1, protocol_binary_engine_param_flush, "max_size", "100");
125 
126     checkeq(100, get_int_stat(h, h1, "ep_max_size"),
127             "Incorrect smaller size.");
128     check(epsilon(get_int_stat(h, h1, "ep_mem_low_wat"), 70),
129           "Incorrect smaller low wat.");
130     check(epsilon(get_int_stat(h, h1, "ep_mem_high_wat"), 80),
131           "Incorrect smaller high wat.");
132     check((get_float_stat(h, h1, "ep_mem_low_wat_percent") == (float)0.7),
133           "Incorrect smaller low wat. percent");
134     check((get_float_stat(h, h1, "ep_mem_high_wat_percent") == (float)0.8),
135           "Incorrect smaller high wat. percent");
136 
137     set_param(h, h1, protocol_binary_engine_param_flush, "mem_low_wat", "50");
138     set_param(h, h1, protocol_binary_engine_param_flush, "mem_high_wat", "70");
139 
140     checkeq(50, get_int_stat(h, h1, "ep_mem_low_wat"),
141             "Incorrect even smaller low wat.");
142     checkeq(70, get_int_stat(h, h1, "ep_mem_high_wat"),
143             "Incorrect even smaller high wat.");
144     check((get_float_stat(h, h1, "ep_mem_low_wat_percent") == (float)0.5),
145           "Incorrect even smaller low wat. percent");
146     check((get_float_stat(h, h1, "ep_mem_high_wat_percent") == (float)0.7),
147           "Incorrect even smaller high wat. percent");
148 
149     testHarness.reload_engine(&h, &h1,
150                               testHarness.engine_path,
151                               testHarness.get_current_testcase()->cfg,
152                               true, true);
153     wait_for_warmup_complete(h, h1);
154 
155     checkeq(1000, get_int_stat(h, h1, "ep_max_size"),
156             "Incorrect initial size.");
157     check(epsilon(get_int_stat(h, h1, "ep_mem_low_wat"), 750),
158           "Incorrect intial low wat.");
159     check(epsilon(get_int_stat(h, h1, "ep_mem_high_wat"), 850),
160           "Incorrect initial high wat.");
161     check((get_float_stat(h, h1, "ep_mem_low_wat_percent") == (float)0.75),
162           "Incorrect initial low wat. percent");
163     check((get_float_stat(h, h1, "ep_mem_high_wat_percent") == (float)0.85),
164           "Incorrect initial high wat. percent");
165 
166     return SUCCESS;
167 }
168 
test_whitespace_db(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)169 static enum test_result test_whitespace_db(ENGINE_HANDLE *h,
170                                            ENGINE_HANDLE_V1 *h1) {
171     vals.clear();
172 
173     checkeq(ENGINE_SUCCESS,
174             get_stats(h, {}, add_stats),
175             "Failed to get stats.");
176 
177     std::string dbname;
178     std::string policy;
179     policy = isPersistentBucket(h, h1)
180                      ? vals.find("ep_item_eviction_policy")->second
181                      : "ephemeral";
182     dbname.assign(policy + std::string(WHITESPACE_DB));
183 
184     std::string oldparam("dbname=" + vals["ep_dbname"]);
185     std::string newparam("dbname=" + dbname);
186     std::string config = testHarness.get_current_testcase()->cfg;
187     std::string::size_type found = config.find(oldparam);
188     if (found != config.npos) {
189         config.replace(found, oldparam.size(), newparam);
190     }
191     testHarness.reload_engine(&h, &h1,
192                               testHarness.engine_path,
193                               config.c_str(),
194                               true, false);
195     wait_for_warmup_complete(h, h1);
196 
197     vals.clear();
198     checkeq(ENGINE_SUCCESS,
199             get_stats(h, {}, add_stats),
200             "Failed to get stats.");
201 
202     if (vals["ep_dbname"] != dbname) {
203         std::cerr << "Expected dbname = '" << dbname << "'"
204                   << ", got '" << vals["ep_dbname"] << "'" << std::endl;
205         return FAIL;
206     }
207 
208     check(access(dbname.c_str(), F_OK) != -1, "I expected the whitespace db to exist");
209     return SUCCESS;
210 }
211 
test_get_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)212 static enum test_result test_get_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
213     checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "k"), "Expected miss.");
214     return SUCCESS;
215 }
216 
test_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)217 static enum test_result test_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
218     item_info info;
219     uint64_t vb_uuid = 0, high_seqno = 0;
220     const int num_sets = 5, num_keys = 4;
221 
222     std::string key_arr[num_keys] = { "dummy_key",
223                                       "checkpoint_start",
224                                       "checkpoint_end",
225                                       "key" };
226 
227 
228     for (int k = 0; k < num_keys; k++) {
229         for (int j = 0; j < num_sets; j++) {
230             memset(&info, 0, sizeof(info));
231             vb_uuid = get_ull_stat(h, h1, "vb_0:0:id", "failovers");
232             high_seqno = get_ull_stat(h, h1, "vb_0:high_seqno",
233                                       "vbucket-seqno");
234 
235             std::string err_str_store("Error setting " + key_arr[k]);
236             checkeq(ENGINE_SUCCESS,
237                     store(h,
238                           h1,
239                           NULL,
240                           OPERATION_SET,
241                           key_arr[k].c_str(),
242                           "somevalue"),
243                     err_str_store.c_str());
244 
245             std::string err_str_get_item_info("Error getting " + key_arr[k]);
246             checkeq(true, get_item_info(h, h1, &info, key_arr[k].c_str()),
247                   err_str_get_item_info.c_str());
248 
249             std::string err_str_vb_uuid("Expected valid vbucket uuid for " +
250                                         key_arr[k]);
251             checkeq(vb_uuid, info.vbucket_uuid, err_str_vb_uuid.c_str());
252 
253             std::string err_str_seqno("Expected valid sequence number for " +
254                                         key_arr[k]);
255             checkeq(high_seqno + 1, info.seqno, err_str_seqno.c_str());
256         }
257     }
258 
259     if (isPersistentBucket(h, h1)) {
260         wait_for_flusher_to_settle(h, h1);
261 
262         std::stringstream error1, error2;
263         error1 << "Expected ep_total_persisted >= num_keys (" << num_keys << ")";
264         error2 << "Expected ep_total_persisted <= num_sets*num_keys ("
265                << num_sets*num_keys << ")";
266 
267         // The flusher could of ran > 1 times. We can only assert
268         // that we persisted between num_keys and upto num_keys*num_sets
269         check(get_int_stat(h, h1, "ep_total_persisted") >= num_keys,
270               error1.str().c_str());
271         check(get_int_stat(h, h1, "ep_total_persisted") <= num_sets*num_keys,
272               error2.str().c_str());
273     }
274     return SUCCESS;
275 }
276 
277 extern "C" {
conc_del_set_thread(void *arg)278     static void conc_del_set_thread(void *arg) {
279         struct handle_pair *hp = static_cast<handle_pair *>(arg);
280 
281         for (int i = 0; i < 5000; ++i) {
282             store(hp->h, hp->h1, NULL, OPERATION_ADD, "key", "somevalue");
283             checkeq(ENGINE_SUCCESS,
284                     store(hp->h,
285                           hp->h1,
286                           NULL,
287                           OPERATION_SET,
288                           "key",
289                           "somevalue"),
290                     "Error setting.");
291             // Ignoring the result here -- we're racing.
292             del(hp->h, hp->h1, "key", 0, 0);
293         }
294     }
295 }
296 
test_conc_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)297 static enum test_result test_conc_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
298 
299     const int n_threads = 8;
300     cb_thread_t threads[n_threads];
301     struct handle_pair hp = {h, h1};
302 
303     wait_for_persisted_value(h, h1, "key", "value1");
304 
305     for (int i = 0; i < n_threads; i++) {
306         int r = cb_create_thread(&threads[i], conc_del_set_thread, &hp, 0);
307         cb_assert(r == 0);
308     }
309 
310     for (int i = 0; i < n_threads; i++) {
311         int r = cb_join_thread(threads[i]);
312         cb_assert(r == 0);
313     }
314 
315     if (isWarmupEnabled(h, h1)) {
316         wait_for_flusher_to_settle(h, h1);
317 
318         testHarness.reload_engine(&h, &h1,
319                                   testHarness.engine_path,
320                                   testHarness.get_current_testcase()->cfg,
321                                   true, false);
322         wait_for_warmup_complete(h, h1);
323 
324         cb_assert(0 == get_int_stat(h, h1, "ep_warmup_dups"));
325     }
326 
327     return SUCCESS;
328 }
329 
330 struct multi_set_args {
331     ENGINE_HANDLE *h;
332     ENGINE_HANDLE_V1 *h1;
333     std::string prefix;
334     int count;
335 };
336 
337 extern "C" {
multi_set_thread(void *arg)338     static void multi_set_thread(void *arg) {
339         struct multi_set_args *msa = static_cast<multi_set_args *>(arg);
340 
341         for (int i = 0; i < msa->count; i++) {
342             std::stringstream s;
343             s << msa->prefix << i;
344             std::string key(s.str());
345             checkeq(ENGINE_SUCCESS,
346                     store(msa->h,
347                           msa->h1,
348                           NULL,
349                           OPERATION_SET,
350                           key.c_str(),
351                           "somevalue"),
352                     "Set failure!");
353         }
354     }
355 }
356 
test_multi_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)357 static enum test_result test_multi_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
358 
359     cb_thread_t thread1, thread2;
360     struct multi_set_args msa1, msa2;
361     msa1.h = h;
362     msa1.h1 = h1;
363     msa1.prefix = "ONE_";
364     msa1.count = 50000;
365     cb_assert(cb_create_thread(&thread1, multi_set_thread, &msa1, 0) == 0);
366 
367     msa2.h = h;
368     msa2.h1 = h1;
369     msa2.prefix = "TWO_";
370     msa2.count = 50000;
371     cb_assert(cb_create_thread(&thread2, multi_set_thread, &msa2, 0) == 0);
372 
373     cb_assert(cb_join_thread(thread1) == 0);
374     cb_assert(cb_join_thread(thread2) == 0);
375 
376     wait_for_flusher_to_settle(h, h1);
377 
378     checkeq(100000, get_int_stat(h, h1, "curr_items"),
379             "Mismatch in number of items inserted");
380     checkeq(100000, get_int_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno"),
381             "Unexpected high sequence number");
382 
383     return SUCCESS;
384 }
385 
test_set_get_hit(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)386 static enum test_result test_set_get_hit(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
387     checkeq(ENGINE_SUCCESS,
388             store(h, h1, NULL, OPERATION_SET, "key", "somevalue"),
389             "store failure");
390     check_key_value(h, h1, "key", "somevalue", 9);
391     return SUCCESS;
392 }
393 
test_getl_delete_with_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)394 static enum test_result test_getl_delete_with_cas(ENGINE_HANDLE *h,
395                                                   ENGINE_HANDLE_V1 *h1) {
396     checkeq(ENGINE_SUCCESS,
397             store(h, h1, NULL, OPERATION_SET, "key", "value"),
398             "Failed to set key");
399 
400     auto ret = getl(h, h1, nullptr, "key", 0, 15);
401     checkeq(cb::engine_errc::success,
402             ret.first,
403             "Expected getl to succeed on key");
404     item_info info;
405     check(h1->get_item_info(h, ret.second.get(), &info),
406           "Failed to get item info");
407 
408     checkeq(ENGINE_SUCCESS, del(h, h1, "key", info.cas, 0), "Expected SUCCESS");
409 
410     return SUCCESS;
411 }
412 
test_getl_delete_with_bad_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)413 static enum test_result test_getl_delete_with_bad_cas(ENGINE_HANDLE *h,
414                                                       ENGINE_HANDLE_V1 *h1) {
415     checkeq(ENGINE_SUCCESS,
416             store(h, h1, NULL, OPERATION_SET, "key", "value"),
417             "Failed to set key");
418 
419     uint64_t cas = last_cas;
420     checkeq(cb::engine_errc::success,
421             getl(h, h1, nullptr, "key", 0, 15).first,
422             "Expected getl to succeed on key");
423 
424     checkeq(ENGINE_LOCKED_TMPFAIL, del(h, h1, "key", cas, 0), "Expected TMPFAIL");
425 
426     return SUCCESS;
427 }
428 
test_getl_set_del_with_meta(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)429 static enum test_result test_getl_set_del_with_meta(ENGINE_HANDLE *h,
430                                                     ENGINE_HANDLE_V1 *h1) {
431     const char *key = "key";
432     const char *val = "value";
433     const char *newval = "newvalue";
434     checkeq(ENGINE_SUCCESS,
435             store(h, h1, NULL, OPERATION_SET, key, val),
436             "Failed to set key");
437 
438     checkeq(cb::engine_errc::success,
439             getl(h, h1, nullptr, key, 0, 15).first,
440             "Expected getl to succeed on key");
441 
442     cb::EngineErrorMetadataPair errorMetaPair;
443     check(get_meta(h, h1, key, errorMetaPair), "Expected to get meta");
444 
445     //init some random metadata
446     ItemMetaData itm_meta(0xdeadbeef, 10, 0xdeadbeef, time(NULL) + 300);
447 
448     //do a set with meta
449     set_with_meta(h,
450                   h1,
451                   key,
452                   strlen(key),
453                   newval,
454                   strlen(newval),
455                   0,
456                   &itm_meta,
457                   errorMetaPair.second.cas);
458     checkeq(PROTOCOL_BINARY_RESPONSE_LOCKED, last_status.load(),
459           "Expected item to be locked");
460 
461     //do a del with meta
462     del_with_meta(h, h1, key, strlen(key), 0, &itm_meta, last_cas);
463     checkeq(PROTOCOL_BINARY_RESPONSE_LOCKED, last_status.load(),
464           "Expected item to be locked");
465     return SUCCESS;
466 }
467 
test_getl(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)468 static enum test_result test_getl(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
469     const char *key = "k1";
470     uint16_t vbucketId = 0;
471     uint32_t expiration = 25;
472 
473     const void* cookie = testHarness.create_cookie();
474 
475     checkeq(cb::engine_errc::no_such_key,
476             getl(h, h1, cookie, key, vbucketId, expiration).first,
477             "expected the key to be missing...");
478 
479     checkeq(ENGINE_SUCCESS,
480             store(h,
481                   h1,
482                   cookie,
483                   OPERATION_SET,
484                   key,
485                   "{\"lock\":\"data\"}",
486                   nullptr,
487                   0,
488                   vbucketId,
489                   3600,
490                   PROTOCOL_BINARY_DATATYPE_JSON),
491             "Failed to store an item.");
492 
493     /* retry getl, should succeed */
494     auto ret = getl(h, h1, cookie, key, vbucketId, expiration);
495     checkeq(cb::engine_errc::success,
496             ret.first,
497             "Expected to be able to getl on first try");
498 
499     item_info info;
500     check(h1->get_item_info(h, ret.second.get(), &info),
501           "Failed to get item info");
502 
503     checkeq(std::string{"{\"lock\":\"data\"}"},
504             std::string((const char*)info.value[0].iov_base,
505                         info.value[0].iov_len),
506             "Body was malformed.");
507     checkeq(static_cast<uint8_t>(PROTOCOL_BINARY_DATATYPE_JSON),
508             info.datatype,
509             "Expected datatype to be JSON");
510 
511     /* wait 16 seconds */
512     testHarness.time_travel(16);
513 
514     /* lock's taken so this should fail */
515     checkeq(cb::engine_errc::locked_tmpfail,
516             getl(h, h1, cookie, key, vbucketId, expiration).first,
517             "Expected to fail getl on second try");
518 
519     checkne(ENGINE_SUCCESS,
520             store(h,
521                   h1,
522                   cookie,
523                   OPERATION_SET,
524                   key,
525                   "lockdata2",
526                   nullptr,
527                   0,
528                   vbucketId),
529             "Should have failed to store an item.");
530 
531     /* wait another 10 seconds */
532     testHarness.time_travel(10);
533 
534     /* retry set, should succeed */
535     checkeq(ENGINE_SUCCESS,
536             store(h,
537                   h1,
538                   cookie,
539                   OPERATION_SET,
540                   key,
541                   "lockdata",
542                   nullptr,
543                   0,
544                   vbucketId),
545             "Failed to store an item.");
546 
547     /* point to wrong vbucket, to test NOT_MY_VB response */
548     checkeq(cb::engine_errc::not_my_vbucket,
549             getl(h, h1, cookie, key, 10, expiration).first,
550             "Should have received not my vbucket response");
551 
552     /* acquire lock, should succeed */
553     ret = getl(h, h1, cookie, key, vbucketId, expiration);
554     checkeq(cb::engine_errc::success,
555             ret.first,
556             "Acquire lock should have succeeded");
557     check(h1->get_item_info(h, ret.second.get(), &info),
558           "Failed to get item info");
559     checkeq(static_cast<uint8_t>(PROTOCOL_BINARY_RAW_BYTES), info.datatype,
560             "Expected datatype to be RAW BYTES");
561 
562     /* try an delete operation which should fail */
563     uint64_t cas = 0;
564 
565     checkeq(ENGINE_LOCKED_TMPFAIL, del(h, h1, key, 0, 0), "Delete failed");
566 
567 
568     /* bug MB 2699 append after getl should fail with ENGINE_TMPFAIL */
569 
570     testHarness.time_travel(26);
571 
572     char binaryData1[] = "abcdefg\0gfedcba";
573 
574     checkeq(cb::engine_errc::success,
575             storeCasVb11(h,
576                          h1,
577                          cookie,
578                          OPERATION_SET,
579                          key,
580                          binaryData1,
581                          sizeof(binaryData1) - 1,
582                          82758,
583                          0,
584                          0)
585                     .first,
586             "Failed set.");
587 
588     /* acquire lock, should succeed */
589     checkeq(cb::engine_errc::success,
590             getl(h, h1, cookie, key, vbucketId, expiration).first,
591             "Acquire lock should have succeeded");
592 
593     /* bug MB 3252 & MB 3354.
594      * 1. Set a key with an expiry value.
595      * 2. Take a lock on the item before it expires
596      * 3. Wait for the item to expire
597      * 4. Perform a CAS operation, should fail
598      * 5. Perform a set operation, should succeed
599      */
600     const char *ekey = "test_expiry";
601     const char *edata = "some test data here.";
602 
603     ret = allocate(h,
604                    h1,
605                    cookie,
606                    ekey,
607                    strlen(edata),
608                    0,
609                    2,
610                    PROTOCOL_BINARY_RAW_BYTES,
611                    0);
612     checkeq(cb::engine_errc::success, ret.first, "Allocation Failed");
613 
614     check(h1->get_item_info(h, ret.second.get(), &info),
615           "Failed to get item info");
616 
617     memcpy(info.value[0].iov_base, edata, strlen(edata));
618 
619     checkeq(ENGINE_SUCCESS,
620             h1->store(h,
621                       cookie,
622                       ret.second.get(),
623                       cas,
624                       OPERATION_SET,
625                       DocumentState::Alive),
626             "Failed to Store item");
627     check_key_value(h, h1, ekey, edata, strlen(edata));
628 
629     testHarness.time_travel(3);
630     cas = last_cas;
631 
632     /* cas should fail */
633     ret = storeCasVb11(h,
634                        h1,
635                        cookie,
636                        OPERATION_CAS,
637                        ekey,
638                        binaryData1,
639                        sizeof(binaryData1) - 1,
640                        82758,
641                        cas,
642                        0);
643     checkne(cb::engine_errc::success, ret.first, "CAS succeeded.");
644 
645     /* but a simple store should succeed */
646     checkeq(ENGINE_SUCCESS,
647             store(h,
648                   h1,
649                   cookie,
650                   OPERATION_SET,
651                   ekey,
652                   edata,
653                   nullptr,
654                   0,
655                   vbucketId),
656             "Failed to store an item.");
657 
658     testHarness.destroy_cookie(cookie);
659     return SUCCESS;
660 }
661 
test_unl(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)662 static enum test_result test_unl(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
663 
664     const char *key = "k2";
665     uint16_t vbucketId = 0;
666 
667     checkeq(ENGINE_SUCCESS,
668             get_stats(h, {}, add_stats),
669             "Failed to get stats.");
670 
671     std::string eviction_policy;
672     auto itr = vals.find("ep_item_eviction_policy");
673     if (itr != vals.end()) {
674         eviction_policy = itr->second;
675     } else {
676         eviction_policy = "value_only";
677     }
678 
679     if (eviction_policy == "full_eviction") {
680         checkeq(ENGINE_TMPFAIL,
681                 unl(h, h1, nullptr, key, vbucketId),
682                 "expected a TMPFAIL");
683     } else {
684         checkeq(ENGINE_KEY_ENOENT,
685                 unl(h, h1, nullptr, key, vbucketId),
686                 "expected the key to be missing...");
687     }
688 
689     checkeq(ENGINE_SUCCESS,
690             store(h,
691                   h1,
692                   NULL,
693                   OPERATION_SET,
694                   key,
695                   "lockdata",
696                   nullptr,
697                   0,
698                   vbucketId),
699             "Failed to store an item.");
700 
701     /* getl, should succeed */
702     auto ret = getl(h, h1, nullptr, key, vbucketId, 0);
703     checkeq(cb::engine_errc::success,
704             ret.first,
705             "Expected to be able to getl on first try");
706     item_info info;
707     checkeq(true,
708             h1->get_item_info(h, ret.second.get(), &info),
709             "failed to get item info");
710     uint64_t cas = info.cas;
711 
712     /* lock's taken unlocking with a random cas value should fail */
713     checkeq(ENGINE_LOCKED_TMPFAIL,
714             unl(h, h1, nullptr, key, vbucketId),
715             "Expected to fail getl on second try");
716 
717     checkeq(ENGINE_SUCCESS,
718             unl(h, h1, nullptr, key, vbucketId, cas),
719             "Expected to succed unl with correct cas");
720 
721     /* acquire lock, should succeed */
722     checkeq(cb::engine_errc::success,
723             getl(h, h1, nullptr, key, vbucketId, 0).first,
724             "Lock should work after unlock");
725 
726     /* wait 16 seconds */
727     testHarness.time_travel(16);
728 
729     /* lock has expired, unl should fail */
730     checkeq(ENGINE_TMPFAIL,
731             unl(h, h1, nullptr, key, vbucketId, last_cas),
732             "Expected to fail unl on lock timeout");
733 
734     return SUCCESS;
735 }
736 
test_unl_nmvb(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)737 static enum test_result test_unl_nmvb(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
738 
739     const char *key = "k2";
740     uint16_t vbucketId = 10;
741 
742     checkeq(ENGINE_NOT_MY_VBUCKET,
743             unl(h, h1, nullptr, key, vbucketId),
744           "expected NOT_MY_VBUCKET to unlocking a key in a vbucket we don't own");
745 
746     return SUCCESS;
747 }
748 
test_set_get_hit_bin(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)749 static enum test_result test_set_get_hit_bin(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
750     char binaryData[] = "abcdefg\0gfedcba";
751     cb_assert(sizeof(binaryData) != strlen(binaryData));
752 
753     checkeq(cb::engine_errc::success,
754             storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
755                          binaryData, sizeof(binaryData), 82758, 0, 0).first,
756             "Failed to set.");
757     check_key_value(h, h1, "key", binaryData, sizeof(binaryData));
758     return SUCCESS;
759 }
760 
test_set_with_cas_non_existent(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)761 static enum test_result test_set_with_cas_non_existent(ENGINE_HANDLE *h,
762                                                        ENGINE_HANDLE_V1 *h1) {
763     const char *key = "test_expiry_flush";
764     const auto* cookie = testHarness.create_cookie();
765     auto ret = allocate(
766             h, h1, cookie, key, 10, 0, 0, PROTOCOL_BINARY_RAW_BYTES, 0);
767     checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
768 
769     Item* it = reinterpret_cast<Item*>(ret.second.get());
770     it->setCas(1234);
771 
772     uint64_t cas = 0;
773     checkeq(ENGINE_KEY_ENOENT,
774             h1->store(h,
775                       cookie,
776                       ret.second.get(),
777                       cas,
778                       OPERATION_SET,
779                       DocumentState::Alive),
780             "Expected not found");
781 
782     testHarness.destroy_cookie(cookie);
783     return SUCCESS;
784 }
785 
test_set_change_flags(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)786 static enum test_result test_set_change_flags(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
787     checkeq(ENGINE_SUCCESS,
788             store(h, h1, NULL, OPERATION_SET, "key", "somevalue"),
789             "Failed to set.");
790 
791     item_info info;
792     uint32_t flags = 828258;
793     check(get_item_info(h, h1, &info, "key"), "Failed to get value.");
794     cb_assert(info.flags != flags);
795 
796     checkeq(cb::engine_errc::success,
797             storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
798                          "newvalue", strlen("newvalue"), flags, 0, 0).first,
799             "Failed to set again.");
800 
801     check(get_item_info(h, h1, &info, "key"), "Failed to get value.");
802 
803     return info.flags == flags ? SUCCESS : FAIL;
804 }
805 
test_add(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)806 static enum test_result test_add(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
807     item_info info;
808     uint64_t vb_uuid = 0;
809     uint64_t high_seqno = 0;
810 
811     memset(&info, 0, sizeof(info));
812 
813     vb_uuid = get_ull_stat(h, h1, "vb_0:0:id", "failovers");
814     high_seqno = get_ull_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno");
815 
816     checkeq(ENGINE_SUCCESS,
817             store(h, h1, NULL, OPERATION_ADD, "key", "somevalue"),
818             "Failed to add value.");
819 
820     check(get_item_info(h, h1, &info, "key"), "Error getting item info");
821     checkeq(vb_uuid, info.vbucket_uuid, "Expected valid vbucket uuid");
822     checkeq(high_seqno + 1, info.seqno, "Expected valid sequence number");
823 
824     checkeq(ENGINE_NOT_STORED,
825             store(h, h1, NULL, OPERATION_ADD, "key", "somevalue"),
826             "Failed to fail to re-add value.");
827 
828     // This aborts on failure.
829     check_key_value(h, h1, "key", "somevalue", 9);
830 
831     // Expiration above was an hour, so let's go to The Future
832     testHarness.time_travel(3800);
833 
834     checkeq(ENGINE_SUCCESS,
835             store(h, h1, NULL, OPERATION_ADD, "key", "newvalue"),
836             "Failed to add value again.");
837 
838     check_key_value(h, h1, "key", "newvalue", 8);
839     return SUCCESS;
840 }
841 
test_add_add_with_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)842 static enum test_result test_add_add_with_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
843     item *i = NULL;
844     checkeq(ENGINE_SUCCESS,
845             store(h, h1, NULL, OPERATION_ADD, "key", "somevalue", &i),
846             "Failed set.");
847     check_key_value(h, h1, "key", "somevalue", 9);
848     item_info info;
849     check(h1->get_item_info(h, i, &info), "Should be able to get info");
850 
851     checkeq(ENGINE_KEY_EEXISTS,
852             store(h,
853                   h1,
854                   NULL,
855                   OPERATION_ADD,
856                   "key",
857                   "somevalue",
858                   nullptr,
859                   info.cas),
860             "Should not be able to add the key two times");
861 
862     h1->release(h, i);
863     return SUCCESS;
864 }
865 
test_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)866 static enum test_result test_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
867     checkeq(ENGINE_SUCCESS,
868             store(h, h1, NULL, OPERATION_SET, "key", "somevalue"),
869             "Failed to do initial set.");
870     check(store(h, h1, NULL, OPERATION_CAS, "key", "failcas") != ENGINE_SUCCESS,
871           "Failed to fail initial CAS.");
872     check_key_value(h, h1, "key", "somevalue", 9);
873 
874     auto ret = get(h, h1, NULL, "key", 0);
875     checkeq(cb::engine_errc::success, ret.first, "Failed to get value.");
876 
877     item_info info;
878     check(h1->get_item_info(h, ret.second.get(), &info),
879           "Failed to get item info.");
880 
881     checkeq(ENGINE_SUCCESS,
882             store(h,
883                   h1,
884                   NULL,
885                   OPERATION_CAS,
886                   "key",
887                   "winCas",
888                   nullptr,
889                   info.cas),
890             "Failed to store CAS");
891     check_key_value(h, h1, "key", "winCas", 6);
892 
893     uint64_t cval = 99999;
894     checkeq(ENGINE_KEY_ENOENT,
895             store(h,
896                   h1,
897                   NULL,
898                   OPERATION_CAS,
899                   "non-existing",
900                   "winCas",
901                   nullptr,
902                   cval),
903             "CAS for non-existing key returned the wrong error code");
904     return SUCCESS;
905 }
906 
test_replace(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)907 static enum test_result test_replace(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
908     item_info info;
909     uint64_t vb_uuid = 0;
910     uint64_t high_seqno = 0;
911 
912     memset(&info, 0, sizeof(info));
913 
914     check(store(h, h1, NULL, OPERATION_REPLACE, "key", "somevalue") !=
915                   ENGINE_SUCCESS,
916           "Failed to fail to replace non-existing value.");
917 
918     checkeq(ENGINE_SUCCESS,
919             store(h, h1, NULL, OPERATION_SET, "key", "somevalue"),
920             "Failed to set value.");
921 
922     vb_uuid = get_ull_stat(h, h1, "vb_0:0:id", "failovers");
923     high_seqno = get_ull_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno");
924 
925     checkeq(ENGINE_SUCCESS,
926             store(h, h1, NULL, OPERATION_REPLACE, "key", "somevalue"),
927             "Failed to replace existing value.");
928 
929     check(get_item_info(h, h1, &info, "key"), "Error getting item info");
930 
931     checkeq(vb_uuid, info.vbucket_uuid, "Expected valid vbucket uuid");
932     checkeq(high_seqno + 1, info.seqno, "Expected valid sequence number");
933 
934     check_key_value(h, h1, "key", "somevalue", 9);
935     return SUCCESS;
936 }
937 
test_touch(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)938 static enum test_result test_touch(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
939     // Try to touch an unknown item...
940     checkeq(ENGINE_KEY_ENOENT, touch(h, h1, "mykey", 0, 0), "Testing unknown key");
941 
942     // illegal vbucket
943     checkeq(ENGINE_NOT_MY_VBUCKET, touch(h, h1, "mykey", 5, 0), "Testing illegal vbucket");
944 
945     // Store the item!
946     checkeq(ENGINE_SUCCESS,
947             store(h, h1, NULL, OPERATION_SET, "mykey", "somevalue"),
948             "Failed set.");
949 
950     check_key_value(h, h1, "mykey", "somevalue", strlen("somevalue"));
951 
952     cb::EngineErrorMetadataPair errorMetaPair;
953 
954     check(get_meta(h, h1, "mykey", errorMetaPair), "Get meta failed");
955 
956     item_info currMeta = errorMetaPair.second;
957 
958     checkeq(ENGINE_SUCCESS,
959             touch(h, h1, "mykey", 0, uint32_t(time(NULL) + 10)),
960             "touch mykey");
961     check(last_cas != currMeta.cas,
962           "touch should have returned an updated CAS");
963 
964     check(get_meta(h, h1, "mykey", errorMetaPair), "Get meta failed");
965 
966     check(errorMetaPair.second.cas != currMeta.cas,
967           "touch should have updated the CAS");
968     check(errorMetaPair.second.exptime != currMeta.exptime,
969           "touch should have updated the expiry time");
970     check(errorMetaPair.second.seqno == currMeta.seqno + 1,
971           "touch should have incremented rev seqno");
972 
973     // time-travel 9 secs..
974     testHarness.time_travel(9);
975 
976     // The item should still exist
977     check_key_value(h, h1, "mykey", "somevalue", 9);
978 
979     // time-travel 2 secs..
980     testHarness.time_travel(2);
981 
982     // The item should have expired now...
983     checkeq(cb::engine_errc::no_such_key,
984             get(h, h1, NULL, "mykey", 0).first,
985             "Item should be gone");
986     return SUCCESS;
987 }
988 
test_touch_mb7342(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)989 static enum test_result test_touch_mb7342(ENGINE_HANDLE *h,
990                                           ENGINE_HANDLE_V1 *h1) {
991     const char *key = "MB-7342";
992     // Store the item!
993     checkeq(ENGINE_SUCCESS,
994             store(h, h1, NULL, OPERATION_SET, key, "v"),
995             "Failed set.");
996 
997     checkeq(ENGINE_SUCCESS, touch(h, h1, key, 0, 0), "touch key");
998 
999     check_key_value(h, h1, key, "v", 1);
1000 
1001     // Travel a loong time to see if the object is still there (the default
1002     // store sets an exp time of 3600
1003     testHarness.time_travel(3700);
1004 
1005     check_key_value(h, h1, key, "v", 1);
1006 
1007     return SUCCESS;
1008 }
1009 
test_touch_mb10277(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1010 static enum test_result test_touch_mb10277(ENGINE_HANDLE *h,
1011                                             ENGINE_HANDLE_V1 *h1) {
1012     const char *key = "MB-10277";
1013     // Store the item!
1014     checkeq(ENGINE_SUCCESS,
1015             store(h, h1, NULL, OPERATION_SET, key, "v"),
1016             "Failed set.");
1017     wait_for_flusher_to_settle(h, h1);
1018     evict_key(h, h1, key, 0, "Ejected.");
1019 
1020     checkeq(ENGINE_SUCCESS,
1021             touch(h, h1, key, 0, 3600), // A new expiration time remains in the same.
1022             "touch key");
1023 
1024     return SUCCESS;
1025 }
1026 
test_gat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1027 static enum test_result test_gat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1028     // Try to gat an unknown item...
1029     auto ret = gat(h, h1, "mykey", 0, 10);
1030     checkeq(ENGINE_KEY_ENOENT, ENGINE_ERROR_CODE(ret.first),
1031             "Testing unknown key");
1032 
1033     // illegal vbucket
1034     ret = gat(h, h1, "mykey", 5, 10);
1035     checkeq(ENGINE_NOT_MY_VBUCKET,
1036             ENGINE_ERROR_CODE(ret.first), "Testing illegal vbucket");
1037 
1038     // Store the item!
1039     checkeq(ENGINE_SUCCESS,
1040             store(h,
1041                   h1,
1042                   NULL,
1043                   OPERATION_SET,
1044                   "mykey",
1045                   "{\"some\":\"value\"}",
1046                   nullptr,
1047                   0,
1048                   0,
1049                   3600,
1050                   PROTOCOL_BINARY_DATATYPE_JSON),
1051             "Failed set.");
1052 
1053     check_key_value(h, h1, "mykey", "{\"some\":\"value\"}",
1054             strlen("{\"some\":\"value\"}"));
1055 
1056     ret = gat(h, h1, "mykey", 0, 10);
1057     checkeq(ENGINE_SUCCESS,
1058             ENGINE_ERROR_CODE(ret.first), "gat mykey");
1059 
1060     item_info info;
1061     check(h1->get_item_info(h, ret.second.get(), &info),
1062           "Getting item info failed");
1063 
1064     checkeq(static_cast<uint8_t>(PROTOCOL_BINARY_DATATYPE_JSON),
1065             info.datatype, "Expected datatype to be JSON");
1066 
1067     std::string body{static_cast<char*>(info.value[0].iov_base),
1068     info.value[0].iov_len};
1069     check(body.compare(0, sizeof("{\"some\":\"value\"}"),
1070                        "{\"some\":\"value\"}") == 0,
1071           "Invalid data returned");
1072 
1073     // time-travel 9 secs..
1074     testHarness.time_travel(9);
1075 
1076     // The item should still exist
1077     check_key_value(h, h1, "mykey", "{\"some\":\"value\"}",
1078                     strlen("{\"some\":\"value\"}"));
1079 
1080     // time-travel 2 secs..
1081     testHarness.time_travel(2);
1082 
1083     // The item should have expired now...
1084     checkeq(cb::engine_errc::no_such_key,
1085             get(h, h1, NULL, "mykey", 0).first,
1086             "Item should be gone");
1087     return SUCCESS;
1088 }
1089 
test_gat_locked(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1090 static enum test_result test_gat_locked(ENGINE_HANDLE *h,
1091                                         ENGINE_HANDLE_V1 *h1) {
1092     checkeq(ENGINE_SUCCESS,
1093             store(h, h1, NULL, OPERATION_SET, "key", "value"),
1094             "Failed to set key");
1095 
1096     checkeq(cb::engine_errc::success,
1097             getl(h, h1, nullptr, "key", 0, 15).first,
1098             "Expected getl to succeed on key");
1099 
1100     auto ret = gat(h, h1, "key", 0, 10);
1101     checkeq(ENGINE_LOCKED, ENGINE_ERROR_CODE(ret.first), "Expected LOCKED");
1102 
1103     testHarness.time_travel(16);
1104     ret = gat(h, h1, "key", 0, 10);
1105     checkeq(ENGINE_SUCCESS, ENGINE_ERROR_CODE(ret.first), "Expected success");
1106 
1107     testHarness.time_travel(11);
1108     checkeq(cb::engine_errc::no_such_key,
1109             get(h, h1, NULL, "key", 0).first,
1110             "Expected value to be expired");
1111     return SUCCESS;
1112 }
1113 
test_touch_locked(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1114 static enum test_result test_touch_locked(ENGINE_HANDLE *h,
1115                                           ENGINE_HANDLE_V1 *h1) {
1116     item *itm = NULL;
1117     checkeq(ENGINE_SUCCESS,
1118             store(h, h1, NULL, OPERATION_SET, "key", "value", &itm),
1119             "Failed to set key");
1120     h1->release(h, itm);
1121 
1122     checkeq(cb::engine_errc::success,
1123             getl(h, h1, nullptr, "key", 0, 15).first,
1124             "Expected getl to succeed on key");
1125 
1126     checkeq(ENGINE_LOCKED, touch(h, h1, "key", 0, 10),
1127             "Expected tmp fail");
1128 
1129     testHarness.time_travel(16);
1130     checkeq(ENGINE_SUCCESS, touch(h, h1, "key", 0, 10), "Expected success");
1131 
1132     testHarness.time_travel(11);
1133     checkeq(cb::engine_errc::no_such_key,
1134             get(h, h1, NULL, "key", 0).first,
1135             "Expected value to be expired");
1136 
1137     return SUCCESS;
1138 }
1139 
test_mb5215(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1140 static enum test_result test_mb5215(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1141     if (!isWarmupEnabled(h, h1)) {
1142         return SKIPPED;
1143     }
1144 
1145     checkeq(ENGINE_SUCCESS,
1146             store(h, h1, NULL, OPERATION_SET, "coolkey", "cooler"),
1147             "Failed set.");
1148 
1149     check_key_value(h, h1, "coolkey", "cooler", strlen("cooler"));
1150 
1151     // set new exptime to 111
1152     int expTime = time(NULL) + 111;
1153 
1154     checkeq(ENGINE_SUCCESS, touch(h, h1, "coolkey", 0, expTime),
1155             "touch coolkey");
1156 
1157     //reload engine
1158     testHarness.reload_engine(&h, &h1,
1159                               testHarness.engine_path,
1160                               testHarness.get_current_testcase()->cfg,
1161                               true, false);
1162 
1163     wait_for_warmup_complete(h, h1);
1164 
1165     //verify persisted expiration time
1166     const char *statkey = "key coolkey 0";
1167     int newExpTime;
1168     checkeq(cb::engine_errc::success,
1169             get(h, h1, NULL, "coolkey", 0).first,
1170             "Missing key");
1171     newExpTime = get_int_stat(h, h1, "key_exptime", statkey);
1172     checkeq(expTime, newExpTime, "Failed to persist new exptime");
1173 
1174     // evict key, touch expiration time, and verify
1175     evict_key(h, h1, "coolkey", 0, "Ejected.");
1176 
1177     expTime = time(NULL) + 222;
1178     checkeq(ENGINE_SUCCESS ,touch(h, h1, "coolkey", 0, expTime),
1179             "touch coolkey");
1180 
1181     testHarness.reload_engine(&h, &h1,
1182                               testHarness.engine_path,
1183                               testHarness.get_current_testcase()->cfg,
1184                               true, false);
1185     wait_for_warmup_complete(h, h1);
1186 
1187     checkeq(cb::engine_errc::success,
1188             get(h, h1, NULL, "coolkey", 0).first,
1189             "Missing key");
1190     newExpTime = get_int_stat(h, h1, "key_exptime", statkey);
1191     checkeq(expTime, newExpTime, "Failed to persist new exptime");
1192 
1193     return SUCCESS;
1194 }
1195 
1196 /* Testing functionality to store a value for a deleted item
1197  * and also retrieve the value of a deleted item.
1198  * Need to check:
1199  *
1200  * - Each possible state transition between Alive, Deleted-with-value and
1201  *   Deleted-no-value.
1202  */
test_delete_with_value(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1)1203 static enum test_result test_delete_with_value(ENGINE_HANDLE* h,
1204                                                ENGINE_HANDLE_V1* h1) {
1205     const uint64_t cas_0 = 0;
1206     const uint16_t vbid = 0;
1207     const void* cookie = testHarness.create_cookie();
1208 
1209     // Store an initial (not-deleted) value.
1210     checkeq(ENGINE_SUCCESS,
1211             store(h, h1, cookie, OPERATION_SET, "key", "somevalue"),
1212             "Failed set");
1213     wait_for_flusher_to_settle(h, h1);
1214 
1215     checkeq(uint64_t(1),
1216             get_stat<uint64_t>(h, h1, "vb_0:num_items", "vbucket-details 0"),
1217             "Unexpected initial item count");
1218 
1219     /* Alive -> Deleted-with-value */
1220     checkeq(ENGINE_SUCCESS,
1221             delete_with_value(h, h1, cookie, cas_0, "key", "deleted"),
1222             "Failed Alive -> Delete-with-value");
1223 
1224     checkeq(uint64_t(0),
1225             get_stat<uint64_t>(h, h1, "vb_0:num_items", "vbucket-details 0"),
1226             "Unexpected num_items after Alive -> Delete-with-value");
1227 
1228     auto res = get_value(h, h1, cookie, "key", vbid, DocStateFilter::Alive);
1229     checkeq(ENGINE_KEY_ENOENT,
1230             res.first,
1231             "Unexpectedly accessed Deleted-with-value via DocState::Alive");
1232 
1233     res = get_value(h, h1, cookie, "key", vbid, DocStateFilter::AliveOrDeleted);
1234     checkeq(ENGINE_SUCCESS,
1235             res.first,
1236             "Failed to fetch Alive -> Delete-with-value");
1237     checkeq(std::string("deleted"), res.second, "Unexpected value (deleted)");
1238 
1239     /* Deleted-with-value -> Deleted-with-value (different value). */
1240     checkeq(ENGINE_SUCCESS,
1241             delete_with_value(h, h1, cookie, cas_0, "key", "deleted 2"),
1242             "Failed Deleted-with-value -> Deleted-with-value");
1243 
1244     checkeq(uint64_t(0),
1245             get_stat<uint64_t>(h, h1, "vb_0:num_items", "vbucket-details 0"),
1246             "Unexpected num_items after Delete-with-value -> "
1247             "Delete-with-value");
1248 
1249     res = get_value(h, h1, cookie, "key", vbid, DocStateFilter::AliveOrDeleted);
1250     checkeq(ENGINE_SUCCESS, res.first, "Failed to fetch key (deleted 2)");
1251     checkeq(std::string("deleted 2"),
1252             res.second,
1253             "Unexpected value (deleted 2)");
1254 
1255     /* Delete-with-value -> Alive */
1256     checkeq(ENGINE_SUCCESS,
1257             store(h, h1, cookie, OPERATION_SET, "key", "alive 2", nullptr),
1258             "Failed Delete-with-value -> Alive");
1259     wait_for_flusher_to_settle(h, h1);
1260 
1261     checkeq(uint64_t(1),
1262             get_stat<uint64_t>(h, h1, "vb_0:num_items", "vbucket-details 0"),
1263             "Unexpected num_items after Delete-with-value -> Alive");
1264 
1265     res = get_value(h, h1, cookie, "key", vbid, DocStateFilter::Alive);
1266     checkeq(ENGINE_SUCCESS,
1267             res.first,
1268             "Failed to fetch Delete-with-value -> Alive via DocState::Alive");
1269     checkeq(std::string("alive 2"), res.second, "Unexpected value (alive 2)");
1270 
1271     // Also check via DocState::Deleted
1272     res = get_value(h, h1, cookie, "key", vbid, DocStateFilter::AliveOrDeleted);
1273     checkeq(ENGINE_SUCCESS,
1274             res.first,
1275             "Failed to fetch Delete-with-value -> Alive via DocState::Deleted");
1276     checkeq(std::string("alive 2"),
1277             res.second,
1278             "Unexpected value (alive 2) via DocState::Deleted");
1279 
1280     /* Alive -> Deleted-no-value */
1281     checkeq(ENGINE_SUCCESS,
1282             del(h, h1, "key", cas_0, vbid, cookie),
1283             "Failed Alive -> Deleted-no-value");
1284     wait_for_flusher_to_settle(h, h1);
1285 
1286     checkeq(uint64_t(0),
1287             get_stat<uint64_t>(h, h1, "vb_0:num_items", "vbucket-details 0"),
1288             "Unexpected num_items after Alive -> Delete-no-value");
1289 
1290     res = get_value(h, h1, cookie, "key", vbid, DocStateFilter::Alive);
1291     checkeq(ENGINE_KEY_ENOENT,
1292             res.first,
1293             "Unexpectedly accessed Deleted-no-value via DocState::Alive");
1294 
1295     /* Deleted-no-value -> Delete-with-value */
1296     checkeq(ENGINE_SUCCESS,
1297             delete_with_value(h, h1, cookie, cas_0, "key", "deleted 3"),
1298             "Failed delete with value (deleted 2)");
1299 
1300     res = get_value(h, h1, cookie, "key", vbid, DocStateFilter::AliveOrDeleted);
1301     checkeq(ENGINE_SUCCESS, res.first, "Failed to fetch key (deleted 3)");
1302     checkeq(std::string("deleted 3"),
1303             res.second,
1304             "Unexpected value (deleted 3)");
1305 
1306     testHarness.destroy_cookie(cookie);
1307 
1308     return SUCCESS;
1309 }
1310 
1311 /* Similar to test_delete_with_value, except also checks that CAS values
1312  */
test_delete_with_value_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1313 static enum test_result test_delete_with_value_cas(ENGINE_HANDLE *h,
1314                                                    ENGINE_HANDLE_V1 *h1) {
1315     checkeq(ENGINE_SUCCESS,
1316             store(h, h1, nullptr, OPERATION_SET, "key1", "somevalue"),
1317             "Failed set");
1318 
1319     cb::EngineErrorMetadataPair errorMetaPair;
1320 
1321     check(get_meta(h, h1, "key1", errorMetaPair), "Get meta failed");
1322 
1323     uint64_t curr_revseqno = errorMetaPair.second.seqno;
1324 
1325     /* Store a deleted item first with CAS 0 */
1326     checkeq(ENGINE_SUCCESS,
1327             store(h, h1, nullptr, OPERATION_SET, "key1", "deletevalue", nullptr,
1328                   0, 0, 3600, 0x00, DocumentState::Deleted),
1329             "Failed delete with value");
1330 
1331     check(get_meta(h, h1, "key1", errorMetaPair), "Get meta failed");
1332 
1333     checkeq(errorMetaPair.second.seqno,
1334             curr_revseqno + 1,
1335             "rev seqno should have incremented");
1336 
1337     item *i = nullptr;
1338     checkeq(ENGINE_SUCCESS,
1339             store(h, h1, nullptr, OPERATION_SET, "key2", "somevalue", &i),
1340             "Failed set");
1341 
1342     item_info info;
1343     check(h1->get_item_info(h, i, &info), "Getting item info failed");
1344 
1345     h1->release(h, i);
1346 
1347     check(get_meta(h, h1, "key2", errorMetaPair), "Get meta failed");
1348 
1349     curr_revseqno = errorMetaPair.second.seqno;
1350 
1351     /* Store a deleted item with the existing CAS value */
1352     checkeq(ENGINE_SUCCESS,
1353             store(h, h1, nullptr, OPERATION_SET, "key2", "deletevaluewithcas",
1354                   nullptr, info.cas, 0, 3600, 0x00, DocumentState::Deleted),
1355             "Failed delete value with cas");
1356 
1357     wait_for_flusher_to_settle(h, h1);
1358 
1359     check(get_meta(h, h1, "key2", errorMetaPair), "Get meta failed");
1360 
1361     checkeq(errorMetaPair.second.seqno,
1362             curr_revseqno + 1,
1363             "rev seqno should have incremented");
1364 
1365     curr_revseqno = errorMetaPair.second.seqno;
1366 
1367     checkeq(ENGINE_SUCCESS,
1368             store(h, h1, nullptr, OPERATION_SET, "key2",
1369                   "newdeletevalue", &i, 0, 0, 3600, 0x00, DocumentState::Deleted),
1370             "Failed delete value with cas");
1371 
1372     wait_for_flusher_to_settle(h, h1);
1373 
1374     check(h1->get_item_info(h, i, &info), "Getting item info failed");
1375     checkeq(int(DocumentState::Deleted),
1376             int(info.document_state),
1377             "Incorrect DocState for deleted item");
1378     checkne(uint64_t(0), info.cas, "Expected non-zero CAS for deleted item");
1379 
1380     h1->release(h, i);
1381 
1382     check(get_meta(h, h1, "key2", errorMetaPair), "Get meta failed");
1383 
1384     checkeq(errorMetaPair.second.seqno,
1385             curr_revseqno + 1,
1386             "rev seqno should have incremented");
1387 
1388     curr_revseqno = errorMetaPair.second.seqno;
1389 
1390     // Attempt to Delete-with-value using incorrect CAS (should fail)
1391     const uint64_t incorrect_CAS = info.cas + 1;
1392     checkeq(ENGINE_KEY_EEXISTS,
1393             store(h, h1, nullptr, OPERATION_SET, "key2",
1394                   "newdeletevaluewithcas", nullptr, incorrect_CAS, 0, 3600,
1395                   0x00, DocumentState::Deleted),
1396             "Expected KEY_EEXISTS with incorrect CAS");
1397 
1398     // Attempt with correct CAS.
1399     checkeq(ENGINE_SUCCESS,
1400             store(h, h1, nullptr, OPERATION_SET, "key2",
1401                   "newdeletevaluewithcas", nullptr, info.cas, 0, 3600, 0x00,
1402                   DocumentState::Deleted), "Failed delete value with cas");
1403 
1404     wait_for_flusher_to_settle(h, h1);
1405 
1406     auto ret = get(h, h1, nullptr, "key2", 0, DocStateFilter::AliveOrDeleted);
1407     checkeq(cb::engine_errc::success, ret.first, "Failed to get value");
1408 
1409     check(get_meta(h, h1, "key2", errorMetaPair), "Get meta failed");
1410 
1411     checkeq(errorMetaPair.second.seqno,
1412             curr_revseqno + 1,
1413             "rev seqno should have incremented");
1414 
1415     check(h1->get_item_info(h, ret.second.get(), &info),
1416           "Getting item info failed");
1417     checkeq(int(DocumentState::Deleted),
1418             int(info.document_state),
1419             "Incorrect DocState for deleted item");
1420 
1421     checkeq(static_cast<uint8_t>(DocumentState::Deleted),
1422             static_cast<uint8_t>(info.document_state),
1423             "document must be in deleted state");
1424 
1425     std::string buf(static_cast<char*>(info.value[0].iov_base),
1426                     info.value[0].iov_len);
1427 
1428     checkeq(0, buf.compare("newdeletevaluewithcas"), "Data mismatch");
1429 
1430     ret = get(h, h1, nullptr, "key", 0, DocStateFilter::Alive);
1431     checkeq(cb::engine_errc::no_such_key,
1432             ret.first,
1433             "Getting value should have failed");
1434 
1435     return SUCCESS;
1436 }
1437 
test_delete(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1438 static enum test_result test_delete(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1439     item *i = NULL;
1440     // First try to delete something we know to not be there.
1441     checkeq(ENGINE_KEY_ENOENT,
1442             del(h, h1, "key", 0, 0), "Failed to fail initial delete.");
1443     checkeq(ENGINE_SUCCESS,
1444             store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1445             "Failed set.");
1446     Item *it = reinterpret_cast<Item*>(i);
1447     uint64_t orig_cas = it->getCas();
1448     h1->release(h, i);
1449     check_key_value(h, h1, "key", "somevalue", 9);
1450 
1451     uint64_t cas = 0;
1452     uint64_t vb_uuid = 0;
1453     mutation_descr_t mut_info;
1454     uint64_t high_seqno = 0;
1455 
1456     memset(&mut_info, 0, sizeof(mut_info));
1457 
1458     vb_uuid = get_ull_stat(h, h1, "vb_0:0:id", "failovers");
1459     high_seqno = get_ull_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno");
1460     checkeq(ENGINE_SUCCESS, del(h, h1, "key", &cas, 0, nullptr, &mut_info),
1461             "Failed remove with value.");
1462     check(orig_cas != cas, "Expected CAS to be updated on delete");
1463     checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"), "Expected missing key");
1464     checkeq(vb_uuid, mut_info.vbucket_uuid, "Expected valid vbucket uuid");
1465     checkeq(high_seqno + 1, mut_info.seqno, "Expected valid sequence number");
1466 
1467     // Can I time travel to an expired object and delete it?
1468     checkeq(ENGINE_SUCCESS, store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1469             "Failed set.");
1470     h1->release(h, i);
1471     testHarness.time_travel(3617);
1472     checkeq(ENGINE_KEY_ENOENT, del(h, h1, "key", 0, 0),
1473             "Did not get ENOENT removing an expired object.");
1474     checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"), "Expected missing key");
1475 
1476     return SUCCESS;
1477 }
1478 
test_set_delete(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1479 static enum test_result test_set_delete(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1480     checkeq(ENGINE_SUCCESS,
1481             store(h, h1, NULL, OPERATION_SET, "key", "somevalue"),
1482             "Failed set.");
1483     check_key_value(h, h1, "key", "somevalue", 9);
1484     checkeq(ENGINE_SUCCESS, del(h, h1, "key", 0, 0),
1485             "Failed remove with value.");
1486     checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"), "Expected missing key");
1487     wait_for_flusher_to_settle(h, h1);
1488     wait_for_stat_to_be(h, h1, "curr_items", 0);
1489     return SUCCESS;
1490 }
1491 
test_set_delete_invalid_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1492 static enum test_result test_set_delete_invalid_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1493     item *i = NULL;
1494     checkeq(ENGINE_SUCCESS,
1495             store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1496             "Failed set.");
1497     check_key_value(h, h1, "key", "somevalue", 9);
1498     item_info info;
1499     check(h1->get_item_info(h, i, &info), "Should be able to get info");
1500     h1->release(h, i);
1501 
1502     checkeq(ENGINE_KEY_EEXISTS, del(h, h1, "key", info.cas + 1, 0),
1503           "Didn't expect to be able to remove the item with wrong cas");
1504 
1505     checkeq(ENGINE_SUCCESS, del(h, h1, "key", info.cas, 0),
1506         "Subsequent delete with correct CAS did not succeed");
1507 
1508     return SUCCESS;
1509 }
1510 
test_delete_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1511 static enum test_result test_delete_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1512     if (!isWarmupEnabled(h, h1)) {
1513         return SKIPPED;
1514     }
1515 
1516     wait_for_persisted_value(h, h1, "key", "value1");
1517 
1518     checkeq(ENGINE_SUCCESS,
1519             del(h, h1, "key", 0, 0), "Failed remove with value.");
1520 
1521     wait_for_persisted_value(h, h1, "key", "value2");
1522 
1523     testHarness.reload_engine(&h, &h1,
1524                               testHarness.engine_path,
1525                               testHarness.get_current_testcase()->cfg,
1526                               true, false);
1527     wait_for_warmup_complete(h, h1);
1528 
1529     check_key_value(h, h1, "key", "value2", 6);
1530     checkeq(ENGINE_SUCCESS,
1531             del(h, h1, "key", 0, 0), "Failed remove with value.");
1532     wait_for_flusher_to_settle(h, h1);
1533 
1534     testHarness.reload_engine(&h, &h1,
1535                               testHarness.engine_path,
1536                               testHarness.get_current_testcase()->cfg,
1537                               true, false);
1538     wait_for_warmup_complete(h, h1);
1539 
1540     checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"), "Expected missing key");
1541 
1542     return SUCCESS;
1543 }
1544 
test_get_delete_missing_file(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1545 static enum test_result test_get_delete_missing_file(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1546     checkeq(ENGINE_SUCCESS,
1547             get_stats(h, {}, add_stats),
1548             "Failed to get stats.");
1549 
1550     const char *key = "key";
1551     wait_for_persisted_value(h, h1, key, "value2delete");
1552 
1553     // whack the db file and directory where the key is stored
1554     std::string dbname = vals["ep_dbname"];
1555     rmdb(dbname.c_str());
1556 
1557     auto ret = get(h, h1, NULL, key, 0);
1558 
1559     // ep engine must be unaware of well-being of the db file as long as
1560     // the item is still in the memory
1561     checkeq(cb::engine_errc::success, ret.first, "Expected success for get");
1562 
1563     evict_key(h, h1, key);
1564     ret = get(h, h1, NULL, key, 0);
1565 
1566     // ep engine must be now aware of the ill-fated db file where
1567     // the item is supposedly stored
1568     checkeq(cb::engine_errc::temporary_failure,
1569             ret.first,
1570             "Expected tmp fail for get");
1571 
1572     return SUCCESS;
1573 }
1574 
test_bug2509(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1575 static enum test_result test_bug2509(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1576     for (int j = 0; j < 10000; ++j) {
1577         checkeq(ENGINE_SUCCESS,
1578                 store(h, h1, NULL, OPERATION_SET, "key", "somevalue"),
1579                 "Failed set.");
1580         usleep(10);
1581         checkeq(ENGINE_SUCCESS, del(h, h1, "key", 0, 0), "Failed remove with value.");
1582         usleep(10);
1583     }
1584 
1585     if (isWarmupEnabled(h, h1)) {
1586         // Restart again, to verify we don't have any duplicates.
1587         testHarness.reload_engine(&h, &h1,
1588                                   testHarness.engine_path,
1589                                   testHarness.get_current_testcase()->cfg,
1590                                   true, false);
1591         wait_for_warmup_complete(h, h1);
1592 
1593         return get_int_stat(h, h1, "ep_warmup_dups") == 0 ? SUCCESS : FAIL;
1594     }
1595     return SUCCESS;
1596 }
1597 
test_bug7023(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1598 static enum test_result test_bug7023(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1599     std::vector<std::string> keys;
1600     // Make a vbucket mess.
1601     const int nitems = 10000;
1602     const int iterations = 5;
1603     for (int j = 0; j < nitems; ++j) {
1604         keys.push_back("key" + std::to_string(j));
1605     }
1606 
1607     std::vector<std::string>::iterator it;
1608     for (int j = 0; j < iterations; ++j) {
1609         check(set_vbucket_state(h, h1, 0, vbucket_state_dead),
1610               "Failed set set vbucket 0 dead.");
1611         checkeq(ENGINE_SUCCESS, vbucketDelete(h, h1, 0), "expected success");
1612         checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS,
1613                 last_status.load(),
1614                 "Expected vbucket deletion to work.");
1615         check(set_vbucket_state(h, h1, 0, vbucket_state_active),
1616               "Failed set set vbucket 0 active.");
1617         for (it = keys.begin(); it != keys.end(); ++it) {
1618             checkeq(ENGINE_SUCCESS,
1619                     store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str()),
1620                     "Failed to store a value");
1621         }
1622     }
1623     wait_for_flusher_to_settle(h, h1);
1624 
1625     if (isWarmupEnabled(h, h1)) {
1626         // Restart again, to verify no data loss.
1627         testHarness.reload_engine(&h, &h1,
1628                                   testHarness.engine_path,
1629                                   testHarness.get_current_testcase()->cfg,
1630                                   true, false);
1631         wait_for_warmup_complete(h, h1);
1632         checkeq(nitems,
1633                 get_int_stat(h, h1, "ep_warmup_value_count", "warmup"),
1634                 "Incorrect items following warmup");
1635     }
1636     return SUCCESS;
1637 }
1638 
test_mb3169(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1639 static enum test_result test_mb3169(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1640     checkeq(ENGINE_SUCCESS,
1641             store(h, h1, NULL, OPERATION_SET, "set", "value"),
1642             "Failed to store a value");
1643     checkeq(ENGINE_SUCCESS,
1644             store(h, h1, NULL, OPERATION_SET, "delete", "0"),
1645             "Failed to store a value");
1646     checkeq(ENGINE_SUCCESS,
1647             store(h, h1, NULL, OPERATION_SET, "get", "getvalue"),
1648             "Failed to store a value");
1649 
1650     wait_for_stat_to_be(h, h1, "ep_total_persisted", 3);
1651 
1652     evict_key(h, h1, "set", 0, "Ejected.");
1653     evict_key(h, h1, "delete", 0, "Ejected.");
1654     evict_key(h, h1, "get", 0, "Ejected.");
1655 
1656     checkeq(3, get_int_stat(h, h1, "curr_items"), "Expected 3 items");
1657     checkeq(3,
1658             get_int_stat(h, h1, "ep_num_non_resident"),
1659             "Expected all items to be resident");
1660 
1661     checkeq(ENGINE_SUCCESS,
1662             store(h, h1, NULL, OPERATION_SET, "set", "value2"),
1663             "Failed to store a value");
1664     wait_for_flusher_to_settle(h, h1);
1665 
1666     checkeq(3, get_int_stat(h, h1, "curr_items"), "Expected 3 items");
1667     checkeq(2, get_int_stat(h, h1, "ep_num_non_resident"),
1668           "Expected mutation to mark item resident");
1669 
1670     checkeq(ENGINE_SUCCESS, del(h, h1, "delete", 0, 0),
1671             "Delete failed");
1672 
1673     wait_for_flusher_to_settle(h, h1);
1674 
1675     checkeq(2, get_int_stat(h, h1, "curr_items"), "Expected 2 items after del");
1676     checkeq(1, get_int_stat(h, h1, "ep_num_non_resident"),
1677             "Expected delete to remove non-resident item");
1678 
1679     check_key_value(h, h1, "get", "getvalue", 8);
1680 
1681     checkeq(2, get_int_stat(h, h1, "curr_items"), "Expected 2 items after get");
1682     checkeq(0, get_int_stat(h, h1, "ep_num_non_resident"),
1683             "Expected all items to be resident");
1684     return SUCCESS;
1685 }
1686 
test_mb5172(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1687 static enum test_result test_mb5172(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1688     if (!isWarmupEnabled(h, h1)) {
1689         return SKIPPED;
1690     }
1691 
1692     checkeq(ENGINE_SUCCESS,
1693             store(h, h1, NULL, OPERATION_SET, "key-1", "value-1"),
1694             "Failed to store a value");
1695 
1696     checkeq(ENGINE_SUCCESS,
1697             store(h, h1, NULL, OPERATION_SET, "key-2", "value-2"),
1698             "Failed to store a value");
1699 
1700     wait_for_flusher_to_settle(h, h1);
1701 
1702     checkeq(0, get_int_stat(h, h1, "ep_num_non_resident"),
1703             "Expected all items to be resident");
1704 
1705     // restart the server.
1706     testHarness.reload_engine(&h, &h1,
1707                               testHarness.engine_path,
1708                               testHarness.get_current_testcase()->cfg,
1709                               true, false);
1710 
1711     wait_for_warmup_complete(h, h1);
1712     checkeq(0, get_int_stat(h, h1, "ep_num_non_resident"),
1713             "Expected all items to be resident");
1714     return SUCCESS;
1715 }
1716 
test_set_vbucket_out_of_range(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1717 static enum test_result test_set_vbucket_out_of_range(ENGINE_HANDLE *h,
1718                                                        ENGINE_HANDLE_V1 *h1) {
1719     check(!set_vbucket_state(h, h1, 10000, vbucket_state_active),
1720           "Shouldn't have been able to set vbucket 10000");
1721     return SUCCESS;
1722 }
1723 
set_max_cas_mb21190(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1724 static enum test_result set_max_cas_mb21190(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1725     uint64_t max_cas = get_ull_stat(h, h1, "vb_0:max_cas", "vbucket-details 0");
1726     std::string max_cas_str = std::to_string(max_cas+1);
1727     set_param(h, h1, protocol_binary_engine_param_vbucket,
1728               "max_cas", max_cas_str.data(), 0);
1729     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1730             "Failed to set_param max_cas");
1731     checkeq(max_cas + 1,
1732             get_ull_stat(h, h1, "vb_0:max_cas", "vbucket-details 0"),
1733             "max_cas didn't change");
1734     set_param(h, h1, protocol_binary_engine_param_vbucket,
1735               "max_cas", max_cas_str.data(), 1);
1736     checkeq(PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET, last_status.load(),
1737             "Expected not my vbucket for vb 1");
1738     set_param(h, h1, protocol_binary_engine_param_vbucket,
1739               "max_cas", "JUNK", 0);
1740     checkeq(PROTOCOL_BINARY_RESPONSE_EINVAL, last_status.load(),
1741             "Expected EINVAL");
1742     return SUCCESS;
1743 }
1744 
warmup_mb21769(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1745 static enum test_result warmup_mb21769(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1746     if (!isWarmupEnabled(h, h1)) {
1747         return SKIPPED;
1748     }
1749 
1750     // Validate some VB data post warmup
1751     // VB 0 will be empty
1752     // VB 1 will not be empty
1753     // VB 2 will not be empty and will have had set_state as the final ops
1754 
1755     check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1756           "Failed to set vbucket state for vb1");
1757     check(set_vbucket_state(h, h1, 2, vbucket_state_active),
1758           "Failed to set vbucket state for vb2");
1759 
1760     const int num_items = 10;
1761     write_items(h, h1, num_items, 0, "vb1", "value", 0/*expiry*/, 1/*vb*/);
1762     write_items(h, h1, num_items, 0, "vb2", "value", 0/*expiry*/, 2/*vb*/);
1763     wait_for_flusher_to_settle(h, h1);
1764 
1765     // flip replica to active to drive more _local writes
1766     check(set_vbucket_state(h, h1, 2, vbucket_state_replica),
1767           "Failed to set vbucket state (replica) for vb2");
1768     wait_for_flusher_to_settle(h, h1);
1769 
1770     check(set_vbucket_state(h, h1, 2, vbucket_state_active),
1771           "Failed to set vbucket state (replica) for vb2");
1772     wait_for_flusher_to_settle(h, h1);
1773 
1774     // Force a shutdown so the warmup will create failover entries
1775     testHarness.reload_engine(&h, &h1,
1776                               testHarness.engine_path,
1777                               testHarness.get_current_testcase()->cfg,
1778                               true, true);
1779 
1780     wait_for_warmup_complete(h, h1);
1781 
1782     // values of interested stats for each VB
1783     std::array<uint64_t, 3> high_seqnos = {{0, num_items, num_items}};
1784     std::array<uint64_t, 3> snap_starts = {{0, num_items, num_items}};
1785     std::array<uint64_t, 3> snap_ends = {{0, num_items, num_items}};
1786     // we will check the seqno of the 0th entry of each vbucket's failover table
1787     std::array<uint64_t, 3> failover_entry0 = {{0, num_items, num_items}};
1788 
1789     for (uint64_t vb = 0; vb <= 2; vb++) {
1790         std::string vb_prefix = "vb_" + std::to_string(vb) + ":";
1791         std::string high_seqno = vb_prefix + "high_seqno";
1792         std::string snap_start = vb_prefix + "last_persisted_snap_start";
1793         std::string snap_end = vb_prefix + "last_persisted_snap_end";
1794         std::string fail0 = vb_prefix + "0:seq";
1795         std::string vb_group_key = "vbucket-seqno " + std::to_string(vb);
1796         std::string failovers_key = "failovers " + std::to_string(vb);
1797 
1798         checkeq(high_seqnos[vb],
1799                 get_ull_stat(h, h1, high_seqno.c_str(), vb_group_key.c_str()),
1800                 std::string("high_seqno incorrect vb:" + std::to_string(vb)).c_str());
1801         checkeq(snap_starts[vb],
1802                 get_ull_stat(h, h1, snap_start.c_str(), vb_group_key.c_str()),
1803                 std::string("snap_start incorrect vb:" + std::to_string(vb)).c_str());
1804         checkeq(snap_ends[vb],
1805                 get_ull_stat(h, h1, snap_end.c_str(), vb_group_key.c_str()),
1806                 std::string("snap_end incorrect vb:" + std::to_string(vb)).c_str());
1807         checkeq(failover_entry0[vb],
1808                 get_ull_stat(h, h1, fail0.c_str(), failovers_key.c_str()),
1809                 std::string("failover table entry 0 is incorrect vb:" + std::to_string(vb)).c_str());
1810     }
1811 
1812     return SUCCESS;
1813 }
1814 
1815 /**
1816  * Callback from the document API being called after the CAS was assigned
1817  * to the object. We're allowed to modify the content, so let's just change
1818  * the string.
1819  *
1820  * @param info info about the document
1821  */
1822 static uint64_t pre_link_seqno(0);
pre_link_doc_callback(item_info& info)1823 static void pre_link_doc_callback(item_info& info) {
1824     checkne(uint64_t(0), info.cas, "CAS value should be set");
1825     // mock the actual value so we can see it was changed
1826     memcpy(info.value[0].iov_base, "valuesome", 9);
1827     pre_link_seqno = info.seqno;
1828 }
1829 
1830 /**
1831  * Verify that we've hooked into the checkpoint and that the pre-link
1832  * document api method is called.
1833  */
pre_link_document(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1834 static test_result pre_link_document(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1835     item_info info;
1836 
1837     PreLinkFunction function = pre_link_doc_callback;
1838     testHarness.set_pre_link_function(function);
1839     checkeq(ENGINE_SUCCESS,
1840             store(h, h1, nullptr, OPERATION_SET, "key", "somevalue"),
1841             "Failed set.");
1842     testHarness.set_pre_link_function({});
1843 
1844     // Fetch the value and verify that the callback was called!
1845     auto ret = get(h, h1, nullptr, "key", 0);
1846     checkeq(cb::engine_errc::success, ret.first, "get failed");
1847     check(h1->get_item_info(h, ret.second.get(), &info),
1848           "Failed to get item info.");
1849     checkeq(0, memcmp(info.value[0].iov_base, "valuesome", 9),
1850            "Expected value to be modified");
1851     checkeq(pre_link_seqno, info.seqno, "Sequence numbers should match");
1852 
1853     return SUCCESS;
1854 }
1855 
1856 /**
1857  * verify that get_if works as expected
1858  */
get_if(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1)1859 static test_result get_if(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1) {
1860     const std::string key("get_if");
1861 
1862     checkeq(ENGINE_SUCCESS,
1863             store(h, h1, nullptr, OPERATION_SET, key.c_str(), "somevalue"),
1864             "Failed set.");
1865 
1866     if (isPersistentBucket(h, h1)) {
1867         wait_for_flusher_to_settle(h, h1);
1868         evict_key(h, h1, key.c_str(), 0, "Ejected.");
1869     }
1870 
1871     const auto* cookie = testHarness.create_cookie();
1872     auto doc = h1->get_if(h,
1873                           cookie,
1874                           DocKey(key, testHarness.doc_namespace),
1875                           0,
1876                           [](const item_info&) { return true; });
1877     check(doc.second, "document should be found");
1878 
1879     doc = h1->get_if(h,
1880                      cookie,
1881                      DocKey(key, testHarness.doc_namespace),
1882                      0,
1883                      [](const item_info&) { return false; });
1884     check(!doc.second, "document should not be found");
1885 
1886     doc = h1->get_if(h,
1887                      cookie,
1888                      DocKey("no", testHarness.doc_namespace),
1889                      0,
1890                      [](const item_info&) { return true; });
1891     check(!doc.second, "non-existing document should not be found");
1892 
1893     checkeq(ENGINE_SUCCESS, del(h, h1, key.c_str(), 0, 0),
1894             "Failed remove with value");
1895 
1896     doc = h1->get_if(h,
1897                      cookie,
1898                      DocKey(key, testHarness.doc_namespace),
1899                      0,
1900                      [](const item_info&) { return true; });
1901     check(!doc.second, "deleted document should not be found");
1902 
1903     testHarness.destroy_cookie(cookie);
1904 
1905     return SUCCESS;
1906 }
1907 
max_ttl_out_of_range(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1)1908 static test_result max_ttl_out_of_range(ENGINE_HANDLE* h,
1909                                         ENGINE_HANDLE_V1* h1) {
1910     // Test absolute first as this is the bigger time travel
1911     check(!set_param(
1912                   h, h1, protocol_binary_engine_param_flush, "max_ttl", "-1"),
1913           "Should not be allowed to set a negative value");
1914     check(!set_param(h,
1915                      h1,
1916                      protocol_binary_engine_param_flush,
1917                      "max_ttl",
1918                      "2147483648"),
1919           "Should not be allowed to set > int32::max");
1920 
1921     return SUCCESS;
1922 }
1923 
max_ttl(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1)1924 static test_result max_ttl(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1) {
1925     // Make limit be greater than 30 days in seconds so that ep-engine must
1926     // create a absolute expiry time internally.
1927     const int absoluteExpiry = (60 * 60 * 24 * 31);
1928     auto absoluteExpiryStr = std::to_string(absoluteExpiry);
1929 
1930     const int relativeExpiry = 100;
1931     auto relativeExpiryStr = std::to_string(relativeExpiry);
1932 
1933     checkeq(0, get_int_stat(h, h1, "ep_max_ttl"), "max_ttl should be 0");
1934 
1935     // Test absolute first as this is the bigger time travel
1936     check(set_param(h,
1937                     h1,
1938                     protocol_binary_engine_param_flush,
1939                     "max_ttl",
1940                     absoluteExpiryStr.c_str()),
1941           "Failed to set max_ttl");
1942     checkeq(absoluteExpiry,
1943             get_int_stat(h, h1, "ep_max_ttl"),
1944             "max_ttl didn't change");
1945 
1946     // Store will set 0 expiry, which results in 100 seconds of ttl
1947     checkeq(ENGINE_SUCCESS,
1948             store(h, h1, nullptr, OPERATION_SET, "key-abs", "somevalue"),
1949             "Failed set.");
1950 
1951     cb::EngineErrorMetadataPair errorMetaPair;
1952     check(get_meta(h, h1, "key-abs", errorMetaPair), "Get meta failed");
1953     checkne(time_t(0),
1954             errorMetaPair.second.exptime,
1955             "expiry should not be zero");
1956 
1957     // Force expiry
1958     testHarness.time_travel(absoluteExpiry + 1);
1959 
1960     auto ret = get(h, h1, NULL, "key-abs", 0);
1961     checkeq(cb::engine_errc::no_such_key,
1962             ret.first,
1963             "Failed, expected no_such_key.");
1964 
1965     check(set_param(h,
1966                     h1,
1967                     protocol_binary_engine_param_flush,
1968                     "max_ttl",
1969                     relativeExpiryStr.c_str()),
1970           "Failed to set max_ttl");
1971     checkeq(relativeExpiry,
1972             get_int_stat(h, h1, "ep_max_ttl"),
1973             "max_ttl didn't change");
1974 
1975     // Store will set 0 expiry, which results in 100 seconds of ttl
1976     checkeq(ENGINE_SUCCESS,
1977             store(h, h1, nullptr, OPERATION_SET, "key-rel", "somevalue"),
1978             "Failed set.");
1979 
1980     check(get_meta(h, h1, "key-rel", errorMetaPair), "Get meta failed");
1981     checkne(time_t(0),
1982             errorMetaPair.second.exptime,
1983             "expiry should not be zero");
1984 
1985     // Force expiry
1986     testHarness.time_travel(relativeExpiry + 1);
1987 
1988     ret = get(h, h1, NULL, "key-rel", 0);
1989     checkeq(cb::engine_errc::no_such_key,
1990             ret.first,
1991             "Failed, expected no_such_key.");
1992 
1993     return SUCCESS;
1994 }
1995 
max_ttl_setWithMeta(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1)1996 static test_result max_ttl_setWithMeta(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1) {
1997     // Make limit be greater than 30 days in seconds so that ep-engine must
1998     // create a absolute expiry time internally.
1999     const int absoluteExpiry = (60 * 60 * 24 * 31);
2000     auto absoluteExpiryStr = std::to_string(absoluteExpiry);
2001     std::string keyAbs = "key-abs";
2002 
2003     const int relativeExpiry = 100;
2004     auto relativeExpiryStr = std::to_string(relativeExpiry);
2005     std::string keyRel = "key-rel";
2006 
2007     checkeq(0, get_int_stat(h, h1, "ep_max_ttl"), "max_ttl should be 0");
2008 
2009     // Test absolute first as this is the bigger time travel
2010     check(set_param(h,
2011                     h1,
2012                     protocol_binary_engine_param_flush,
2013                     "max_ttl",
2014                     absoluteExpiryStr.c_str()),
2015           "Failed to set max_ttl");
2016     checkeq(absoluteExpiry,
2017             get_int_stat(h, h1, "ep_max_ttl"),
2018             "max_ttl didn't change");
2019 
2020     // SWM with 0 expiry which results in an expiry being set
2021     ItemMetaData itemMeta(0xdeadbeef, 10, 0xf1a95, 0 /*expiry*/);
2022     set_with_meta(h,
2023                   h1,
2024                   keyAbs.c_str(),
2025                   keyAbs.size(),
2026                   keyAbs.c_str(),
2027                   keyAbs.size(),
2028                   0 /*vb*/,
2029                   &itemMeta,
2030                   0 /*cas*/);
2031 
2032     cb::EngineErrorMetadataPair errorMetaPair;
2033     check(get_meta(h, h1, keyAbs.c_str(), errorMetaPair), "Get meta failed");
2034     checkne(time_t(0),
2035             errorMetaPair.second.exptime,
2036             "expiry should not be zero");
2037 
2038     // Force expiry
2039     testHarness.time_travel(absoluteExpiry + 1);
2040 
2041     auto ret = get(h, h1, NULL, keyAbs.c_str(), 0);
2042     checkeq(cb::engine_errc::no_such_key,
2043             ret.first,
2044             "Failed, expected no_such_key.");
2045 
2046     check(set_param(h,
2047                     h1,
2048                     protocol_binary_engine_param_flush,
2049                     "max_ttl",
2050                     relativeExpiryStr.c_str()),
2051           "Failed to set max_ttl");
2052     checkeq(relativeExpiry,
2053             get_int_stat(h, h1, "ep_max_ttl"),
2054             "max_ttl didn't change");
2055 
2056     set_with_meta(h,
2057                   h1,
2058                   keyRel.c_str(),
2059                   keyRel.size(),
2060                   keyRel.c_str(),
2061                   keyRel.size(),
2062                   0 /*vb*/,
2063                   &itemMeta,
2064                   0 /*cas*/);
2065 
2066     check(get_meta(h, h1, keyRel.c_str(), errorMetaPair), "Get meta failed");
2067     checkne(time_t(0),
2068             errorMetaPair.second.exptime,
2069             "expiry should not be zero");
2070 
2071     // Force expiry
2072     testHarness.time_travel(relativeExpiry + 1);
2073 
2074     ret = get(h, h1, NULL, keyRel.c_str(), 0);
2075     checkeq(cb::engine_errc::no_such_key,
2076             ret.first,
2077             "Failed, expected no_such_key.");
2078 
2079     // Final test, exceed the maxTTL and check we got capped!
2080     itemMeta.exptime = errorMetaPair.second.exptime + 1000;
2081     set_with_meta(h,
2082                   h1,
2083                   keyRel.c_str(),
2084                   keyRel.size(),
2085                   keyRel.c_str(),
2086                   keyRel.size(),
2087                   0 /*vb*/,
2088                   &itemMeta,
2089                   0 /*cas*/);
2090 
2091     check(get_meta(h, h1, keyRel.c_str(), errorMetaPair), "Get meta failed");
2092     checkne(itemMeta.exptime,
2093             errorMetaPair.second.exptime,
2094             "expiry should have been changed/capped");
2095 
2096     return SUCCESS;
2097 }
2098 
2099 ///////////////////////////////////////////////////////////////////////////////
2100 // Test manifest //////////////////////////////////////////////////////////////
2101 ///////////////////////////////////////////////////////////////////////////////
2102 
2103 const char *default_dbname = "./ep_testsuite_basic";
2104 
2105 BaseTestCase testsuite_testcases[] = {
2106         TestCase("test alloc limit",
2107                  test_alloc_limit,
2108                  test_setup,
2109                  teardown,
2110                  NULL,
2111                  prepare,
2112                  cleanup),
2113         TestCase("test_memory_tracking",
2114                  test_memory_tracking,
2115                  test_setup,
2116                  teardown,
2117                  NULL,
2118                  prepare,
2119                  cleanup),
2120         TestCase("test max_size - water_mark changes",
2121                  test_max_size_and_water_marks_settings,
2122                  test_setup,
2123                  teardown,
2124                  "max_size=1000;ht_locks=1;ht_size=3",
2125                  prepare,
2126                  cleanup),
2127         TestCase("test whitespace dbname",
2128                  test_whitespace_db,
2129                  test_setup,
2130                  teardown,
2131                  "dbname=" WHITESPACE_DB ";ht_locks=1;ht_size=3",
2132                  prepare,
2133                  cleanup),
2134         TestCase("get miss",
2135                  test_get_miss,
2136                  test_setup,
2137                  teardown,
2138                  NULL,
2139                  prepare,
2140                  cleanup),
2141         TestCase("set", test_set, test_setup, teardown, NULL, prepare, cleanup),
2142         TestCase("concurrent set",
2143                  test_conc_set,
2144                  test_setup,
2145                  teardown,
2146                  NULL,
2147                  prepare,
2148                  cleanup),
2149         TestCase("multi set",
2150                  test_multi_set,
2151                  test_setup,
2152                  teardown,
2153                  NULL,
2154                  prepare,
2155                  cleanup),
2156         TestCase("set+get hit",
2157                  test_set_get_hit,
2158                  test_setup,
2159                  teardown,
2160                  NULL,
2161                  prepare,
2162                  cleanup),
2163         TestCase("test getl then del with cas",
2164                  test_getl_delete_with_cas,
2165                  test_setup,
2166                  teardown,
2167                  NULL,
2168                  prepare,
2169                  cleanup),
2170         TestCase("test getl then del with bad cas",
2171                  test_getl_delete_with_bad_cas,
2172                  test_setup,
2173                  teardown,
2174                  NULL,
2175                  prepare,
2176                  cleanup),
2177         TestCase("test getl then set with meta",
2178                  test_getl_set_del_with_meta,
2179                  test_setup,
2180                  teardown,
2181                  NULL,
2182                  prepare,
2183                  cleanup),
2184         TestCase("getl",
2185                  test_getl,
2186                  test_setup,
2187                  teardown,
2188                  NULL,
2189                  prepare,
2190                  cleanup),
2191         TestCase("unl", test_unl, test_setup, teardown, NULL, prepare, cleanup),
2192         TestCase("unl not my vbucket",
2193                  test_unl_nmvb,
2194                  test_setup,
2195                  teardown,
2196                  NULL,
2197                  prepare,
2198                  cleanup),
2199         TestCase("set+get hit (bin)",
2200                  test_set_get_hit_bin,
2201                  test_setup,
2202                  teardown,
2203                  NULL,
2204                  prepare,
2205                  cleanup),
2206         TestCase("set with cas non-existent",
2207                  test_set_with_cas_non_existent,
2208                  test_setup,
2209                  teardown,
2210                  NULL,
2211                  prepare,
2212                  cleanup),
2213         TestCase("set+change flags",
2214                  test_set_change_flags,
2215                  test_setup,
2216                  teardown,
2217                  NULL,
2218                  prepare,
2219                  cleanup),
2220         TestCase("add", test_add, test_setup, teardown, NULL, prepare, cleanup),
2221         TestCase("add+add(same cas)",
2222                  test_add_add_with_cas,
2223                  test_setup,
2224                  teardown,
2225                  NULL,
2226                  prepare,
2227                  cleanup),
2228         TestCase("cas", test_cas, test_setup, teardown, NULL, prepare, cleanup),
2229         TestCase("replace",
2230                  test_replace,
2231                  test_setup,
2232                  teardown,
2233                  NULL,
2234                  prepare,
2235                  cleanup),
2236         TestCase("test touch",
2237                  test_touch,
2238                  test_setup,
2239                  teardown,
2240                  NULL,
2241                  prepare,
2242                  cleanup),
2243         TestCase("test touch (MB-7342)",
2244                  test_touch_mb7342,
2245                  test_setup,
2246                  teardown,
2247                  NULL,
2248                  prepare,
2249                  cleanup),
2250         TestCase("test touch (MB-10277)",
2251                  test_touch_mb10277,
2252                  test_setup,
2253                  teardown,
2254                  NULL,
2255                  prepare_ep_bucket,
2256                  cleanup),
2257         TestCase("test gat",
2258                  test_gat,
2259                  test_setup,
2260                  teardown,
2261                  NULL,
2262                  prepare,
2263                  cleanup),
2264         TestCase("test locked gat",
2265                  test_gat_locked,
2266                  test_setup,
2267                  teardown,
2268                  NULL,
2269                  prepare,
2270                  cleanup),
2271         TestCase("test locked touch",
2272                  test_touch_locked,
2273                  test_setup,
2274                  teardown,
2275                  NULL,
2276                  prepare,
2277                  cleanup),
2278         TestCase("test mb5215",
2279                  test_mb5215,
2280                  test_setup,
2281                  teardown,
2282                  NULL,
2283                  // TODO RDB: implement getItemCount. Needs the
2284                  // 'ep_num_non_resident' stat.
2285                  prepare_ep_bucket_skip_broken_under_rocks,
2286                  cleanup),
2287         TestCase("delete",
2288                  test_delete,
2289                  test_setup,
2290                  teardown,
2291                  NULL,
2292                  prepare,
2293                  cleanup),
2294         TestCase("delete with value",
2295                  test_delete_with_value,
2296                  test_setup,
2297                  teardown,
2298                  NULL,
2299                  prepare,
2300                  cleanup),
2301         TestCase("delete with value CAS",
2302                  test_delete_with_value_cas,
2303                  test_setup,
2304                  teardown,
2305                  NULL,
2306                  prepare,
2307                  cleanup),
2308         TestCase("set/delete",
2309                  test_set_delete,
2310                  test_setup,
2311                  teardown,
2312                  NULL,
2313                  prepare,
2314                  cleanup),
2315         TestCase("set/delete (invalid cas)",
2316                  test_set_delete_invalid_cas,
2317                  test_setup,
2318                  teardown,
2319                  NULL,
2320                  prepare,
2321                  cleanup),
2322         TestCase("delete/set/delete",
2323                  test_delete_set,
2324                  test_setup,
2325                  teardown,
2326                  NULL,
2327                  prepare,
2328                  cleanup),
2329         TestCase("get/delete with missing db file",
2330                  test_get_delete_missing_file,
2331                  test_setup,
2332                  teardown,
2333                  NULL,
2334                  // TODO RDB: This test fails because under RocksDB we can
2335                  // still find a key after deleting the DB file and evicting
2336                  // the key in the internal MemTable (which is also used as
2337                  // read-cache).
2338                  prepare_ep_bucket_skip_broken_under_rocks,
2339                  cleanup),
2340         TestCase("retain rowid over a soft delete",
2341                  test_bug2509,
2342                  test_setup,
2343                  teardown,
2344                  NULL,
2345                  prepare,
2346                  cleanup),
2347         TestCase("vbucket deletion doesn't affect new data",
2348                  test_bug7023,
2349                  test_setup,
2350                  teardown,
2351                  NULL,
2352                  // TODO RDB: implement getItemCount. Needs the
2353                  // 'ep_warmup_value_count' stat.
2354                  prepare_skip_broken_under_rocks,
2355                  cleanup),
2356         TestCase("non-resident decrementers",
2357                  test_mb3169,
2358                  test_setup,
2359                  teardown,
2360                  NULL,
2361                  // TODO RDB: implement getItemCount. Needs the 'curr_items'
2362                  // stat.
2363                  prepare_ep_bucket_skip_broken_under_rocks,
2364                  cleanup),
2365         TestCase("resident ratio after warmup",
2366                  test_mb5172,
2367                  test_setup,
2368                  teardown,
2369                  NULL,
2370                  prepare,
2371                  cleanup),
2372         TestCase("set vb 10000",
2373                  test_set_vbucket_out_of_range,
2374                  test_setup,
2375                  teardown,
2376                  "max_vbuckets=1024",
2377                  prepare,
2378                  cleanup),
2379         TestCase("set max_cas MB21190",
2380                  set_max_cas_mb21190,
2381                  test_setup,
2382                  teardown,
2383                  nullptr,
2384                  prepare,
2385                  cleanup),
2386         TestCase("warmup_mb21769",
2387                  warmup_mb21769,
2388                  test_setup,
2389                  teardown,
2390                  nullptr,
2391                  prepare,
2392                  cleanup),
2393 
2394         TestCase("pre_link_document",
2395                  pre_link_document,
2396                  test_setup,
2397                  teardown,
2398                  nullptr,
2399                  prepare,
2400                  cleanup),
2401 
2402         TestCase("engine get_if",
2403                  get_if,
2404                  test_setup,
2405                  teardown,
2406                  nullptr,
2407                  prepare,
2408                  cleanup),
2409 
2410         TestCase("test max_ttl range",
2411                  max_ttl_out_of_range,
2412                  test_setup,
2413                  teardown,
2414                  nullptr,
2415                  prepare,
2416                  cleanup),
2417 
2418         TestCase("test max_ttl",
2419                  max_ttl,
2420                  test_setup,
2421                  teardown,
2422                  nullptr,
2423                  prepare,
2424                  cleanup),
2425 
2426         TestCase("test max_ttl_setWithMeta",
2427                  max_ttl_setWithMeta,
2428                  test_setup,
2429                  teardown,
2430                  nullptr,
2431                  prepare,
2432                  cleanup),
2433 
2434         // sentinel
2435         TestCase(NULL, NULL, NULL, NULL, NULL, prepare, cleanup)};
2436