1 /* -*- MODE: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3  *     Copyright 2016 Couchbase, Inc
4  *
5  *   Licensed under the Apache License, Version 2.0 (the "License");
6  *   you may not use this file except in compliance with the License.
7  *   You may obtain a copy of the License at
8  *
9  *       http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *   Unless required by applicable law or agreed to in writing, software
12  *   distributed under the License is distributed on an "AS IS" BASIS,
13  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *   See the License for the specific language governing permissions and
15  *   limitations under the License.
16  */
17 
18 /*
19  * Testsuite for 'basic' key-value functionality in ep-engine.
20  */
21 
22 #include "ep_test_apis.h"
23 #include "ep_testsuite_common.h"
24 
25 #include <platform/cb_malloc.h>
26 #include <platform/cbassert.h>
27 #include <platform/dirutils.h>
28 #include <platform/platform_thread.h>
29 
30 #include <array>
31 #include <memcached/types.h>
32 
33 #define WHITESPACE_DB "whitespace sucks.db"
34 
35 // Types //////////////////////////////////////////////////////////////////////
36 
37 
38 // Helper functions ///////////////////////////////////////////////////////////
39 
epsilon(int val, int target, int ep=5)40 static bool epsilon(int val, int target, int ep=5) {
41     return abs(val - target) < ep;
42 }
43 
44 
45 // Testcases //////////////////////////////////////////////////////////////////
46 
test_alloc_limit(EngineIface* h)47 static enum test_result test_alloc_limit(EngineIface* h) {
48     auto rv = allocate(h,
49                        NULL,
50                        "key",
51                        20 * 1024 * 1024,
52                        0,
53                        0,
54                        PROTOCOL_BINARY_RAW_BYTES,
55                        Vbid(0));
56     checkeq(cb::engine_errc::success, rv.first, "Allocated 20MB item");
57 
58     rv = allocate(h,
59                   NULL,
60                   "key",
61                   (20 * 1024 * 1024) + 1,
62                   0,
63                   0,
64                   PROTOCOL_BINARY_RAW_BYTES,
65                   Vbid(0));
66     checkeq(cb::engine_errc::too_big, rv.first, "Object too big");
67 
68     return SUCCESS;
69 }
70 
test_memory_tracking(EngineIface* h)71 static enum test_result test_memory_tracking(EngineIface* h) {
72     // Need memory tracker to be able to check our memory usage.
73     std::string tracker = get_str_stat(h, "ep_mem_tracker_enabled");
74     if (tracker == "true") {
75         return SUCCESS;
76     } else {
77         std::cerr << "Memory tracker not enabled ...";
78         return SKIPPED;
79     }
80 }
81 
test_max_size_and_water_marks_settings(EngineIface* h)82 static enum test_result test_max_size_and_water_marks_settings(EngineIface* h) {
83     checkeq(1000, get_int_stat(h, "ep_max_size"), "Incorrect initial size.");
84     check(epsilon(get_int_stat(h, "ep_mem_low_wat"), 750),
85           "Incorrect initial low wat.");
86     check(epsilon(get_int_stat(h, "ep_mem_high_wat"), 850),
87           "Incorrect initial high wat.");
88     checkeq(0.75f,
89             get_float_stat(h, "ep_mem_low_wat_percent"),
90             "Incorrect initial low wat. percent");
91     checkeq(0.85f,
92             get_float_stat(h, "ep_mem_high_wat_percent"),
93             "Incorrect initial high wat. percent");
94 
95     set_param(h,
96               cb::mcbp::request::SetParamPayload::Type::Flush,
97               "max_size",
98               "1000000");
99 
100     checkeq(1000000, get_int_stat(h, "ep_max_size"), "Incorrect new size.");
101     check(epsilon(get_int_stat(h, "ep_mem_low_wat"), 750000),
102           "Incorrect larger low wat.");
103     check(epsilon(get_int_stat(h, "ep_mem_high_wat"), 850000),
104           "Incorrect larger high wat.");
105     checkeq(0.75f,
106             get_float_stat(h, "ep_mem_low_wat_percent"),
107             "Incorrect larger low wat. percent");
108     checkeq(0.85f,
109             get_float_stat(h, "ep_mem_high_wat_percent"),
110             "Incorrect larger high wat. percent");
111 
112     set_param(h,
113               cb::mcbp::request::SetParamPayload::Type::Flush,
114               "mem_low_wat",
115               "700000");
116     set_param(h,
117               cb::mcbp::request::SetParamPayload::Type::Flush,
118               "mem_high_wat",
119               "800000");
120 
121     checkeq(700000,
122             get_int_stat(h, "ep_mem_low_wat"),
123             "Incorrect even larger low wat.");
124     checkeq(800000,
125             get_int_stat(h, "ep_mem_high_wat"),
126             "Incorrect even larger high wat.");
127     checkeq(0.7f,
128             get_float_stat(h, "ep_mem_low_wat_percent"),
129             "Incorrect even larger low wat. percent");
130     checkeq(0.8f,
131             get_float_stat(h, "ep_mem_high_wat_percent"),
132             "Incorrect even larger high wat. percent");
133 
134     set_param(h,
135               cb::mcbp::request::SetParamPayload::Type::Flush,
136               "max_size",
137               "100");
138 
139     checkeq(100, get_int_stat(h, "ep_max_size"), "Incorrect smaller size.");
140     check(epsilon(get_int_stat(h, "ep_mem_low_wat"), 70),
141           "Incorrect smaller low wat.");
142     check(epsilon(get_int_stat(h, "ep_mem_high_wat"), 80),
143           "Incorrect smaller high wat.");
144     checkeq(0.7f,
145             get_float_stat(h, "ep_mem_low_wat_percent"),
146             "Incorrect smaller low wat. percent");
147     checkeq(0.8f,
148             get_float_stat(h, "ep_mem_high_wat_percent"),
149             "Incorrect smaller high wat. percent");
150 
151     set_param(h,
152               cb::mcbp::request::SetParamPayload::Type::Flush,
153               "mem_low_wat",
154               "50");
155     set_param(h,
156               cb::mcbp::request::SetParamPayload::Type::Flush,
157               "mem_high_wat",
158               "70");
159 
160     checkeq(50,
161             get_int_stat(h, "ep_mem_low_wat"),
162             "Incorrect even smaller low wat.");
163     checkeq(70,
164             get_int_stat(h, "ep_mem_high_wat"),
165             "Incorrect even smaller high wat.");
166     checkeq(0.5f, get_float_stat(h, "ep_mem_low_wat_percent"),
167             "Incorrect even smaller low wat. percent");
168     checkeq(0.7f, get_float_stat(h, "ep_mem_high_wat_percent"),
169             "Incorrect even smaller high wat. percent");
170 
171     testHarness->reload_engine(&h,
172                                testHarness->engine_path,
173                                testHarness->get_current_testcase()->cfg,
174                                true,
175                                true);
176 
177     wait_for_warmup_complete(h);
178 
179     checkeq(1000, get_int_stat(h, "ep_max_size"), "Incorrect initial size.");
180     check(epsilon(get_int_stat(h, "ep_mem_low_wat"), 750),
181           "Incorrect intial low wat.");
182     check(epsilon(get_int_stat(h, "ep_mem_high_wat"), 850),
183           "Incorrect initial high wat.");
184     checkeq(0.75f, get_float_stat(h, "ep_mem_low_wat_percent"),
185             "Incorrect initial low wat. percent");
186     checkeq(0.85f, get_float_stat(h, "ep_mem_high_wat_percent"),
187             "Incorrect initial high wat. percent");
188 
189     return SUCCESS;
190 }
191 
test_whitespace_db(EngineIface* h)192 static enum test_result test_whitespace_db(EngineIface* h) {
193     vals.clear();
194 
195     checkeq(ENGINE_SUCCESS,
196             get_stats(h, {}, {}, add_stats),
197             "Failed to get stats.");
198 
199     std::string dbname;
200     std::string policy;
201     policy = isPersistentBucket(h)
202                      ? vals.find("ep_item_eviction_policy")->second
203                      : "ephemeral";
204     dbname.assign(policy + std::string(WHITESPACE_DB));
205 
206     std::string oldparam("dbname=" + vals["ep_dbname"]);
207     std::string newparam("dbname=" + dbname);
208     std::string config = testHarness->get_current_testcase()->cfg;
209     std::string::size_type found = config.find(oldparam);
210     if (found != config.npos) {
211         config.replace(found, oldparam.size(), newparam);
212     }
213     testHarness->reload_engine(
214             &h, testHarness->engine_path, config.c_str(), true, false);
215     wait_for_warmup_complete(h);
216 
217     vals.clear();
218     checkeq(ENGINE_SUCCESS,
219             get_stats(h, {}, {}, add_stats),
220             "Failed to get stats.");
221 
222     if (vals["ep_dbname"] != dbname) {
223         std::cerr << "Expected dbname = '" << dbname << "'"
224                   << ", got '" << vals["ep_dbname"] << "'" << std::endl;
225         return FAIL;
226     }
227 
228     check(cb::io::isDirectory(dbname), "I expected the whitespace db to exist");
229     return SUCCESS;
230 }
231 
test_get_miss(EngineIface* h)232 static enum test_result test_get_miss(EngineIface* h) {
233     checkeq(ENGINE_KEY_ENOENT, verify_key(h, "k"), "Expected miss.");
234     return SUCCESS;
235 }
236 
test_set(EngineIface* h)237 static enum test_result test_set(EngineIface* h) {
238     item_info info;
239     uint64_t vb_uuid = 0, high_seqno = 0;
240     const int num_sets = 5, num_keys = 4;
241 
242     std::string key_arr[num_keys] = { "dummy_key",
243                                       "checkpoint_start",
244                                       "checkpoint_end",
245                                       "key" };
246 
247 
248     for (int k = 0; k < num_keys; k++) {
249         for (int j = 0; j < num_sets; j++) {
250             memset(&info, 0, sizeof(info));
251             vb_uuid = get_ull_stat(h, "vb_0:0:id", "failovers");
252             high_seqno = get_ull_stat(h, "vb_0:high_seqno", "vbucket-seqno");
253 
254             std::string err_str_store("Error setting " + key_arr[k]);
255             checkeq(ENGINE_SUCCESS,
256                     store(h,
257                           NULL,
258                           OPERATION_SET,
259                           key_arr[k].c_str(),
260                           "somevalue"),
261                     err_str_store.c_str());
262 
263             std::string err_str_get_item_info("Error getting " + key_arr[k]);
264             checkeq(true,
265                     get_item_info(h, &info, key_arr[k].c_str()),
266                     err_str_get_item_info.c_str());
267 
268             std::string err_str_vb_uuid("Expected valid vbucket uuid for " +
269                                         key_arr[k]);
270             checkeq(vb_uuid, info.vbucket_uuid, err_str_vb_uuid.c_str());
271 
272             std::string err_str_seqno("Expected valid sequence number for " +
273                                         key_arr[k]);
274             checkeq(high_seqno + 1, info.seqno, err_str_seqno.c_str());
275         }
276     }
277 
278     if (isPersistentBucket(h)) {
279         wait_for_flusher_to_settle(h);
280 
281         std::stringstream error1, error2;
282         error1 << "Expected ep_total_persisted >= num_keys (" << num_keys << ")";
283         error2 << "Expected ep_total_persisted <= num_sets*num_keys ("
284                << num_sets*num_keys << ")";
285 
286         // The flusher could of ran > 1 times. We can only assert
287         // that we persisted between num_keys and upto num_keys*num_sets
288         checkle(num_keys, get_int_stat(h, "ep_total_persisted"),
289                 error1.str().c_str());
290         checkge((num_sets * num_keys), get_int_stat(h, "ep_total_persisted"),
291                 error2.str().c_str());
292     }
293     return SUCCESS;
294 }
295 
296 extern "C" {
conc_del_set_thread(void *arg)297     static void conc_del_set_thread(void *arg) {
298         auto* h = static_cast<EngineIface*>(arg);
299 
300         for (int i = 0; i < 5000; ++i) {
301             store(h, NULL, OPERATION_ADD, "key", "somevalue");
302             checkeq(ENGINE_SUCCESS,
303                     store(h, nullptr, OPERATION_SET, "key", "somevalue"),
304                     "Error setting.");
305             // Ignoring the result here -- we're racing.
306             del(h, "key", 0, Vbid(0));
307         }
308     }
309 }
310 
test_conc_set(EngineIface* h)311 static enum test_result test_conc_set(EngineIface* h) {
312     const int n_threads = 8;
313     cb_thread_t threads[n_threads];
314 
315     wait_for_persisted_value(h, "key", "value1");
316 
317     for (int i = 0; i < n_threads; i++) {
318         int r = cb_create_thread(&threads[i], conc_del_set_thread, h, 0);
319         cb_assert(r == 0);
320     }
321 
322     for (int i = 0; i < n_threads; i++) {
323         int r = cb_join_thread(threads[i]);
324         cb_assert(r == 0);
325     }
326 
327     if (isWarmupEnabled(h)) {
328         wait_for_flusher_to_settle(h);
329 
330         testHarness->reload_engine(&h,
331                                    testHarness->engine_path,
332                                    testHarness->get_current_testcase()->cfg,
333                                    true,
334                                    false);
335         wait_for_warmup_complete(h);
336 
337         cb_assert(0 == get_int_stat(h, "ep_warmup_dups"));
338     }
339 
340     return SUCCESS;
341 }
342 
343 struct multi_set_args {
344     EngineIface* h;
345     std::string prefix;
346     int count;
347 };
348 
349 extern "C" {
multi_set_thread(void *arg)350     static void multi_set_thread(void *arg) {
351         struct multi_set_args *msa = static_cast<multi_set_args *>(arg);
352 
353         for (int i = 0; i < msa->count; i++) {
354             std::stringstream s;
355             s << msa->prefix << i;
356             std::string key(s.str());
357             checkeq(ENGINE_SUCCESS,
358                     store(msa->h,
359                           NULL,
360                           OPERATION_SET,
361                           key.c_str(),
362                           "somevalue"),
363                     "Set failure!");
364         }
365     }
366 }
367 
test_multi_set(EngineIface* h)368 static enum test_result test_multi_set(EngineIface* h) {
369     cb_thread_t thread1, thread2;
370     struct multi_set_args msa1, msa2;
371     msa1.h = h;
372     msa1.prefix = "ONE_";
373     msa1.count = 50000;
374     cb_assert(cb_create_thread(&thread1, multi_set_thread, &msa1, 0) == 0);
375 
376     msa2.h = h;
377     msa2.prefix = "TWO_";
378     msa2.count = 50000;
379     cb_assert(cb_create_thread(&thread2, multi_set_thread, &msa2, 0) == 0);
380 
381     cb_assert(cb_join_thread(thread1) == 0);
382     cb_assert(cb_join_thread(thread2) == 0);
383 
384     wait_for_flusher_to_settle(h);
385 
386     checkeq(100000,
387             get_int_stat(h, "curr_items"),
388             "Mismatch in number of items inserted");
389     checkeq(100000,
390             get_int_stat(h, "vb_0:high_seqno", "vbucket-seqno"),
391             "Unexpected high sequence number");
392 
393     return SUCCESS;
394 }
395 
test_set_get_hit(EngineIface* h)396 static enum test_result test_set_get_hit(EngineIface* h) {
397     checkeq(ENGINE_SUCCESS,
398             store(h, NULL, OPERATION_SET, "key", "somevalue"),
399             "store failure");
400     check_key_value(h, "key", "somevalue", 9);
401     return SUCCESS;
402 }
403 
test_getl_delete_with_cas(EngineIface* h)404 static enum test_result test_getl_delete_with_cas(EngineIface* h) {
405     checkeq(ENGINE_SUCCESS,
406             store(h, NULL, OPERATION_SET, "key", "value"),
407             "Failed to set key");
408 
409     auto ret = getl(h, nullptr, "key", Vbid(0), 15);
410     checkeq(cb::engine_errc::success,
411             ret.first,
412             "Expected getl to succeed on key");
413     item_info info;
414     check(h->get_item_info(ret.second.get(), &info), "Failed to get item info");
415 
416     checkeq(ENGINE_SUCCESS,
417             del(h, "key", info.cas, Vbid(0)),
418             "Expected SUCCESS");
419 
420     return SUCCESS;
421 }
422 
test_getl_delete_with_bad_cas(EngineIface* h)423 static enum test_result test_getl_delete_with_bad_cas(EngineIface* h) {
424     checkeq(ENGINE_SUCCESS,
425             store(h, NULL, OPERATION_SET, "key", "value"),
426             "Failed to set key");
427 
428     uint64_t cas = last_cas;
429     checkeq(cb::engine_errc::success,
430             getl(h, nullptr, "key", Vbid(0), 15).first,
431             "Expected getl to succeed on key");
432 
433     checkeq(ENGINE_LOCKED_TMPFAIL,
434             del(h, "key", cas, Vbid(0)),
435             "Expected TMPFAIL");
436 
437     return SUCCESS;
438 }
439 
test_getl_set_del_with_meta(EngineIface* h)440 static enum test_result test_getl_set_del_with_meta(EngineIface* h) {
441     const char *key = "key";
442     const char *val = "value";
443     const char *newval = "newvalue";
444     checkeq(ENGINE_SUCCESS,
445             store(h, NULL, OPERATION_SET, key, val),
446             "Failed to set key");
447 
448     checkeq(cb::engine_errc::success,
449             getl(h, nullptr, key, Vbid(0), 15).first,
450             "Expected getl to succeed on key");
451 
452     cb::EngineErrorMetadataPair errorMetaPair;
453     check(get_meta(h, key, errorMetaPair), "Expected to get meta");
454 
455     //init some random metadata
456     ItemMetaData itm_meta(0xdeadbeef, 10, 0xdeadbeef, time(NULL) + 300);
457 
458     //do a set with meta
459     checkeq(ENGINE_LOCKED,
460             set_with_meta(h,
461                           key,
462                           strlen(key),
463                           newval,
464                           strlen(newval),
465                           Vbid(0),
466                           &itm_meta,
467                           errorMetaPair.second.cas),
468             "Expected item to be locked");
469 
470     //do a del with meta
471     checkeq(ENGINE_LOCKED_TMPFAIL,
472             del_with_meta(h, key, strlen(key), Vbid(0), &itm_meta, last_cas),
473             "Expected item to be locked");
474     return SUCCESS;
475 }
476 
test_getl(EngineIface* h)477 static enum test_result test_getl(EngineIface* h) {
478     const char *key = "k1";
479     Vbid vbucketId = Vbid(0);
480     uint32_t expiration = 25;
481 
482     const void* cookie = testHarness->create_cookie();
483 
484     checkeq(cb::engine_errc::no_such_key,
485             getl(h, cookie, key, vbucketId, expiration).first,
486             "expected the key to be missing...");
487 
488     checkeq(ENGINE_SUCCESS,
489             store(h,
490                   cookie,
491                   OPERATION_SET,
492                   key,
493                   "{\"lock\":\"data\"}",
494                   nullptr,
495                   0,
496                   vbucketId,
497                   3600,
498                   PROTOCOL_BINARY_DATATYPE_JSON),
499             "Failed to store an item.");
500 
501     /* retry getl, should succeed */
502     auto ret = getl(h, cookie, key, vbucketId, expiration);
503     checkeq(cb::engine_errc::success,
504             ret.first,
505             "Expected to be able to getl on first try");
506 
507     item_info info;
508     check(h->get_item_info(ret.second.get(), &info), "Failed to get item info");
509 
510     checkeq(std::string{"{\"lock\":\"data\"}"},
511             std::string((const char*)info.value[0].iov_base,
512                         info.value[0].iov_len),
513             "Body was malformed.");
514     checkeq(static_cast<uint8_t>(PROTOCOL_BINARY_DATATYPE_JSON),
515             info.datatype,
516             "Expected datatype to be JSON");
517 
518     /* wait 16 seconds */
519     testHarness->time_travel(16);
520 
521     /* lock's taken so this should fail */
522     checkeq(cb::engine_errc::locked_tmpfail,
523             getl(h, cookie, key, vbucketId, expiration).first,
524             "Expected to fail getl on second try");
525 
526     checkne(ENGINE_SUCCESS,
527             store(h,
528                   cookie,
529                   OPERATION_SET,
530                   key,
531                   "lockdata2",
532                   nullptr,
533                   0,
534                   vbucketId),
535             "Should have failed to store an item.");
536 
537     /* wait another 10 seconds */
538     testHarness->time_travel(10);
539 
540     /* retry set, should succeed */
541     checkeq(ENGINE_SUCCESS,
542             store(h,
543                   cookie,
544                   OPERATION_SET,
545                   key,
546                   "lockdata",
547                   nullptr,
548                   0,
549                   vbucketId),
550             "Failed to store an item.");
551 
552     /* point to wrong vbucket, to test NOT_MY_VB response */
553     checkeq(cb::engine_errc::not_my_vbucket,
554             getl(h, cookie, key, Vbid(10), expiration).first,
555             "Should have received not my vbucket response");
556 
557     /* acquire lock, should succeed */
558     ret = getl(h, cookie, key, vbucketId, expiration);
559     checkeq(cb::engine_errc::success,
560             ret.first,
561             "Acquire lock should have succeeded");
562     check(h->get_item_info(ret.second.get(), &info), "Failed to get item info");
563     checkeq(static_cast<uint8_t>(PROTOCOL_BINARY_RAW_BYTES), info.datatype,
564             "Expected datatype to be RAW BYTES");
565 
566     /* try an delete operation which should fail */
567     uint64_t cas = 0;
568 
569     checkeq(ENGINE_LOCKED_TMPFAIL, del(h, key, 0, Vbid(0)), "Delete failed");
570 
571     /* bug MB 2699 append after getl should fail with ENGINE_TMPFAIL */
572 
573     testHarness->time_travel(26);
574 
575     char binaryData1[] = "abcdefg\0gfedcba";
576 
577     checkeq(cb::engine_errc::success,
578             storeCasVb11(h,
579                          cookie,
580                          OPERATION_SET,
581                          key,
582                          binaryData1,
583                          sizeof(binaryData1) - 1,
584                          82758,
585                          0,
586                          Vbid(0))
587                     .first,
588             "Failed set.");
589 
590     /* acquire lock, should succeed */
591     checkeq(cb::engine_errc::success,
592             getl(h, cookie, key, vbucketId, expiration).first,
593             "Acquire lock should have succeeded");
594 
595     /* bug MB 3252 & MB 3354.
596      * 1. Set a key with an expiry value.
597      * 2. Take a lock on the item before it expires
598      * 3. Wait for the item to expire
599      * 4. Perform a CAS operation, should fail
600      * 5. Perform a set operation, should succeed
601      */
602     const char *ekey = "test_expiry";
603     const char *edata = "some test data here.";
604 
605     ret = allocate(h,
606                    cookie,
607                    ekey,
608                    strlen(edata),
609                    0,
610                    2,
611                    PROTOCOL_BINARY_RAW_BYTES,
612                    Vbid(0));
613     checkeq(cb::engine_errc::success, ret.first, "Allocation Failed");
614 
615     check(h->get_item_info(ret.second.get(), &info), "Failed to get item info");
616 
617     memcpy(info.value[0].iov_base, edata, strlen(edata));
618 
619     checkeq(ENGINE_SUCCESS,
620             h->store(cookie,
621                      ret.second.get(),
622                      cas,
623                      OPERATION_SET,
624                      {},
625                      DocumentState::Alive),
626             "Failed to Store item");
627     check_key_value(h, ekey, edata, strlen(edata));
628 
629     testHarness->time_travel(3);
630     cas = last_cas;
631 
632     /* cas should fail */
633     ret = storeCasVb11(h,
634                        cookie,
635                        OPERATION_CAS,
636                        ekey,
637                        binaryData1,
638                        sizeof(binaryData1) - 1,
639                        82758,
640                        cas,
641                        Vbid(0));
642     checkne(cb::engine_errc::success, ret.first, "CAS succeeded.");
643 
644     /* but a simple store should succeed */
645     checkeq(ENGINE_SUCCESS,
646             store(h,
647                   cookie,
648                   OPERATION_SET,
649                   ekey,
650                   edata,
651                   nullptr,
652                   0,
653                   vbucketId),
654             "Failed to store an item.");
655 
656     testHarness->destroy_cookie(cookie);
657     return SUCCESS;
658 }
659 
test_unl(EngineIface* h)660 static enum test_result test_unl(EngineIface* h) {
661     const char *key = "k2";
662     Vbid vbucketId = Vbid(0);
663 
664     checkeq(ENGINE_SUCCESS,
665             get_stats(h, {}, {}, add_stats),
666             "Failed to get stats.");
667 
668     std::string eviction_policy;
669     auto itr = vals.find("ep_item_eviction_policy");
670     if (itr != vals.end()) {
671         eviction_policy = itr->second;
672     } else {
673         eviction_policy = "value_only";
674     }
675 
676     if (eviction_policy == "full_eviction") {
677         checkeq(ENGINE_TMPFAIL,
678                 unl(h, nullptr, key, vbucketId),
679                 "expected a TMPFAIL");
680     } else {
681         checkeq(ENGINE_KEY_ENOENT,
682                 unl(h, nullptr, key, vbucketId),
683                 "expected the key to be missing...");
684     }
685 
686     checkeq(ENGINE_SUCCESS,
687             store(h,
688                   NULL,
689                   OPERATION_SET,
690                   key,
691                   "lockdata",
692                   nullptr,
693                   0,
694                   vbucketId),
695             "Failed to store an item.");
696 
697     /* getl, should succeed */
698     auto ret = getl(h, nullptr, key, vbucketId, 0);
699     checkeq(cb::engine_errc::success,
700             ret.first,
701             "Expected to be able to getl on first try");
702     item_info info;
703     checkeq(true,
704             h->get_item_info(ret.second.get(), &info),
705             "failed to get item info");
706     uint64_t cas = info.cas;
707 
708     /* lock's taken unlocking with a random cas value should fail */
709     checkeq(ENGINE_LOCKED_TMPFAIL,
710             unl(h, nullptr, key, vbucketId),
711             "Expected to fail getl on second try");
712 
713     checkeq(ENGINE_SUCCESS,
714             unl(h, nullptr, key, vbucketId, cas),
715             "Expected to succed unl with correct cas");
716 
717     /* acquire lock, should succeed */
718     checkeq(cb::engine_errc::success,
719             getl(h, nullptr, key, vbucketId, 0).first,
720             "Lock should work after unlock");
721 
722     /* wait 16 seconds */
723     testHarness->time_travel(16);
724 
725     /* lock has expired, unl should fail */
726     checkeq(ENGINE_TMPFAIL,
727             unl(h, nullptr, key, vbucketId, last_cas),
728             "Expected to fail unl on lock timeout");
729 
730     return SUCCESS;
731 }
732 
test_unl_nmvb(EngineIface* h)733 static enum test_result test_unl_nmvb(EngineIface* h) {
734     const char *key = "k2";
735     Vbid vbucketId = Vbid(10);
736 
737     checkeq(ENGINE_NOT_MY_VBUCKET,
738             unl(h, nullptr, key, vbucketId),
739             "expected NOT_MY_VBUCKET to unlocking a key in a vbucket we don't "
740             "own");
741 
742     return SUCCESS;
743 }
744 
test_set_get_hit_bin(EngineIface* h)745 static enum test_result test_set_get_hit_bin(EngineIface* h) {
746     char binaryData[] = "abcdefg\0gfedcba";
747     cb_assert(sizeof(binaryData) != strlen(binaryData));
748 
749     checkeq(cb::engine_errc::success,
750             storeCasVb11(h,
751                          nullptr,
752                          OPERATION_SET,
753                          "key",
754                          binaryData,
755                          sizeof(binaryData),
756                          82758,
757                          0,
758                          Vbid(0))
759                     .first,
760             "Failed to set.");
761     check_key_value(h, "key", binaryData, sizeof(binaryData));
762     return SUCCESS;
763 }
764 
test_set_with_cas_non_existent(EngineIface* h)765 static enum test_result test_set_with_cas_non_existent(EngineIface* h) {
766     const char *key = "test_expiry_flush";
767     const auto* cookie = testHarness->create_cookie();
768     auto ret = allocate(
769             h, cookie, key, 10, 0, 0, PROTOCOL_BINARY_RAW_BYTES, Vbid(0));
770     checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
771 
772     Item* it = reinterpret_cast<Item*>(ret.second.get());
773     it->setCas(1234);
774 
775     uint64_t cas = 0;
776     checkeq(ENGINE_KEY_ENOENT,
777             h->store(cookie,
778                      ret.second.get(),
779                      cas,
780                      OPERATION_SET,
781                      {},
782                      DocumentState::Alive),
783             "Expected not found");
784 
785     testHarness->destroy_cookie(cookie);
786     return SUCCESS;
787 }
788 
test_set_change_flags(EngineIface* h)789 static enum test_result test_set_change_flags(EngineIface* h) {
790     checkeq(ENGINE_SUCCESS,
791             store(h, NULL, OPERATION_SET, "key", "somevalue"),
792             "Failed to set.");
793 
794     item_info info;
795     uint32_t flags = 828258;
796     check(get_item_info(h, &info, "key"), "Failed to get value.");
797     cb_assert(info.flags != flags);
798 
799     checkeq(cb::engine_errc::success,
800             storeCasVb11(h,
801                          nullptr,
802                          OPERATION_SET,
803                          "key",
804                          "newvalue",
805                          strlen("newvalue"),
806                          flags,
807                          0,
808                          Vbid(0))
809                     .first,
810             "Failed to set again.");
811 
812     check(get_item_info(h, &info, "key"), "Failed to get value.");
813 
814     return info.flags == flags ? SUCCESS : FAIL;
815 }
816 
test_add(EngineIface* h)817 static enum test_result test_add(EngineIface* h) {
818     item_info info;
819     uint64_t vb_uuid = 0;
820     uint64_t high_seqno = 0;
821 
822     memset(&info, 0, sizeof(info));
823 
824     vb_uuid = get_ull_stat(h, "vb_0:0:id", "failovers");
825     high_seqno = get_ull_stat(h, "vb_0:high_seqno", "vbucket-seqno");
826 
827     checkeq(ENGINE_SUCCESS,
828             store(h, NULL, OPERATION_ADD, "key", "somevalue"),
829             "Failed to add value.");
830 
831     check(get_item_info(h, &info, "key"), "Error getting item info");
832     checkeq(vb_uuid, info.vbucket_uuid, "Expected valid vbucket uuid");
833     checkeq(high_seqno + 1, info.seqno, "Expected valid sequence number");
834 
835     checkeq(ENGINE_NOT_STORED,
836             store(h, NULL, OPERATION_ADD, "key", "somevalue"),
837             "Failed to fail to re-add value.");
838 
839     // This aborts on failure.
840     check_key_value(h, "key", "somevalue", 9);
841 
842     // Expiration above was an hour, so let's go to The Future
843     testHarness->time_travel(3800);
844 
845     checkeq(ENGINE_SUCCESS,
846             store(h, NULL, OPERATION_ADD, "key", "newvalue"),
847             "Failed to add value again.");
848 
849     check_key_value(h, "key", "newvalue", 8);
850     return SUCCESS;
851 }
852 
test_add_add_with_cas(EngineIface* h)853 static enum test_result test_add_add_with_cas(EngineIface* h) {
854     item *i = NULL;
855     checkeq(ENGINE_SUCCESS,
856             store(h, NULL, OPERATION_ADD, "key", "somevalue", &i),
857             "Failed set.");
858     check_key_value(h, "key", "somevalue", 9);
859     item_info info;
860     check(h->get_item_info(i, &info), "Should be able to get info");
861 
862     checkeq(ENGINE_KEY_EEXISTS,
863             store(h,
864                   NULL,
865                   OPERATION_ADD,
866                   "key",
867                   "somevalue",
868                   nullptr,
869                   info.cas),
870             "Should not be able to add the key two times");
871 
872     h->release(i);
873     return SUCCESS;
874 }
875 
test_cas(EngineIface* h)876 static enum test_result test_cas(EngineIface* h) {
877     checkeq(ENGINE_SUCCESS,
878             store(h, NULL, OPERATION_SET, "key", "somevalue"),
879             "Failed to do initial set.");
880     checkne(ENGINE_SUCCESS, store(h, NULL, OPERATION_CAS, "key", "failcas"),
881             "Failed to fail initial CAS.");
882     check_key_value(h, "key", "somevalue", 9);
883 
884     auto ret = get(h, NULL, "key", Vbid(0));
885     checkeq(cb::engine_errc::success, ret.first, "Failed to get value.");
886 
887     item_info info;
888     check(h->get_item_info(ret.second.get(), &info),
889           "Failed to get item info.");
890 
891     checkeq(ENGINE_SUCCESS,
892             store(h,
893                   NULL,
894                   OPERATION_CAS,
895                   "key",
896                   "winCas",
897                   nullptr,
898                   info.cas),
899             "Failed to store CAS");
900     check_key_value(h, "key", "winCas", 6);
901 
902     uint64_t cval = 99999;
903     checkeq(ENGINE_KEY_ENOENT,
904             store(h,
905                   NULL,
906                   OPERATION_CAS,
907                   "non-existing",
908                   "winCas",
909                   nullptr,
910                   cval),
911             "CAS for non-existing key returned the wrong error code");
912     return SUCCESS;
913 }
914 
test_replace(EngineIface* h)915 static enum test_result test_replace(EngineIface* h) {
916     item_info info;
917     uint64_t vb_uuid = 0;
918     uint64_t high_seqno = 0;
919 
920     memset(&info, 0, sizeof(info));
921 
922     checkne(ENGINE_SUCCESS,
923             store(h, NULL, OPERATION_REPLACE, "key", "somevalue"),
924             "Failed to fail to replace non-existing value.");
925 
926     checkeq(ENGINE_SUCCESS,
927             store(h, NULL, OPERATION_SET, "key", "somevalue"),
928             "Failed to set value.");
929 
930     vb_uuid = get_ull_stat(h, "vb_0:0:id", "failovers");
931     high_seqno = get_ull_stat(h, "vb_0:high_seqno", "vbucket-seqno");
932 
933     checkeq(ENGINE_SUCCESS,
934             store(h, NULL, OPERATION_REPLACE, "key", "somevalue"),
935             "Failed to replace existing value.");
936 
937     check(get_item_info(h, &info, "key"), "Error getting item info");
938 
939     checkeq(vb_uuid, info.vbucket_uuid, "Expected valid vbucket uuid");
940     checkeq(high_seqno + 1, info.seqno, "Expected valid sequence number");
941 
942     check_key_value(h, "key", "somevalue", 9);
943     return SUCCESS;
944 }
945 
test_touch(EngineIface* h)946 static enum test_result test_touch(EngineIface* h) {
947     // Try to touch an unknown item...
948     checkeq(ENGINE_KEY_ENOENT,
949             touch(h, "mykey", Vbid(0), 0),
950             "Testing unknown key");
951 
952     // illegal vbucket
953     checkeq(ENGINE_NOT_MY_VBUCKET,
954             touch(h, "mykey", Vbid(5), 0),
955             "Testing illegal vbucket");
956 
957     // Store the item!
958     checkeq(ENGINE_SUCCESS,
959             store(h, NULL, OPERATION_SET, "mykey", "somevalue"),
960             "Failed set.");
961 
962     check_key_value(h, "mykey", "somevalue", strlen("somevalue"));
963 
964     cb::EngineErrorMetadataPair errorMetaPair;
965 
966     check(get_meta(h, "mykey", errorMetaPair), "Get meta failed");
967 
968     item_info currMeta = errorMetaPair.second;
969 
970     checkeq(ENGINE_SUCCESS,
971             touch(h, "mykey", Vbid(0), uint32_t(time(NULL) + 10)),
972             "touch mykey");
973     checkne(last_cas.load(),
974             currMeta.cas,
975             "touch should have returned an updated CAS");
976 
977     check(get_meta(h, "mykey", errorMetaPair), "Get meta failed");
978 
979     checkne(errorMetaPair.second.cas, currMeta.cas,
980           "touch should have updated the CAS");
981     checkne(errorMetaPair.second.exptime, currMeta.exptime,
982           "touch should have updated the expiry time");
983     checkeq(errorMetaPair.second.seqno, (currMeta.seqno + 1),
984           "touch should have incremented rev seqno");
985 
986     // time-travel 9 secs..
987     testHarness->time_travel(9);
988 
989     // The item should still exist
990     check_key_value(h, "mykey", "somevalue", 9);
991 
992     // time-travel 2 secs..
993     testHarness->time_travel(2);
994 
995     // The item should have expired now...
996     checkeq(cb::engine_errc::no_such_key,
997             get(h, NULL, "mykey", Vbid(0)).first,
998             "Item should be gone");
999     return SUCCESS;
1000 }
1001 
test_touch_mb7342(EngineIface* h)1002 static enum test_result test_touch_mb7342(EngineIface* h) {
1003     const char *key = "MB-7342";
1004     // Store the item!
1005     checkeq(ENGINE_SUCCESS,
1006             store(h, NULL, OPERATION_SET, key, "v"),
1007             "Failed set.");
1008 
1009     checkeq(ENGINE_SUCCESS, touch(h, key, Vbid(0), 0), "touch key");
1010 
1011     check_key_value(h, key, "v", 1);
1012 
1013     // Travel a loong time to see if the object is still there (the default
1014     // store sets an exp time of 3600
1015     testHarness->time_travel(3700);
1016 
1017     check_key_value(h, key, "v", 1);
1018 
1019     return SUCCESS;
1020 }
1021 
test_touch_mb10277(EngineIface* h)1022 static enum test_result test_touch_mb10277(EngineIface* h) {
1023     const char *key = "MB-10277";
1024     // Store the item!
1025     checkeq(ENGINE_SUCCESS,
1026             store(h, NULL, OPERATION_SET, key, "v"),
1027             "Failed set.");
1028     wait_for_flusher_to_settle(h);
1029     evict_key(h, key, Vbid(0), "Ejected.");
1030 
1031     checkeq(ENGINE_SUCCESS,
1032             touch(h,
1033                   key,
1034                   Vbid(0),
1035                   3600), // A new expiration time remains in the same.
1036             "touch key");
1037 
1038     return SUCCESS;
1039 }
1040 
test_gat(EngineIface* h)1041 static enum test_result test_gat(EngineIface* h) {
1042     // Try to gat an unknown item...
1043     auto ret = gat(h, "mykey", Vbid(0), 10);
1044     checkeq(ENGINE_KEY_ENOENT, ENGINE_ERROR_CODE(ret.first),
1045             "Testing unknown key");
1046 
1047     // illegal vbucket
1048     ret = gat(h, "mykey", Vbid(5), 10);
1049     checkeq(ENGINE_NOT_MY_VBUCKET,
1050             ENGINE_ERROR_CODE(ret.first), "Testing illegal vbucket");
1051 
1052     // Store the item!
1053     checkeq(ENGINE_SUCCESS,
1054             store(h,
1055                   NULL,
1056                   OPERATION_SET,
1057                   "mykey",
1058                   "{\"some\":\"value\"}",
1059                   nullptr,
1060                   0,
1061                   Vbid(0),
1062                   3600,
1063                   PROTOCOL_BINARY_DATATYPE_JSON),
1064             "Failed set.");
1065 
1066     check_key_value(
1067             h, "mykey", "{\"some\":\"value\"}", strlen("{\"some\":\"value\"}"));
1068 
1069     ret = gat(h, "mykey", Vbid(0), 10);
1070     checkeq(ENGINE_SUCCESS,
1071             ENGINE_ERROR_CODE(ret.first), "gat mykey");
1072 
1073     item_info info;
1074     check(h->get_item_info(ret.second.get(), &info),
1075           "Getting item info failed");
1076 
1077     checkeq(static_cast<uint8_t>(PROTOCOL_BINARY_DATATYPE_JSON),
1078             info.datatype, "Expected datatype to be JSON");
1079 
1080     std::string body{static_cast<char*>(info.value[0].iov_base),
1081     info.value[0].iov_len};
1082     check(body.compare(0, sizeof("{\"some\":\"value\"}"),
1083                        "{\"some\":\"value\"}") == 0,
1084           "Invalid data returned");
1085 
1086     // time-travel 9 secs..
1087     testHarness->time_travel(9);
1088 
1089     // The item should still exist
1090     check_key_value(
1091             h, "mykey", "{\"some\":\"value\"}", strlen("{\"some\":\"value\"}"));
1092 
1093     // time-travel 2 secs..
1094     testHarness->time_travel(2);
1095 
1096     // The item should have expired now...
1097     checkeq(cb::engine_errc::no_such_key,
1098             get(h, NULL, "mykey", Vbid(0)).first,
1099             "Item should be gone");
1100     return SUCCESS;
1101 }
1102 
test_gat_locked(EngineIface* h)1103 static enum test_result test_gat_locked(EngineIface* h) {
1104     checkeq(ENGINE_SUCCESS,
1105             store(h, NULL, OPERATION_SET, "key", "value"),
1106             "Failed to set key");
1107 
1108     checkeq(cb::engine_errc::success,
1109             getl(h, nullptr, "key", Vbid(0), 15).first,
1110             "Expected getl to succeed on key");
1111 
1112     auto ret = gat(h, "key", Vbid(0), 10);
1113     checkeq(ENGINE_LOCKED, ENGINE_ERROR_CODE(ret.first), "Expected LOCKED");
1114 
1115     testHarness->time_travel(16);
1116     ret = gat(h, "key", Vbid(0), 10);
1117     checkeq(ENGINE_SUCCESS, ENGINE_ERROR_CODE(ret.first), "Expected success");
1118 
1119     testHarness->time_travel(11);
1120     checkeq(cb::engine_errc::no_such_key,
1121             get(h, NULL, "key", Vbid(0)).first,
1122             "Expected value to be expired");
1123     return SUCCESS;
1124 }
1125 
test_touch_locked(EngineIface* h)1126 static enum test_result test_touch_locked(EngineIface* h) {
1127     item *itm = NULL;
1128     checkeq(ENGINE_SUCCESS,
1129             store(h, NULL, OPERATION_SET, "key", "value", &itm),
1130             "Failed to set key");
1131     h->release(itm);
1132 
1133     checkeq(cb::engine_errc::success,
1134             getl(h, nullptr, "key", Vbid(0), 15).first,
1135             "Expected getl to succeed on key");
1136 
1137     checkeq(ENGINE_LOCKED, touch(h, "key", Vbid(0), 10), "Expected tmp fail");
1138 
1139     testHarness->time_travel(16);
1140     checkeq(ENGINE_SUCCESS, touch(h, "key", Vbid(0), 10), "Expected success");
1141 
1142     testHarness->time_travel(11);
1143     checkeq(cb::engine_errc::no_such_key,
1144             get(h, NULL, "key", Vbid(0)).first,
1145             "Expected value to be expired");
1146 
1147     return SUCCESS;
1148 }
1149 
test_mb5215(EngineIface* h)1150 static enum test_result test_mb5215(EngineIface* h) {
1151     if (!isWarmupEnabled(h)) {
1152         return SKIPPED;
1153     }
1154 
1155     checkeq(ENGINE_SUCCESS,
1156             store(h, NULL, OPERATION_SET, "coolkey", "cooler"),
1157             "Failed set.");
1158 
1159     check_key_value(h, "coolkey", "cooler", strlen("cooler"));
1160 
1161     // set new exptime to 111
1162     int expTime = time(NULL) + 111;
1163 
1164     checkeq(ENGINE_SUCCESS,
1165             touch(h, "coolkey", Vbid(0), expTime),
1166             "touch coolkey");
1167 
1168     //reload engine
1169     testHarness->reload_engine(&h,
1170                                testHarness->engine_path,
1171                                testHarness->get_current_testcase()->cfg,
1172                                true,
1173                                false);
1174     wait_for_warmup_complete(h);
1175 
1176     //verify persisted expiration time
1177     const char *statkey = "key coolkey 0";
1178     int newExpTime;
1179     checkeq(cb::engine_errc::success,
1180             get(h, NULL, "coolkey", Vbid(0)).first,
1181             "Missing key");
1182     newExpTime = get_int_stat(h, "key_exptime", statkey);
1183     checkeq(expTime, newExpTime, "Failed to persist new exptime");
1184 
1185     // evict key, touch expiration time, and verify
1186     evict_key(h, "coolkey", Vbid(0), "Ejected.");
1187 
1188     expTime = time(NULL) + 222;
1189     checkeq(ENGINE_SUCCESS,
1190             touch(h, "coolkey", Vbid(0), expTime),
1191             "touch coolkey");
1192 
1193     testHarness->reload_engine(&h,
1194                                testHarness->engine_path,
1195                                testHarness->get_current_testcase()->cfg,
1196                                true,
1197                                false);
1198 
1199     wait_for_warmup_complete(h);
1200 
1201     checkeq(cb::engine_errc::success,
1202             get(h, NULL, "coolkey", Vbid(0)).first,
1203             "Missing key");
1204     newExpTime = get_int_stat(h, "key_exptime", statkey);
1205     checkeq(expTime, newExpTime, "Failed to persist new exptime");
1206 
1207     return SUCCESS;
1208 }
1209 
1210 /* Testing functionality to store a value for a deleted item
1211  * and also retrieve the value of a deleted item.
1212  * Need to check:
1213  *
1214  * - Each possible state transition between Alive, Deleted-with-value and
1215  *   Deleted-no-value.
1216  */
test_delete_with_value(EngineIface* h)1217 static enum test_result test_delete_with_value(EngineIface* h) {
1218     const uint64_t cas_0 = 0;
1219     const Vbid vbid = Vbid(0);
1220     const void* cookie = testHarness->create_cookie();
1221 
1222     // Store an initial (not-deleted) value.
1223     checkeq(ENGINE_SUCCESS,
1224             store(h, cookie, OPERATION_SET, "key", "somevalue"),
1225             "Failed set");
1226     wait_for_flusher_to_settle(h);
1227 
1228     checkeq(uint64_t(1),
1229             get_stat<uint64_t>(h, "vb_0:num_items", "vbucket-details 0"),
1230             "Unexpected initial item count");
1231 
1232     /* Alive -> Deleted-with-value */
1233     checkeq(ENGINE_SUCCESS,
1234             delete_with_value(h, cookie, cas_0, "key", "deleted"),
1235             "Failed Alive -> Delete-with-value");
1236 
1237     checkeq(uint64_t(0),
1238             get_stat<uint64_t>(h, "vb_0:num_items", "vbucket-details 0"),
1239             "Unexpected num_items after Alive -> Delete-with-value");
1240 
1241     auto res = get_value(h, cookie, "key", vbid, DocStateFilter::Alive);
1242     checkeq(ENGINE_KEY_ENOENT,
1243             res.first,
1244             "Unexpectedly accessed Deleted-with-value via DocState::Alive");
1245 
1246     res = get_value(h, cookie, "key", vbid, DocStateFilter::AliveOrDeleted);
1247     checkeq(ENGINE_SUCCESS,
1248             res.first,
1249             "Failed to fetch Alive -> Delete-with-value");
1250     checkeq(std::string("deleted"), res.second, "Unexpected value (deleted)");
1251 
1252     /* Deleted-with-value -> Deleted-with-value (different value). */
1253     checkeq(ENGINE_SUCCESS,
1254             delete_with_value(h, cookie, cas_0, "key", "deleted 2"),
1255             "Failed Deleted-with-value -> Deleted-with-value");
1256 
1257     checkeq(uint64_t(0),
1258             get_stat<uint64_t>(h, "vb_0:num_items", "vbucket-details 0"),
1259             "Unexpected num_items after Delete-with-value -> "
1260             "Delete-with-value");
1261 
1262     res = get_value(h, cookie, "key", vbid, DocStateFilter::AliveOrDeleted);
1263     checkeq(ENGINE_SUCCESS, res.first, "Failed to fetch key (deleted 2)");
1264     checkeq(std::string("deleted 2"),
1265             res.second,
1266             "Unexpected value (deleted 2)");
1267 
1268     /* Delete-with-value -> Alive */
1269     checkeq(ENGINE_SUCCESS,
1270             store(h, cookie, OPERATION_SET, "key", "alive 2", nullptr),
1271             "Failed Delete-with-value -> Alive");
1272     wait_for_flusher_to_settle(h);
1273 
1274     checkeq(uint64_t(1),
1275             get_stat<uint64_t>(h, "vb_0:num_items", "vbucket-details 0"),
1276             "Unexpected num_items after Delete-with-value -> Alive");
1277 
1278     res = get_value(h, cookie, "key", vbid, DocStateFilter::Alive);
1279     checkeq(ENGINE_SUCCESS,
1280             res.first,
1281             "Failed to fetch Delete-with-value -> Alive via DocState::Alive");
1282     checkeq(std::string("alive 2"), res.second, "Unexpected value (alive 2)");
1283 
1284     // Also check via DocState::Deleted
1285     res = get_value(h, cookie, "key", vbid, DocStateFilter::AliveOrDeleted);
1286     checkeq(ENGINE_SUCCESS,
1287             res.first,
1288             "Failed to fetch Delete-with-value -> Alive via DocState::Deleted");
1289     checkeq(std::string("alive 2"),
1290             res.second,
1291             "Unexpected value (alive 2) via DocState::Deleted");
1292 
1293     /* Alive -> Deleted-no-value */
1294     checkeq(ENGINE_SUCCESS,
1295             del(h, "key", cas_0, vbid, cookie),
1296             "Failed Alive -> Deleted-no-value");
1297     wait_for_flusher_to_settle(h);
1298 
1299     checkeq(uint64_t(0),
1300             get_stat<uint64_t>(h, "vb_0:num_items", "vbucket-details 0"),
1301             "Unexpected num_items after Alive -> Delete-no-value");
1302 
1303     res = get_value(h, cookie, "key", vbid, DocStateFilter::Alive);
1304     checkeq(ENGINE_KEY_ENOENT,
1305             res.first,
1306             "Unexpectedly accessed Deleted-no-value via DocState::Alive");
1307 
1308     /* Deleted-no-value -> Delete-with-value */
1309     checkeq(ENGINE_SUCCESS,
1310             delete_with_value(h, cookie, cas_0, "key", "deleted 3"),
1311             "Failed delete with value (deleted 2)");
1312 
1313     res = get_value(h, cookie, "key", vbid, DocStateFilter::AliveOrDeleted);
1314     checkeq(ENGINE_SUCCESS, res.first, "Failed to fetch key (deleted 3)");
1315     checkeq(std::string("deleted 3"),
1316             res.second,
1317             "Unexpected value (deleted 3)");
1318 
1319     testHarness->destroy_cookie(cookie);
1320 
1321     return SUCCESS;
1322 }
1323 
1324 /* Similar to test_delete_with_value, except also checks that CAS values
1325  */
test_delete_with_value_cas(EngineIface* h)1326 static enum test_result test_delete_with_value_cas(EngineIface* h) {
1327     checkeq(ENGINE_SUCCESS,
1328             store(h, nullptr, OPERATION_SET, "key1", "somevalue"),
1329             "Failed set");
1330 
1331     cb::EngineErrorMetadataPair errorMetaPair;
1332 
1333     check(get_meta(h, "key1", errorMetaPair), "Get meta failed");
1334 
1335     uint64_t curr_revseqno = errorMetaPair.second.seqno;
1336 
1337     /* Store a deleted item first with CAS 0 */
1338     checkeq(ENGINE_SUCCESS,
1339             store(h,
1340                   nullptr,
1341                   OPERATION_SET,
1342                   "key1",
1343                   "deletevalue",
1344                   nullptr,
1345                   0,
1346                   Vbid(0),
1347                   3600,
1348                   0x00,
1349                   DocumentState::Deleted),
1350             "Failed delete with value");
1351 
1352     check(get_meta(h, "key1", errorMetaPair), "Get meta failed");
1353 
1354     checkeq(errorMetaPair.second.seqno,
1355             curr_revseqno + 1,
1356             "rev seqno should have incremented");
1357 
1358     item *i = nullptr;
1359     checkeq(ENGINE_SUCCESS,
1360             store(h, nullptr, OPERATION_SET, "key2", "somevalue", &i),
1361             "Failed set");
1362 
1363     item_info info;
1364     check(h->get_item_info(i, &info), "Getting item info failed");
1365 
1366     h->release(i);
1367 
1368     check(get_meta(h, "key2", errorMetaPair), "Get meta failed");
1369 
1370     curr_revseqno = errorMetaPair.second.seqno;
1371 
1372     /* Store a deleted item with the existing CAS value */
1373     checkeq(ENGINE_SUCCESS,
1374             store(h,
1375                   nullptr,
1376                   OPERATION_SET,
1377                   "key2",
1378                   "deletevaluewithcas",
1379                   nullptr,
1380                   info.cas,
1381                   Vbid(0),
1382                   3600,
1383                   0x00,
1384                   DocumentState::Deleted),
1385             "Failed delete value with cas");
1386 
1387     wait_for_flusher_to_settle(h);
1388 
1389     check(get_meta(h, "key2", errorMetaPair), "Get meta failed");
1390 
1391     checkeq(errorMetaPair.second.seqno,
1392             curr_revseqno + 1,
1393             "rev seqno should have incremented");
1394 
1395     curr_revseqno = errorMetaPair.second.seqno;
1396 
1397     checkeq(ENGINE_SUCCESS,
1398             store(h,
1399                   nullptr,
1400                   OPERATION_SET,
1401                   "key2",
1402                   "newdeletevalue",
1403                   &i,
1404                   0,
1405                   Vbid(0),
1406                   3600,
1407                   0x00,
1408                   DocumentState::Deleted),
1409             "Failed delete value with cas");
1410 
1411     wait_for_flusher_to_settle(h);
1412 
1413     check(h->get_item_info(i, &info), "Getting item info failed");
1414     checkeq(int(DocumentState::Deleted),
1415             int(info.document_state),
1416             "Incorrect DocState for deleted item");
1417     checkne(uint64_t(0), info.cas, "Expected non-zero CAS for deleted item");
1418 
1419     h->release(i);
1420 
1421     check(get_meta(h, "key2", errorMetaPair), "Get meta failed");
1422 
1423     checkeq(errorMetaPair.second.seqno,
1424             curr_revseqno + 1,
1425             "rev seqno should have incremented");
1426 
1427     curr_revseqno = errorMetaPair.second.seqno;
1428 
1429     // Attempt to Delete-with-value using incorrect CAS (should fail)
1430     const uint64_t incorrect_CAS = info.cas + 1;
1431     checkeq(ENGINE_KEY_EEXISTS,
1432             store(h,
1433                   nullptr,
1434                   OPERATION_SET,
1435                   "key2",
1436                   "newdeletevaluewithcas",
1437                   nullptr,
1438                   incorrect_CAS,
1439                   Vbid(0),
1440                   3600,
1441                   0x00,
1442                   DocumentState::Deleted),
1443             "Expected KEY_EEXISTS with incorrect CAS");
1444 
1445     // Attempt with correct CAS.
1446     checkeq(ENGINE_SUCCESS,
1447             store(h,
1448                   nullptr,
1449                   OPERATION_SET,
1450                   "key2",
1451                   "newdeletevaluewithcas",
1452                   nullptr,
1453                   info.cas,
1454                   Vbid(0),
1455                   3600,
1456                   0x00,
1457                   DocumentState::Deleted),
1458             "Failed delete value with cas");
1459 
1460     wait_for_flusher_to_settle(h);
1461 
1462     auto ret = get(h, nullptr, "key2", Vbid(0), DocStateFilter::AliveOrDeleted);
1463     checkeq(cb::engine_errc::success, ret.first, "Failed to get value");
1464 
1465     check(get_meta(h, "key2", errorMetaPair), "Get meta failed");
1466 
1467     checkeq(errorMetaPair.second.seqno,
1468             curr_revseqno + 1,
1469             "rev seqno should have incremented");
1470 
1471     check(h->get_item_info(ret.second.get(), &info),
1472           "Getting item info failed");
1473     checkeq(int(DocumentState::Deleted),
1474             int(info.document_state),
1475             "Incorrect DocState for deleted item");
1476 
1477     checkeq(static_cast<uint8_t>(DocumentState::Deleted),
1478             static_cast<uint8_t>(info.document_state),
1479             "document must be in deleted state");
1480 
1481     std::string buf(static_cast<char*>(info.value[0].iov_base),
1482                     info.value[0].iov_len);
1483 
1484     checkeq(0, buf.compare("newdeletevaluewithcas"), "Data mismatch");
1485 
1486     ret = get(h, nullptr, "key", Vbid(0), DocStateFilter::Alive);
1487     checkeq(cb::engine_errc::no_such_key,
1488             ret.first,
1489             "Getting value should have failed");
1490 
1491     return SUCCESS;
1492 }
1493 
test_delete(EngineIface* h)1494 static enum test_result test_delete(EngineIface* h) {
1495     item *i = NULL;
1496     // First try to delete something we know to not be there.
1497     checkeq(ENGINE_KEY_ENOENT,
1498             del(h, "key", 0, Vbid(0)),
1499             "Failed to fail initial delete.");
1500     checkeq(ENGINE_SUCCESS,
1501             store(h, NULL, OPERATION_SET, "key", "somevalue", &i),
1502             "Failed set.");
1503     Item *it = reinterpret_cast<Item*>(i);
1504     uint64_t orig_cas = it->getCas();
1505     h->release(i);
1506     check_key_value(h, "key", "somevalue", 9);
1507 
1508     uint64_t cas = 0;
1509     uint64_t vb_uuid = 0;
1510     mutation_descr_t mut_info;
1511     uint64_t high_seqno = 0;
1512 
1513     memset(&mut_info, 0, sizeof(mut_info));
1514 
1515     vb_uuid = get_ull_stat(h, "vb_0:0:id", "failovers");
1516     high_seqno = get_ull_stat(h, "vb_0:high_seqno", "vbucket-seqno");
1517     checkeq(ENGINE_SUCCESS,
1518             del(h, "key", &cas, Vbid(0), nullptr, &mut_info),
1519             "Failed remove with value.");
1520     checkne(orig_cas, cas, "Expected CAS to be updated on delete");
1521     checkeq(ENGINE_KEY_ENOENT, verify_key(h, "key"), "Expected missing key");
1522     checkeq(vb_uuid, mut_info.vbucket_uuid, "Expected valid vbucket uuid");
1523     checkeq(high_seqno + 1, mut_info.seqno, "Expected valid sequence number");
1524 
1525     // Can I time travel to an expired object and delete it?
1526     checkeq(ENGINE_SUCCESS,
1527             store(h, NULL, OPERATION_SET, "key", "somevalue", &i),
1528             "Failed set.");
1529     h->release(i);
1530     testHarness->time_travel(3617);
1531     checkeq(ENGINE_KEY_ENOENT,
1532             del(h, "key", 0, Vbid(0)),
1533             "Did not get ENOENT removing an expired object.");
1534     checkeq(ENGINE_KEY_ENOENT, verify_key(h, "key"), "Expected missing key");
1535 
1536     return SUCCESS;
1537 }
1538 
test_set_delete(EngineIface* h)1539 static enum test_result test_set_delete(EngineIface* h) {
1540     checkeq(ENGINE_SUCCESS,
1541             store(h, NULL, OPERATION_SET, "key", "somevalue"),
1542             "Failed set.");
1543     check_key_value(h, "key", "somevalue", 9);
1544     checkeq(ENGINE_SUCCESS,
1545             del(h, "key", 0, Vbid(0)),
1546             "Failed remove with value.");
1547     checkeq(ENGINE_KEY_ENOENT, verify_key(h, "key"), "Expected missing key");
1548     wait_for_flusher_to_settle(h);
1549     wait_for_stat_to_be(h, "curr_items", 0);
1550     return SUCCESS;
1551 }
1552 
test_set_delete_invalid_cas(EngineIface* h)1553 static enum test_result test_set_delete_invalid_cas(EngineIface* h) {
1554     item *i = NULL;
1555     checkeq(ENGINE_SUCCESS,
1556             store(h, NULL, OPERATION_SET, "key", "somevalue", &i),
1557             "Failed set.");
1558     check_key_value(h, "key", "somevalue", 9);
1559     item_info info;
1560     check(h->get_item_info(i, &info), "Should be able to get info");
1561     h->release(i);
1562 
1563     checkeq(ENGINE_KEY_EEXISTS,
1564             del(h, "key", info.cas + 1, Vbid(0)),
1565             "Didn't expect to be able to remove the item with wrong cas");
1566 
1567     checkeq(ENGINE_SUCCESS,
1568             del(h, "key", info.cas, Vbid(0)),
1569             "Subsequent delete with correct CAS did not succeed");
1570 
1571     return SUCCESS;
1572 }
1573 
test_delete_set(EngineIface* h)1574 static enum test_result test_delete_set(EngineIface* h) {
1575     if (!isWarmupEnabled(h)) {
1576         return SKIPPED;
1577     }
1578 
1579     wait_for_persisted_value(h, "key", "value1");
1580 
1581     checkeq(ENGINE_SUCCESS,
1582             del(h, "key", 0, Vbid(0)),
1583             "Failed remove with value.");
1584 
1585     wait_for_persisted_value(h, "key", "value2");
1586 
1587     testHarness->reload_engine(&h,
1588                                testHarness->engine_path,
1589                                testHarness->get_current_testcase()->cfg,
1590                                true,
1591                                false);
1592 
1593     wait_for_warmup_complete(h);
1594 
1595     check_key_value(h, "key", "value2", 6);
1596     checkeq(ENGINE_SUCCESS,
1597             del(h, "key", 0, Vbid(0)),
1598             "Failed remove with value.");
1599     wait_for_flusher_to_settle(h);
1600 
1601     testHarness->reload_engine(&h,
1602                                testHarness->engine_path,
1603                                testHarness->get_current_testcase()->cfg,
1604                                true,
1605                                false);
1606 
1607     wait_for_warmup_complete(h);
1608 
1609     checkeq(ENGINE_KEY_ENOENT, verify_key(h, "key"), "Expected missing key");
1610 
1611     return SUCCESS;
1612 }
1613 
test_get_delete_missing_file(EngineIface* h)1614 static enum test_result test_get_delete_missing_file(EngineIface* h) {
1615     checkeq(ENGINE_SUCCESS,
1616             get_stats(h, {}, {}, add_stats),
1617             "Failed to get stats.");
1618 
1619     const char *key = "key";
1620     wait_for_persisted_value(h, key, "value2delete");
1621 
1622     // Make the couchstore files in the db directory totally inaccessible.
1623     std::string dbname = vals["ep_dbname"];
1624     CouchstoreFileAccessGuard makeCouchstoreFileInaccessible(
1625             dbname, CouchstoreFileAccessGuard::Mode::DenyAll);
1626 
1627     auto ret = get(h, NULL, key, Vbid(0));
1628 
1629     // ep engine must be unaware of well-being of the db file as long as
1630     // the item is still in the memory
1631     checkeq(cb::engine_errc::success, ret.first, "Expected success for get");
1632 
1633     evict_key(h, key);
1634     ret = get(h, NULL, key, Vbid(0));
1635 
1636     // ep engine must be now aware of the ill-fated db file where
1637     // the item is supposedly stored
1638     checkeq(cb::engine_errc::temporary_failure,
1639             ret.first,
1640             "Expected tmp fail for get");
1641 
1642     return SUCCESS;
1643 }
1644 
test_bug7023(EngineIface* h)1645 static enum test_result test_bug7023(EngineIface* h) {
1646     std::vector<std::string> keys;
1647     // Make a vbucket mess.
1648     const int nitems = 10000;
1649     const int iterations = 5;
1650     for (int j = 0; j < nitems; ++j) {
1651         keys.push_back("key" + std::to_string(j));
1652     }
1653 
1654     std::vector<std::string>::iterator it;
1655     for (int j = 0; j < iterations; ++j) {
1656         check(set_vbucket_state(h, Vbid(0), vbucket_state_dead),
1657               "Failed set set vbucket 0 dead.");
1658         checkeq(ENGINE_SUCCESS, vbucketDelete(h, Vbid(0)), "expected success");
1659         checkeq(cb::mcbp::Status::Success,
1660                 last_status.load(),
1661                 "Expected vbucket deletion to work.");
1662         check(set_vbucket_state(h, Vbid(0), vbucket_state_active),
1663               "Failed set set vbucket 0 active.");
1664         for (it = keys.begin(); it != keys.end(); ++it) {
1665             checkeq(ENGINE_SUCCESS,
1666                     store(h, NULL, OPERATION_SET, it->c_str(), it->c_str()),
1667                     "Failed to store a value");
1668         }
1669     }
1670     wait_for_flusher_to_settle(h);
1671 
1672     if (isWarmupEnabled(h)) {
1673         // Restart again, to verify no data loss.
1674         testHarness->reload_engine(&h,
1675                                    testHarness->engine_path,
1676                                    testHarness->get_current_testcase()->cfg,
1677                                    true,
1678                                    false);
1679 
1680         wait_for_warmup_complete(h);
1681         checkeq(nitems,
1682                 get_int_stat(h, "ep_warmup_value_count", "warmup"),
1683                 "Incorrect items following warmup");
1684     }
1685     return SUCCESS;
1686 }
1687 
test_mb3169(EngineIface* h)1688 static enum test_result test_mb3169(EngineIface* h) {
1689     checkeq(ENGINE_SUCCESS,
1690             store(h, NULL, OPERATION_SET, "set", "value"),
1691             "Failed to store a value");
1692     checkeq(ENGINE_SUCCESS,
1693             store(h, NULL, OPERATION_SET, "delete", "0"),
1694             "Failed to store a value");
1695     checkeq(ENGINE_SUCCESS,
1696             store(h, NULL, OPERATION_SET, "get", "getvalue"),
1697             "Failed to store a value");
1698 
1699     wait_for_stat_to_be(h, "ep_total_persisted", 3);
1700 
1701     evict_key(h, "set", Vbid(0), "Ejected.");
1702     evict_key(h, "delete", Vbid(0), "Ejected.");
1703     evict_key(h, "get", Vbid(0), "Ejected.");
1704 
1705     checkeq(3, get_int_stat(h, "curr_items"), "Expected 3 items");
1706     checkeq(3,
1707             get_int_stat(h, "ep_num_non_resident"),
1708             "Expected all items to be resident");
1709 
1710     checkeq(ENGINE_SUCCESS,
1711             store(h, NULL, OPERATION_SET, "set", "value2"),
1712             "Failed to store a value");
1713     wait_for_flusher_to_settle(h);
1714 
1715     checkeq(3, get_int_stat(h, "curr_items"), "Expected 3 items");
1716     checkeq(2,
1717             get_int_stat(h, "ep_num_non_resident"),
1718             "Expected mutation to mark item resident");
1719 
1720     checkeq(ENGINE_SUCCESS, del(h, "delete", 0, Vbid(0)), "Delete failed");
1721 
1722     wait_for_flusher_to_settle(h);
1723 
1724     checkeq(2, get_int_stat(h, "curr_items"), "Expected 2 items after del");
1725     checkeq(1,
1726             get_int_stat(h, "ep_num_non_resident"),
1727             "Expected delete to remove non-resident item");
1728 
1729     check_key_value(h, "get", "getvalue", 8);
1730 
1731     checkeq(2, get_int_stat(h, "curr_items"), "Expected 2 items after get");
1732     checkeq(0,
1733             get_int_stat(h, "ep_num_non_resident"),
1734             "Expected all items to be resident");
1735     return SUCCESS;
1736 }
1737 
test_mb5172(EngineIface* h)1738 static enum test_result test_mb5172(EngineIface* h) {
1739     if (!isWarmupEnabled(h)) {
1740         return SKIPPED;
1741     }
1742 
1743     checkeq(ENGINE_SUCCESS,
1744             store(h, NULL, OPERATION_SET, "key-1", "value-1"),
1745             "Failed to store a value");
1746 
1747     checkeq(ENGINE_SUCCESS,
1748             store(h, NULL, OPERATION_SET, "key-2", "value-2"),
1749             "Failed to store a value");
1750 
1751     wait_for_flusher_to_settle(h);
1752 
1753     checkeq(0,
1754             get_int_stat(h, "ep_num_non_resident"),
1755             "Expected all items to be resident");
1756 
1757     // restart the server.
1758     testHarness->reload_engine(&h,
1759                                testHarness->engine_path,
1760                                testHarness->get_current_testcase()->cfg,
1761                                true,
1762                                false);
1763 
1764     wait_for_warmup_complete(h);
1765     checkeq(0,
1766             get_int_stat(h, "ep_num_non_resident"),
1767             "Expected all items to be resident");
1768     return SUCCESS;
1769 }
1770 
test_set_vbucket_out_of_range(EngineIface* h)1771 static enum test_result test_set_vbucket_out_of_range(EngineIface* h) {
1772     check(!set_vbucket_state(h, Vbid(10000), vbucket_state_active),
1773           "Shouldn't have been able to set vbucket 10000");
1774     return SUCCESS;
1775 }
1776 
set_max_cas_mb21190(EngineIface* h)1777 static enum test_result set_max_cas_mb21190(EngineIface* h) {
1778     uint64_t max_cas = get_ull_stat(h, "vb_0:max_cas", "vbucket-details 0");
1779     std::string max_cas_str = std::to_string(max_cas+1);
1780     set_param(h,
1781               cb::mcbp::request::SetParamPayload::Type::Vbucket,
1782               "max_cas",
1783               max_cas_str.data(),
1784               Vbid(0));
1785     checkeq(cb::mcbp::Status::Success, last_status.load(),
1786             "Failed to set_param max_cas");
1787     checkeq(max_cas + 1,
1788             get_ull_stat(h, "vb_0:max_cas", "vbucket-details 0"),
1789             "max_cas didn't change");
1790     set_param(h,
1791               cb::mcbp::request::SetParamPayload::Type::Vbucket,
1792               "max_cas",
1793               max_cas_str.data(),
1794               Vbid(1));
1795     checkeq(cb::mcbp::Status::NotMyVbucket, last_status.load(),
1796             "Expected not my vbucket for vb 1");
1797     set_param(h,
1798               cb::mcbp::request::SetParamPayload::Type::Vbucket,
1799               "max_cas",
1800               "JUNK",
1801               Vbid(0));
1802     checkeq(cb::mcbp::Status::Einval, last_status.load(),
1803             "Expected EINVAL");
1804     return SUCCESS;
1805 }
1806 
warmup_mb21769(EngineIface* h)1807 static enum test_result warmup_mb21769(EngineIface* h) {
1808     if (!isWarmupEnabled(h)) {
1809         return SKIPPED;
1810     }
1811 
1812     // Validate some VB data post warmup
1813     // VB 0 will be empty
1814     // VB 1 will not be empty
1815     // VB 2 will not be empty and will have had set_state as the final ops
1816 
1817     check(set_vbucket_state(h, Vbid(1), vbucket_state_active),
1818           "Failed to set vbucket state for vb1");
1819     check(set_vbucket_state(h, Vbid(2), vbucket_state_active),
1820           "Failed to set vbucket state for vb2");
1821 
1822     const int num_items = 10;
1823     write_items(h, num_items, 0, "vb1", "value", 0 /*expiry*/, Vbid(1));
1824     write_items(h, num_items, 0, "vb2", "value", 0 /*expiry*/, Vbid(2));
1825     wait_for_flusher_to_settle(h);
1826 
1827     // flip replica to active to drive more _local writes
1828     check(set_vbucket_state(h, Vbid(2), vbucket_state_replica),
1829           "Failed to set vbucket state (replica) for vb2");
1830     wait_for_flusher_to_settle(h);
1831 
1832     check(set_vbucket_state(h, Vbid(2), vbucket_state_active),
1833           "Failed to set vbucket state (replica) for vb2");
1834     wait_for_flusher_to_settle(h);
1835 
1836     // Force a shutdown so the warmup will create failover entries
1837     testHarness->reload_engine(&h,
1838                                testHarness->engine_path,
1839                                testHarness->get_current_testcase()->cfg,
1840                                true,
1841                                true);
1842 
1843     wait_for_warmup_complete(h);
1844 
1845     // values of interested stats for each VB
1846     std::array<uint64_t, 3> high_seqnos = {{0, num_items, num_items}};
1847     std::array<uint64_t, 3> snap_starts = {{0, num_items, num_items}};
1848     std::array<uint64_t, 3> snap_ends = {{0, num_items, num_items}};
1849     // we will check the seqno of the 0th entry of each vbucket's failover table
1850     std::array<uint64_t, 3> failover_entry0 = {{0, num_items, num_items}};
1851 
1852     for (uint64_t vb = 0; vb <= 2; vb++) {
1853         std::string vb_prefix = "vb_" + std::to_string(vb) + ":";
1854         std::string high_seqno = vb_prefix + "high_seqno";
1855         std::string snap_start = vb_prefix + "last_persisted_snap_start";
1856         std::string snap_end = vb_prefix + "last_persisted_snap_end";
1857         std::string fail0 = vb_prefix + "0:seq";
1858         std::string vb_group_key = "vbucket-seqno " + std::to_string(vb);
1859         std::string failovers_key = "failovers " + std::to_string(vb);
1860 
1861         checkeq(high_seqnos[vb],
1862                 get_ull_stat(h, high_seqno.c_str(), vb_group_key.c_str()),
1863                 std::string("high_seqno incorrect vb:" + std::to_string(vb))
1864                         .c_str());
1865         checkeq(snap_starts[vb],
1866                 get_ull_stat(h, snap_start.c_str(), vb_group_key.c_str()),
1867                 std::string("snap_start incorrect vb:" + std::to_string(vb))
1868                         .c_str());
1869         checkeq(snap_ends[vb],
1870                 get_ull_stat(h, snap_end.c_str(), vb_group_key.c_str()),
1871                 std::string("snap_end incorrect vb:" + std::to_string(vb))
1872                         .c_str());
1873         auto failoverTable = get_all_stats(h, failovers_key.c_str());
1874         if (failoverTable[fail0] != std::to_string(failover_entry0[vb])) {
1875             std::cerr << "failover table entry 0 is incorrect for vb:" << vb
1876                       << " expected:" << failover_entry0[vb]
1877                       << " got:" << failoverTable[fail0]
1878                       << " dumping failover table\n";
1879             for (const auto& stat : failoverTable) {
1880                 std::cerr << stat.first << ":" << stat.second << std::endl;
1881             }
1882             std::string detail = "vbucket-details " + std::to_string(vb);
1883             auto details = get_all_stats(h, detail.c_str());
1884             std::cerr << detail << std::endl;
1885             for (const auto& stat : details) {
1886                 std::cerr << stat.first << ":" << stat.second << std::endl;
1887             }
1888             return FAIL;
1889         }
1890     }
1891 
1892     return SUCCESS;
1893 }
1894 
1895 /**
1896  * Callback from the document API being called after the CAS was assigned
1897  * to the object. We're allowed to modify the content, so let's just change
1898  * the string.
1899  *
1900  * @param info info about the document
1901  */
1902 static uint64_t pre_link_seqno(0);
pre_link_doc_callback(item_info& info)1903 static void pre_link_doc_callback(item_info& info) {
1904     checkne(uint64_t(0), info.cas, "CAS value should be set");
1905     // mock the actual value so we can see it was changed
1906     memcpy(info.value[0].iov_base, "valuesome", 9);
1907     pre_link_seqno = info.seqno;
1908 }
1909 
1910 /**
1911  * Verify that we've hooked into the checkpoint and that the pre-link
1912  * document api method is called.
1913  */
pre_link_document(EngineIface* h)1914 static test_result pre_link_document(EngineIface* h) {
1915     item_info info;
1916 
1917     PreLinkFunction function = pre_link_doc_callback;
1918     testHarness->set_pre_link_function(function);
1919     checkeq(ENGINE_SUCCESS,
1920             store(h, nullptr, OPERATION_SET, "key", "somevalue"),
1921             "Failed set.");
1922     testHarness->set_pre_link_function({});
1923 
1924     // Fetch the value and verify that the callback was called!
1925     auto ret = get(h, nullptr, "key", Vbid(0));
1926     checkeq(cb::engine_errc::success, ret.first, "get failed");
1927     check(h->get_item_info(ret.second.get(), &info),
1928           "Failed to get item info.");
1929     checkeq(0, memcmp(info.value[0].iov_base, "valuesome", 9),
1930            "Expected value to be modified");
1931     checkeq(pre_link_seqno, info.seqno, "Sequence numbers should match");
1932 
1933     return SUCCESS;
1934 }
1935 
1936 /**
1937  * verify that get_if works as expected
1938  */
get_if(EngineIface* h)1939 static test_result get_if(EngineIface* h) {
1940     const std::string key("get_if");
1941 
1942     checkeq(ENGINE_SUCCESS,
1943             store(h, nullptr, OPERATION_SET, key.c_str(), "somevalue"),
1944             "Failed set.");
1945 
1946     if (isPersistentBucket(h)) {
1947         wait_for_flusher_to_settle(h);
1948         evict_key(h, key.c_str(), Vbid(0), "Ejected.");
1949     }
1950 
1951     const auto* cookie = testHarness->create_cookie();
1952     auto doc = h->get_if(cookie,
1953                          DocKey(key, DocKeyEncodesCollectionId::No),
1954                          Vbid(0),
1955                          [](const item_info&) { return true; });
1956     check(doc.second, "document should be found");
1957 
1958     doc = h->get_if(cookie,
1959                     DocKey(key, DocKeyEncodesCollectionId::No),
1960                     Vbid(0),
1961                     [](const item_info&) { return false; });
1962     check(!doc.second, "document should not be found");
1963 
1964     doc = h->get_if(cookie,
1965                     DocKey("no", DocKeyEncodesCollectionId::No),
1966                     Vbid(0),
1967                     [](const item_info&) { return true; });
1968     check(!doc.second, "non-existing document should not be found");
1969 
1970     checkeq(ENGINE_SUCCESS,
1971             del(h, key.c_str(), 0, Vbid(0)),
1972             "Failed remove with value");
1973 
1974     doc = h->get_if(cookie,
1975                     DocKey(key, DocKeyEncodesCollectionId::No),
1976                     Vbid(0),
1977                     [](const item_info&) { return true; });
1978     check(!doc.second, "deleted document should not be found");
1979 
1980     testHarness->destroy_cookie(cookie);
1981 
1982     return SUCCESS;
1983 }
1984 
max_ttl_out_of_range(EngineIface* h)1985 static test_result max_ttl_out_of_range(EngineIface* h) {
1986     // Test absolute first as this is the bigger time travel
1987     check(!set_param(h,
1988                      cb::mcbp::request::SetParamPayload::Type::Flush,
1989                      "max_ttl",
1990                      "-1"),
1991           "Should not be allowed to set a negative value");
1992     check(!set_param(h,
1993                      cb::mcbp::request::SetParamPayload::Type::Flush,
1994                      "max_ttl",
1995                      "2147483648"),
1996           "Should not be allowed to set > int32::max");
1997 
1998     return SUCCESS;
1999 }
2000 
max_ttl(EngineIface* h)2001 static test_result max_ttl(EngineIface* h) {
2002     // Make limit be greater than 30 days in seconds so that ep-engine must
2003     // create a absolute expiry time internally.
2004     const int absoluteExpiry = (60 * 60 * 24 * 31);
2005     auto absoluteExpiryStr = std::to_string(absoluteExpiry);
2006 
2007     const int relativeExpiry = 100;
2008     auto relativeExpiryStr = std::to_string(relativeExpiry);
2009 
2010     checkeq(0, get_int_stat(h, "ep_max_ttl"), "max_ttl should be 0");
2011 
2012     // Test absolute first as this is the bigger time travel
2013     check(set_param(h,
2014                     cb::mcbp::request::SetParamPayload::Type::Flush,
2015                     "max_ttl",
2016                     absoluteExpiryStr.c_str()),
2017           "Failed to set max_ttl");
2018     checkeq(absoluteExpiry,
2019             get_int_stat(h, "ep_max_ttl"),
2020             "max_ttl didn't change");
2021 
2022     // Store will set 0 expiry, which results in 100 seconds of ttl
2023     checkeq(ENGINE_SUCCESS,
2024             store(h,
2025                   nullptr,
2026                   OPERATION_SET,
2027                   "key-abs",
2028                   "somevalue",
2029                   nullptr,
2030                   0,
2031                   Vbid(0),
2032                   0 /*exp*/),
2033             "Failed set.");
2034 
2035     cb::EngineErrorMetadataPair errorMetaPair;
2036     check(get_meta(h, "key-abs", errorMetaPair), "Get meta failed");
2037     checkne(time_t(0),
2038             errorMetaPair.second.exptime,
2039             "expiry should not be zero");
2040 
2041     // Force expiry
2042     testHarness->time_travel(absoluteExpiry + 1);
2043 
2044     auto ret = get(h, NULL, "key-abs", Vbid(0));
2045     checkeq(cb::engine_errc::no_such_key,
2046             ret.first,
2047             "Failed, expected no_such_key.");
2048 
2049     check(set_param(h,
2050                     cb::mcbp::request::SetParamPayload::Type::Flush,
2051                     "max_ttl",
2052                     relativeExpiryStr.c_str()),
2053           "Failed to set max_ttl");
2054     checkeq(relativeExpiry,
2055             get_int_stat(h, "ep_max_ttl"),
2056             "max_ttl didn't change");
2057 
2058     // Store will set 0 expiry, which results in 100 seconds of ttl
2059     checkeq(ENGINE_SUCCESS,
2060             store(h,
2061                   nullptr,
2062                   OPERATION_SET,
2063                   "key-rel",
2064                   "somevalue",
2065                   nullptr,
2066                   0,
2067                   Vbid(0),
2068                   0 /*exp*/),
2069             "Failed set.");
2070 
2071     check(get_meta(h, "key-rel", errorMetaPair), "Get meta failed");
2072     checkne(time_t(0),
2073             errorMetaPair.second.exptime,
2074             "expiry should not be zero");
2075 
2076     // Force expiry
2077     testHarness->time_travel(relativeExpiry + 1);
2078 
2079     ret = get(h, NULL, "key-rel", Vbid(0));
2080     checkeq(cb::engine_errc::no_such_key,
2081             ret.first,
2082             "Failed, expected no_such_key.");
2083 
2084     return SUCCESS;
2085 }
2086 
max_ttl_setWithMeta(EngineIface* h)2087 static test_result max_ttl_setWithMeta(EngineIface* h) {
2088     // Make limit be greater than 30 days in seconds so that ep-engine must
2089     // create a absolute expiry time internally.
2090     const int absoluteExpiry = (60 * 60 * 24 * 31);
2091     auto absoluteExpiryStr = std::to_string(absoluteExpiry);
2092     std::string keyAbs = "key-abs";
2093 
2094     const int relativeExpiry = 100;
2095     auto relativeExpiryStr = std::to_string(relativeExpiry);
2096     std::string keyRel = "key-rel";
2097 
2098     checkeq(0, get_int_stat(h, "ep_max_ttl"), "max_ttl should be 0");
2099 
2100     // Test absolute first as this is the bigger time travel
2101     check(set_param(h,
2102                     cb::mcbp::request::SetParamPayload::Type::Flush,
2103                     "max_ttl",
2104                     absoluteExpiryStr.c_str()),
2105           "Failed to set max_ttl");
2106     checkeq(absoluteExpiry,
2107             get_int_stat(h, "ep_max_ttl"),
2108             "max_ttl didn't change");
2109 
2110     // SWM with 0 expiry which results in an expiry being set
2111     ItemMetaData itemMeta(0xdeadbeef, 10, 0xf1a95, 0 /*expiry*/);
2112     checkeq(ENGINE_SUCCESS,
2113             set_with_meta(h,
2114                           keyAbs.c_str(),
2115                           keyAbs.size(),
2116                           keyAbs.c_str(),
2117                           keyAbs.size(),
2118                           Vbid(0),
2119                           &itemMeta,
2120                           0 /*cas*/),
2121             "Expected to store item");
2122 
2123     cb::EngineErrorMetadataPair errorMetaPair;
2124     check(get_meta(h, keyAbs.c_str(), errorMetaPair), "Get meta failed");
2125     checkne(time_t(0),
2126             errorMetaPair.second.exptime,
2127             "expiry should not be zero");
2128 
2129     // Force expiry
2130     testHarness->time_travel(absoluteExpiry + 1);
2131 
2132     auto ret = get(h, NULL, keyAbs.c_str(), Vbid(0));
2133     checkeq(cb::engine_errc::no_such_key,
2134             ret.first,
2135             "Failed, expected no_such_key.");
2136 
2137     check(set_param(h,
2138                     cb::mcbp::request::SetParamPayload::Type::Flush,
2139                     "max_ttl",
2140                     relativeExpiryStr.c_str()),
2141           "Failed to set max_ttl");
2142     checkeq(relativeExpiry,
2143             get_int_stat(h, "ep_max_ttl"),
2144             "max_ttl didn't change");
2145 
2146     checkeq(ENGINE_SUCCESS,
2147             set_with_meta(h,
2148                           keyRel.c_str(),
2149                           keyRel.size(),
2150                           keyRel.c_str(),
2151                           keyRel.size(),
2152                           Vbid(0),
2153                           &itemMeta,
2154                           0 /*cas*/),
2155             "Expected to store item");
2156 
2157     check(get_meta(h, keyRel.c_str(), errorMetaPair), "Get meta failed");
2158     checkne(time_t(0),
2159             errorMetaPair.second.exptime,
2160             "expiry should not be zero");
2161 
2162     // Force expiry
2163     testHarness->time_travel(relativeExpiry + 1);
2164 
2165     ret = get(h, NULL, keyRel.c_str(), Vbid(0));
2166     checkeq(cb::engine_errc::no_such_key,
2167             ret.first,
2168             "Failed, expected no_such_key.");
2169 
2170     // Final test, exceed the maxTTL and check we got capped!
2171 #if 0
2172     // TN: This piece of the test is broken as the set call fails with
2173     //     KeyEExists.. I haven't looked in details on what we're actually
2174     //     trying to test.. Disable for now
2175     itemMeta.exptime = errorMetaPair.second.exptime + 1000;
2176     checkeq(ENGINE_SUCCESS,
2177             set_with_meta(h,
2178                           keyRel.c_str(),
2179                           keyRel.size(),
2180                           keyRel.c_str(),
2181                           keyRel.size(),
2182                           Vbid(0),
2183                           &itemMeta,
2184                           0 /*cas*/),
2185             "Expected to store item");
2186 
2187     check(get_meta(h, keyRel.c_str(), errorMetaPair), "Get meta failed");
2188     checkne(itemMeta.exptime,
2189             errorMetaPair.second.exptime,
2190             "expiry should have been changed/capped");
2191 #endif
2192     return SUCCESS;
2193 }
2194 
2195 ///////////////////////////////////////////////////////////////////////////////
2196 // Test manifest //////////////////////////////////////////////////////////////
2197 ///////////////////////////////////////////////////////////////////////////////
2198 
2199 const char *default_dbname = "./ep_testsuite_basic";
2200 
2201 BaseTestCase testsuite_testcases[] = {
2202         TestCase("test alloc limit",
2203                  test_alloc_limit,
2204                  test_setup,
2205                  teardown,
2206                  NULL,
2207                  prepare,
2208                  cleanup),
2209         TestCase("test_memory_tracking",
2210                  test_memory_tracking,
2211                  test_setup,
2212                  teardown,
2213                  NULL,
2214                  prepare,
2215                  cleanup),
2216         TestCase("test max_size - water_mark changes",
2217                  test_max_size_and_water_marks_settings,
2218                  test_setup,
2219                  teardown,
2220                  "max_size=1000;ht_locks=1;ht_size=3",
2221                  prepare,
2222                  cleanup),
2223         TestCase("test whitespace dbname",
2224                  test_whitespace_db,
2225                  test_setup,
2226                  teardown,
2227                  "dbname=" WHITESPACE_DB ";ht_locks=1;ht_size=3",
2228                  prepare,
2229                  cleanup),
2230         TestCase("get miss",
2231                  test_get_miss,
2232                  test_setup,
2233                  teardown,
2234                  NULL,
2235                  prepare,
2236                  cleanup),
2237         TestCase("set", test_set, test_setup, teardown, NULL, prepare, cleanup),
2238         TestCase("concurrent set",
2239                  test_conc_set,
2240                  test_setup,
2241                  teardown,
2242                  NULL,
2243                  prepare,
2244                  cleanup),
2245         TestCase("multi set",
2246                  test_multi_set,
2247                  test_setup,
2248                  teardown,
2249                  NULL,
2250                  prepare_ep_bucket_skip_broken_under_magma, // MB-36547
2251                  cleanup),
2252         TestCase("set+get hit",
2253                  test_set_get_hit,
2254                  test_setup,
2255                  teardown,
2256                  NULL,
2257                  prepare,
2258                  cleanup),
2259         TestCase("test getl then del with cas",
2260                  test_getl_delete_with_cas,
2261                  test_setup,
2262                  teardown,
2263                  NULL,
2264                  prepare,
2265                  cleanup),
2266         TestCase("test getl then del with bad cas",
2267                  test_getl_delete_with_bad_cas,
2268                  test_setup,
2269                  teardown,
2270                  NULL,
2271                  prepare,
2272                  cleanup),
2273         TestCase("test getl then set with meta",
2274                  test_getl_set_del_with_meta,
2275                  test_setup,
2276                  teardown,
2277                  NULL,
2278                  prepare,
2279                  cleanup),
2280         TestCase("getl",
2281                  test_getl,
2282                  test_setup,
2283                  teardown,
2284                  NULL,
2285                  prepare,
2286                  cleanup),
2287         TestCase("unl", test_unl, test_setup, teardown, NULL, prepare, cleanup),
2288         TestCase("unl not my vbucket",
2289                  test_unl_nmvb,
2290                  test_setup,
2291                  teardown,
2292                  NULL,
2293                  prepare,
2294                  cleanup),
2295         TestCase("set+get hit (bin)",
2296                  test_set_get_hit_bin,
2297                  test_setup,
2298                  teardown,
2299                  NULL,
2300                  prepare,
2301                  cleanup),
2302         TestCase("set with cas non-existent",
2303                  test_set_with_cas_non_existent,
2304                  test_setup,
2305                  teardown,
2306                  NULL,
2307                  prepare,
2308                  cleanup),
2309         TestCase("set+change flags",
2310                  test_set_change_flags,
2311                  test_setup,
2312                  teardown,
2313                  NULL,
2314                  prepare,
2315                  cleanup),
2316         TestCase("add", test_add, test_setup, teardown, NULL, prepare, cleanup),
2317         TestCase("add+add(same cas)",
2318                  test_add_add_with_cas,
2319                  test_setup,
2320                  teardown,
2321                  NULL,
2322                  prepare,
2323                  cleanup),
2324         TestCase("cas", test_cas, test_setup, teardown, NULL, prepare, cleanup),
2325         TestCase("replace",
2326                  test_replace,
2327                  test_setup,
2328                  teardown,
2329                  NULL,
2330                  prepare,
2331                  cleanup),
2332         TestCase("test touch",
2333                  test_touch,
2334                  test_setup,
2335                  teardown,
2336                  NULL,
2337                  prepare,
2338                  cleanup),
2339         TestCase("test touch (MB-7342)",
2340                  test_touch_mb7342,
2341                  test_setup,
2342                  teardown,
2343                  NULL,
2344                  prepare,
2345                  cleanup),
2346         TestCase("test touch (MB-10277)",
2347                  test_touch_mb10277,
2348                  test_setup,
2349                  teardown,
2350                  NULL,
2351                  prepare_ep_bucket,
2352                  cleanup),
2353         TestCase("test gat",
2354                  test_gat,
2355                  test_setup,
2356                  teardown,
2357                  NULL,
2358                  prepare,
2359                  cleanup),
2360         TestCase("test locked gat",
2361                  test_gat_locked,
2362                  test_setup,
2363                  teardown,
2364                  NULL,
2365                  prepare,
2366                  cleanup),
2367         TestCase("test locked touch",
2368                  test_touch_locked,
2369                  test_setup,
2370                  teardown,
2371                  NULL,
2372                  prepare,
2373                  cleanup),
2374         TestCase("test mb5215",
2375                  test_mb5215,
2376                  test_setup,
2377                  teardown,
2378                  NULL,
2379                  // TODO RDB: implement getItemCount. Needs the
2380                  // 'ep_num_non_resident' stat.
2381                  prepare_ep_bucket_skip_broken_under_rocks,
2382                  cleanup),
2383         TestCase("delete",
2384                  test_delete,
2385                  test_setup,
2386                  teardown,
2387                  NULL,
2388                  prepare,
2389                  cleanup),
2390         TestCase("delete with value",
2391                  test_delete_with_value,
2392                  test_setup,
2393                  teardown,
2394                  NULL,
2395                  /* TODO RDB: vBucket num_items not correct under Rocks when
2396                   * full eviction */
2397                  prepare_skip_broken_under_rocks_full_eviction,
2398                  cleanup),
2399         TestCase("delete with value CAS",
2400                  test_delete_with_value_cas,
2401                  test_setup,
2402                  teardown,
2403                  NULL,
2404                  prepare,
2405                  cleanup),
2406         TestCase("set/delete",
2407                  test_set_delete,
2408                  test_setup,
2409                  teardown,
2410                  NULL,
2411                  prepare,
2412                  cleanup),
2413         TestCase("set/delete (invalid cas)",
2414                  test_set_delete_invalid_cas,
2415                  test_setup,
2416                  teardown,
2417                  NULL,
2418                  prepare,
2419                  cleanup),
2420         TestCase("delete/set/delete",
2421                  test_delete_set,
2422                  test_setup,
2423                  teardown,
2424                  NULL,
2425                  prepare,
2426                  cleanup),
2427         TestCase("get/delete with missing db file",
2428                  test_get_delete_missing_file,
2429                  test_setup,
2430                  teardown,
2431                  NULL,
2432                  // TODO RDB: This test fails because under RocksDB we can
2433                  // still find a key after deleting the DB file and evicting
2434                  // the key in the internal MemTable (which is also used as
2435                  // read-cache).
2436                  // TODO magma: uses couchstore specific functions
2437                  prepare_ep_bucket_skip_broken_under_rocks_and_magma,
2438                  cleanup),
2439         TestCase("vbucket deletion doesn't affect new data",
2440                  test_bug7023,
2441                  test_setup,
2442                  teardown,
2443                  NULL,
2444                  // TODO RDB: implement getItemCount. Needs the
2445                  // 'ep_warmup_value_count' stat.
2446                  prepare_skip_broken_under_rocks,
2447                  cleanup),
2448         TestCase("non-resident decrementers",
2449                  test_mb3169,
2450                  test_setup,
2451                  teardown,
2452                  NULL,
2453                  // TODO RDB: implement getItemCount. Needs the 'curr_items'
2454                  // stat.
2455                  prepare_ep_bucket_skip_broken_under_rocks,
2456                  cleanup),
2457         TestCase("resident ratio after warmup",
2458                  test_mb5172,
2459                  test_setup,
2460                  teardown,
2461                  NULL,
2462                  prepare,
2463                  cleanup),
2464         TestCase("set vb 10000",
2465                  test_set_vbucket_out_of_range,
2466                  test_setup,
2467                  teardown,
2468                  "max_vbuckets=1024",
2469                  prepare,
2470                  cleanup),
2471         TestCase("set max_cas MB21190",
2472                  set_max_cas_mb21190,
2473                  test_setup,
2474                  teardown,
2475                  nullptr,
2476                  prepare,
2477                  cleanup),
2478         TestCase("warmup_mb21769",
2479                  warmup_mb21769,
2480                  test_setup,
2481                  teardown,
2482                  nullptr,
2483                  prepare,
2484                  cleanup),
2485 
2486         TestCase("pre_link_document",
2487                  pre_link_document,
2488                  test_setup,
2489                  teardown,
2490                  nullptr,
2491                  prepare,
2492                  cleanup),
2493 
2494         TestCase("engine get_if",
2495                  get_if,
2496                  test_setup,
2497                  teardown,
2498                  nullptr,
2499                  prepare,
2500                  cleanup),
2501 
2502         TestCase("test max_ttl range",
2503                  max_ttl_out_of_range,
2504                  test_setup,
2505                  teardown,
2506                  nullptr,
2507                  prepare,
2508                  cleanup),
2509 
2510         TestCase("test max_ttl",
2511                  max_ttl,
2512                  test_setup,
2513                  teardown,
2514                  nullptr,
2515                  prepare,
2516                  cleanup),
2517 
2518         TestCase("test max_ttl_setWithMeta",
2519                  max_ttl_setWithMeta,
2520                  test_setup,
2521                  teardown,
2522                  nullptr,
2523                  prepare,
2524                  cleanup),
2525 
2526         // sentinel
2527         TestCase(NULL, NULL, NULL, NULL, NULL, prepare, cleanup)};
2528