1 /* -*- MODE: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3  *     Copyright 2010 Couchbase, Inc
4  *
5  *   Licensed under the Apache License, Version 2.0 (the "License");
6  *   you may not use this file except in compliance with the License.
7  *   You may obtain a copy of the License at
8  *
9  *       http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *   Unless required by applicable law or agreed to in writing, software
12  *   distributed under the License is distributed on an "AS IS" BASIS,
13  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *   See the License for the specific language governing permissions and
15  *   limitations under the License.
16  */
17 
18 // Usage: (to run just a single test case)
19 // make engine_tests EP_TEST_NUM=3
20 
21 #include <stdio.h>
22 #include <stdlib.h>
23 #include <string.h>
24 
25 #include <chrono>
26 #include <condition_variable>
27 #include <cstdlib>
28 #include <iomanip>
29 #include <iostream>
30 #include <map>
31 #include <mutex>
32 #include <regex>
33 #include <set>
34 #include <sstream>
35 #include <string>
36 #include <thread>
37 #include <unordered_map>
38 #include <unordered_set>
39 #include <vector>
40 
41 #include "atomic.h"
42 #include "couch-kvstore/couch-kvstore-metadata.h"
43 #include "ep_test_apis.h"
44 
45 #include "ep_testsuite_common.h"
46 #include "locks.h"
47 #include <libcouchstore/couch_db.h>
48 #include <memcached/engine.h>
49 #include <memcached/engine_error.h>
50 #include <memcached/engine_testapp.h>
51 #include <memcached/types.h>
52 #include <nlohmann/json.hpp>
53 #include <platform/cb_malloc.h>
54 #include <platform/cbassert.h>
55 #include <platform/dirutils.h>
56 #include <platform/platform_thread.h>
57 #include <platform/platform_time.h>
58 #include <platform/strerror.h>
59 #include <string_utilities.h>
60 #include <xattr/blob.h>
61 #include <xattr/utils.h>
62 
63 #ifdef linux
64 /* /usr/include/netinet/in.h defines macros from ntohs() to _bswap_nn to
65  * optimize the conversion functions, but the prototypes generate warnings
66  * from gcc. The conversion methods isn't the bottleneck for my app, so
67  * just remove the warnings by undef'ing the optimization ..
68  */
69 #undef ntohs
70 #undef ntohl
71 #undef htons
72 #undef htonl
73 #endif
74 
75 using namespace std::string_literals;
76 
77 // ptr_fun don't like the extern "C" thing for unlock cookie.. cast it
78 // away ;)
79 typedef void (*UNLOCK_COOKIE_T)(const void *cookie);
80 
81 class ThreadData {
82 public:
ThreadData(EngineIface* eh, int e = 0)83     ThreadData(EngineIface* eh, int e = 0) : h(eh), extra(e) {
84     }
85     EngineIface* h;
86     int extra;
87 };
88 
89 enum class BucketType { EP, Ephemeral };
90 
check_observe_seqno(bool failover, BucketType bucket_type, uint8_t format_type, Vbid vb_id, uint64_t vb_uuid, uint64_t last_persisted_seqno, uint64_t current_seqno, uint64_t failover_vbuuid = 0, uint64_t failover_seqno = 0)91 static void check_observe_seqno(bool failover,
92                                 BucketType bucket_type,
93                                 uint8_t format_type,
94                                 Vbid vb_id,
95                                 uint64_t vb_uuid,
96                                 uint64_t last_persisted_seqno,
97                                 uint64_t current_seqno,
98                                 uint64_t failover_vbuuid = 0,
99                                 uint64_t failover_seqno = 0) {
100     uint8_t recv_format_type;
101     Vbid recv_vb_id;
102     uint64_t recv_vb_uuid;
103     uint64_t recv_last_persisted_seqno;
104     uint64_t recv_current_seqno;
105     uint64_t recv_failover_vbuuid;
106     uint64_t recv_failover_seqno;
107 
108     memcpy(&recv_format_type, last_body.data(), sizeof(uint8_t));
109     checkeq(format_type, recv_format_type, "Wrong format type in result");
110     memcpy(&recv_vb_id, last_body.data() + 1, sizeof(uint16_t));
111     checkeq(vb_id.get(), ntohs(recv_vb_id.get()), "Wrong vbucket id in result");
112     memcpy(&recv_vb_uuid, last_body.data() + 3, sizeof(uint64_t));
113     checkeq(vb_uuid, ntohll(recv_vb_uuid), "Wrong vbucket uuid in result");
114     memcpy(&recv_last_persisted_seqno, last_body.data() + 11, sizeof(uint64_t));
115 
116     switch (bucket_type) {
117     case BucketType::EP:
118         // Should get the "real" persisted seqno:
119         checkeq(last_persisted_seqno,
120                 ntohll(recv_last_persisted_seqno),
121                 "Wrong persisted seqno in result (EP)");
122         break;
123     case BucketType::Ephemeral:
124         // For ephemeral, this should always be zero, as there is no
125         // persistence.
126         checkeq(uint64_t(0),
127                 ntohll(recv_last_persisted_seqno),
128                 "Wrong persisted seqno in result (Ephemeral)");
129         break;
130     }
131 
132     memcpy(&recv_current_seqno, last_body.data() + 19, sizeof(uint64_t));
133     checkeq(current_seqno, ntohll(recv_current_seqno), "Wrong current seqno in result");
134 
135     if (failover) {
136         memcpy(&recv_failover_vbuuid, last_body.data() + 27, sizeof(uint64_t));
137         checkeq(failover_vbuuid, ntohll(recv_failover_vbuuid),
138                 "Wrong failover uuid in result");
139         memcpy(&recv_failover_seqno, last_body.data() + 35, sizeof(uint64_t));
140         checkeq(failover_seqno, ntohll(recv_failover_seqno),
141                 "Wrong failover seqno in result");
142     }
143 }
144 
test_replace_with_eviction(EngineIface* h)145 static enum test_result test_replace_with_eviction(EngineIface* h) {
146     checkeq(ENGINE_SUCCESS,
147             store(h, NULL, OPERATION_SET, "key", "somevalue"),
148             "Failed to set value.");
149     wait_for_flusher_to_settle(h);
150     evict_key(h, "key");
151     int numBgFetched = get_int_stat(h, "ep_bg_fetched");
152 
153     checkeq(ENGINE_SUCCESS,
154             store(h, NULL, OPERATION_REPLACE, "key", "somevalue1"),
155             "Failed to replace existing value.");
156 
157     checkeq(ENGINE_SUCCESS,
158             get_stats(h, {}, {}, add_stats),
159             "Failed to get stats.");
160     std::string eviction_policy = vals.find("ep_item_eviction_policy")->second;
161     if (eviction_policy == "full_eviction") {
162         numBgFetched++;
163     }
164 
165     checkeq(numBgFetched,
166             get_int_stat(h, "ep_bg_fetched"),
167             "Bg fetched value didn't match");
168 
169     check_key_value(h, "key", "somevalue1", 10);
170     return SUCCESS;
171 }
172 
test_wrong_vb_mutation(EngineIface* h, ENGINE_STORE_OPERATION op)173 static enum test_result test_wrong_vb_mutation(EngineIface* h,
174                                                ENGINE_STORE_OPERATION op) {
175     int numNotMyVBucket = get_int_stat(h, "ep_num_not_my_vbuckets");
176     uint64_t cas = 11;
177     if (op == OPERATION_ADD) {
178         // Add operation with cas != 0 doesn't make sense
179         cas = 0;
180     }
181     checkeq(ENGINE_NOT_MY_VBUCKET,
182             store(h, NULL, op, "key", "somevalue", nullptr, cas, Vbid(1)),
183             "Expected not_my_vbucket");
184     wait_for_stat_change(h, "ep_num_not_my_vbuckets", numNotMyVBucket);
185     return SUCCESS;
186 }
187 
test_pending_vb_mutation(EngineIface* h, ENGINE_STORE_OPERATION op)188 static enum test_result test_pending_vb_mutation(EngineIface* h,
189                                                  ENGINE_STORE_OPERATION op) {
190     const void* cookie = testHarness->create_cookie();
191     testHarness->set_ewouldblock_handling(cookie, false);
192     check(set_vbucket_state(h, Vbid(1), vbucket_state_pending),
193           "Failed to set vbucket state.");
194     check(verify_vbucket_state(h, Vbid(1), vbucket_state_pending),
195           "Bucket state was not set to pending.");
196     uint64_t cas = 11;
197     if (op == OPERATION_ADD) {
198         // Add operation with cas != 0 doesn't make sense..
199         cas = 0;
200     }
201     checkeq(ENGINE_EWOULDBLOCK,
202             store(h, cookie, op, "key", "somevalue", nullptr, cas, Vbid(1)),
203             "Expected ewouldblock");
204     testHarness->destroy_cookie(cookie);
205     return SUCCESS;
206 }
207 
test_replica_vb_mutation(EngineIface* h, ENGINE_STORE_OPERATION op)208 static enum test_result test_replica_vb_mutation(EngineIface* h,
209                                                  ENGINE_STORE_OPERATION op) {
210     check(set_vbucket_state(h, Vbid(1), vbucket_state_replica),
211           "Failed to set vbucket state.");
212     check(verify_vbucket_state(h, Vbid(1), vbucket_state_replica),
213           "Bucket state was not set to replica.");
214     int numNotMyVBucket = get_int_stat(h, "ep_num_not_my_vbuckets");
215 
216     uint64_t cas = 11;
217     if (op == OPERATION_ADD) {
218         // performing add with a CAS != 0 doesn't make sense...
219         cas = 0;
220     }
221     checkeq(ENGINE_NOT_MY_VBUCKET,
222             store(h, NULL, op, "key", "somevalue", nullptr, cas, Vbid(1)),
223             "Expected not my vbucket");
224     wait_for_stat_change(h, "ep_num_not_my_vbuckets", numNotMyVBucket);
225     return SUCCESS;
226 }
227 
228 //
229 // ----------------------------------------------------------------------
230 // The actual tests are below.
231 // ----------------------------------------------------------------------
232 //
233 
checkCurrItemsAfterShutdown(EngineIface* h, int numItems2Load, bool shutdownForce)234 static int checkCurrItemsAfterShutdown(EngineIface* h,
235                                        int numItems2Load,
236                                        bool shutdownForce) {
237     if (!isWarmupEnabled(h)) {
238         return SKIPPED;
239     }
240 
241     std::vector<std::string> keys;
242     for (int index = 0; index < numItems2Load; ++index) {
243         std::stringstream s;
244         s << "keys_2_load-" << index;
245         std::string key(s.str());
246         keys.push_back(key);
247     }
248 
249     // Check preconditions.
250     checkeq(0,
251             get_int_stat(h, "ep_total_persisted"),
252             "Expected ep_total_persisted equals 0");
253     checkeq(0, get_int_stat(h, "curr_items"), "Expected curr_items equals 0");
254 
255     // stop flusher before loading new items
256     auto pkt = createPacket(cb::mcbp::ClientOpcode::StopPersistence);
257     checkeq(ENGINE_SUCCESS,
258             h->unknown_command(NULL, *pkt, add_response),
259             "CMD_STOP_PERSISTENCE failed!");
260     checkeq(cb::mcbp::Status::Success,
261             last_status.load(),
262             "Failed to stop persistence!");
263 
264     std::vector<std::string>::iterator itr;
265     for (itr = keys.begin(); itr != keys.end(); ++itr) {
266         checkeq(ENGINE_SUCCESS,
267                 store(h, NULL, OPERATION_SET, itr->c_str(), "oracle"),
268                 "Failed to store a value");
269     }
270 
271     checkeq(0,
272             get_int_stat(h, "ep_total_persisted"),
273             "Incorrect ep_total_persisted, expected 0");
274 
275     // Can only check curr_items in value_only eviction; full-eviction
276     // relies on persistence to complete (via flusher) to update count.
277     const auto evictionPolicy = get_str_stat(h, "ep_item_eviction_policy");
278     if (evictionPolicy == "value_only") {
279         checkeq(numItems2Load,
280                 get_int_stat(h, "curr_items"),
281                 "Expected curr_items to reflect item count");
282     }
283 
284     // resume flusher before shutdown + warmup
285     pkt = createPacket(cb::mcbp::ClientOpcode::StartPersistence);
286     checkeq(ENGINE_SUCCESS,
287             h->unknown_command(NULL, *pkt, add_response),
288             "CMD_START_PERSISTENCE failed!");
289     checkeq(cb::mcbp::Status::Success, last_status.load(),
290           "Failed to start persistence!");
291 
292     // shutdown engine force and restart
293     testHarness->reload_engine(&h,
294                                testHarness->engine_path,
295                                testHarness->get_current_testcase()->cfg,
296                                true,
297                                shutdownForce);
298 
299     wait_for_warmup_complete(h);
300     return get_int_stat(h, "curr_items");
301 }
302 
test_flush_shutdown_force(EngineIface* h)303 static enum test_result test_flush_shutdown_force(EngineIface* h) {
304     if (!isWarmupEnabled(h)) {
305         return SKIPPED;
306     }
307 
308     int numItems2load = 3000;
309     bool shutdownForce = true;
310     int currItems =
311             checkCurrItemsAfterShutdown(h, numItems2load, shutdownForce);
312     checkle(currItems, numItems2load,
313            "Number of curr items should be <= 3000, unless previous "
314            "shutdown force had to wait for the flusher");
315     return SUCCESS;
316 }
317 
test_flush_shutdown_noforce(EngineIface* h)318 static enum test_result test_flush_shutdown_noforce(EngineIface* h) {
319     if (!isWarmupEnabled(h)) {
320         return SKIPPED;
321     }
322 
323     int numItems2load = 3000;
324     bool shutdownForce = false;
325     int currItems =
326             checkCurrItemsAfterShutdown(h, numItems2load, shutdownForce);
327     checkeq(numItems2load, currItems,
328            "Number of curr items should be equal to 3000, unless previous "
329            "shutdown did not wait for the flusher");
330     return SUCCESS;
331 }
332 
test_shutdown_snapshot_range(EngineIface* h)333 static enum test_result test_shutdown_snapshot_range(EngineIface* h) {
334     if (!isWarmupEnabled(h)) {
335         return SKIPPED;
336     }
337 
338     const int num_items = 100;
339     for (int j = 0; j < num_items; ++j) {
340         std::stringstream ss;
341         ss << "key" << j;
342         checkeq(ENGINE_SUCCESS,
343                 store(h, NULL, OPERATION_SET, ss.str().c_str(), "data"),
344                 "Failed to store a value");
345     }
346 
347     wait_for_flusher_to_settle(h);
348     int end = get_int_stat(h, "vb_0:high_seqno", "vbucket-seqno");
349 
350     /* change vb state to replica before restarting (as it happens in graceful
351        failover)*/
352     check(set_vbucket_state(h, Vbid(0), vbucket_state_replica),
353           "Failed set vbucket 0 to replica state.");
354 
355     /* trigger persist vb state task */
356     check(set_param(h,
357                     cb::mcbp::request::SetParamPayload::Type::Flush,
358                     "vb_state_persist_run",
359                     "0"),
360           "Failed to trigger vb state persist");
361 
362     /* restart the engine */
363     testHarness->reload_engine(&h,
364                                testHarness->engine_path,
365                                testHarness->get_current_testcase()->cfg,
366                                true,
367                                false);
368 
369     wait_for_warmup_complete(h);
370 
371     /* Check if snapshot range is persisted correctly */
372     checkeq(end,
373             get_int_stat(h, "vb_0:last_persisted_snap_start", "vbucket-seqno"),
374             "Wrong snapshot start persisted");
375     checkeq(end,
376             get_int_stat(h, "vb_0:last_persisted_snap_end", "vbucket-seqno"),
377             "Wrong snapshot end persisted");
378 
379     return SUCCESS;
380 }
381 
test_restart(EngineIface* h)382 static enum test_result test_restart(EngineIface* h) {
383     if (!isWarmupEnabled(h)) {
384         return SKIPPED;
385     }
386 
387     static const char val[] = "somevalue";
388     ENGINE_ERROR_CODE ret = store(h, NULL, OPERATION_SET, "key", val);
389     checkeq(ENGINE_SUCCESS, ret, "Failed set.");
390 
391     testHarness->reload_engine(&h,
392                                testHarness->engine_path,
393                                testHarness->get_current_testcase()->cfg,
394                                true,
395                                false);
396 
397     wait_for_warmup_complete(h);
398     check_key_value(h, "key", val, strlen(val));
399     return SUCCESS;
400 }
401 
test_specialKeys(EngineIface* h)402 static enum test_result test_specialKeys(EngineIface* h) {
403     ENGINE_ERROR_CODE ret;
404 
405     // Simplified Chinese "Couchbase"
406     static const char key0[] = "沙发数据库";
407     static const char val0[] = "some Chinese value";
408     checkeq(ENGINE_SUCCESS, (ret = store(h, NULL, OPERATION_SET, key0, val0)),
409           "Failed set Chinese key");
410     check_key_value(h, key0, val0, strlen(val0));
411 
412     // Traditional Chinese "Couchbase"
413     static const char key1[] = "沙發數據庫";
414     static const char val1[] = "some Traditional Chinese value";
415     checkeq(ENGINE_SUCCESS, (ret = store(h, NULL, OPERATION_SET, key1, val1)),
416           "Failed set Traditional Chinese key");
417 
418     // Korean "couch potato"
419     static const char key2[] = "쇼파감자";
420     static const char val2[] = "some Korean value";
421     checkeq(ENGINE_SUCCESS, (ret = store(h, NULL, OPERATION_SET, key2, val2)),
422           "Failed set Korean key");
423 
424     // Russian "couch potato"
425     static const char key3[] = "лодырь, лентяй";
426     static const char val3[] = "some Russian value";
427     checkeq(ENGINE_SUCCESS, (ret = store(h, NULL, OPERATION_SET, key3, val3)),
428           "Failed set Russian key");
429 
430     // Japanese "couch potato"
431     static const char key4[] = "カウチポテト";
432     static const char val4[] = "some Japanese value";
433     checkeq(ENGINE_SUCCESS, (ret = store(h, NULL, OPERATION_SET, key4, val4)),
434           "Failed set Japanese key");
435 
436     // Indian char key, and no idea what it is
437     static const char key5[] = "हरियानवी";
438     static const char val5[] = "some Indian value";
439     checkeq(ENGINE_SUCCESS, (ret = store(h, NULL, OPERATION_SET, key5, val5)),
440           "Failed set Indian key");
441 
442     // Portuguese translation "couch potato"
443     static const char key6[] = "sedentário";
444     static const char val6[] = "some Portuguese value";
445     checkeq(ENGINE_SUCCESS, (ret = store(h, NULL, OPERATION_SET, key6, val6)),
446           "Failed set Portuguese key");
447 
448     // Arabic translation "couch potato"
449     static const char key7[] = "الحافلةالبطاطة";
450     static const char val7[] = "some Arabic value";
451     checkeq(ENGINE_SUCCESS, (ret = store(h, NULL, OPERATION_SET, key7, val7)),
452           "Failed set Arabic key");
453 
454     if (isWarmupEnabled(h)) {
455         // Check that after warmup the keys are still present.
456         testHarness->reload_engine(&h,
457                                    testHarness->engine_path,
458                                    testHarness->get_current_testcase()->cfg,
459                                    true,
460                                    false);
461 
462         wait_for_warmup_complete(h);
463         check_key_value(h, key0, val0, strlen(val0));
464         check_key_value(h, key1, val1, strlen(val1));
465         check_key_value(h, key2, val2, strlen(val2));
466         check_key_value(h, key3, val3, strlen(val3));
467         check_key_value(h, key4, val4, strlen(val4));
468         check_key_value(h, key5, val5, strlen(val5));
469         check_key_value(h, key6, val6, strlen(val6));
470         check_key_value(h, key7, val7, strlen(val7));
471     }
472     return SUCCESS;
473 }
474 
test_binKeys(EngineIface* h)475 static enum test_result test_binKeys(EngineIface* h) {
476     ENGINE_ERROR_CODE ret;
477 
478     // binary key with char values beyond 0x7F
479     static const char key0[] = "\xe0\xed\xf1\x6f\x7f\xf8\xfa";
480     static const char val0[] = "some value val8";
481     checkeq(ENGINE_SUCCESS, (ret = store(h, NULL, OPERATION_SET, key0, val0)),
482           "Failed set binary key0");
483     check_key_value(h, key0, val0, strlen(val0));
484 
485     // binary keys with char values beyond 0x7F
486     static const char key1[] = "\xf1\xfd\xfe\xff\xf0\xf8\xef";
487     static const char val1[] = "some value val9";
488     checkeq(ENGINE_SUCCESS, (ret = store(h, NULL, OPERATION_SET, key1, val1)),
489           "Failed set binary key1");
490     check_key_value(h, key1, val1, strlen(val1));
491 
492     // binary keys with special utf-8 BOM (Byte Order Mark) values 0xBB 0xBF
493     // 0xEF
494     static const char key2[] = "\xff\xfe\xbb\xbf\xef";
495     static const char val2[] = "some utf-8 bom value";
496     checkeq(ENGINE_SUCCESS, (ret = store(h, NULL, OPERATION_SET, key2, val2)),
497           "Failed set binary utf-8 bom key");
498     check_key_value(h, key2, val2, strlen(val2));
499 
500     // binary keys with special utf-16BE BOM values "U+FEFF"
501     static const char key3[] = "U+\xfe\xff\xefU+\xff\xfe";
502     static const char val3[] = "some utf-16 bom value";
503     checkeq(ENGINE_SUCCESS, (ret = store(h, NULL, OPERATION_SET, key3, val3)),
504           "Failed set binary utf-16 bom key");
505     check_key_value(h, key3, val3, strlen(val3));
506 
507     if (isWarmupEnabled(h)) {
508         testHarness->reload_engine(&h,
509                                    testHarness->engine_path,
510                                    testHarness->get_current_testcase()->cfg,
511                                    true,
512                                    false);
513 
514         wait_for_warmup_complete(h);
515         check_key_value(h, key0, val0, strlen(val0));
516         check_key_value(h, key1, val1, strlen(val1));
517         check_key_value(h, key2, val2, strlen(val2));
518         check_key_value(h, key3, val3, strlen(val3));
519     }
520     return SUCCESS;
521 }
522 
test_restart_bin_val(EngineIface* h)523 static enum test_result test_restart_bin_val(EngineIface* h) {
524     if (!isWarmupEnabled(h)) {
525         return SKIPPED;
526     }
527 
528     char binaryData[] = "abcdefg\0gfedcba";
529     cb_assert(sizeof(binaryData) != strlen(binaryData));
530 
531     checkeq(cb::engine_errc::success,
532             storeCasVb11(h,
533                          NULL,
534                          OPERATION_SET,
535                          "key",
536                          binaryData,
537                          sizeof(binaryData),
538                          82758,
539                          0,
540                          Vbid(0))
541                     .first,
542             "Failed set.");
543 
544     testHarness->reload_engine(&h,
545                                testHarness->engine_path,
546                                testHarness->get_current_testcase()->cfg,
547                                true,
548                                false);
549 
550     wait_for_warmup_complete(h);
551 
552     check_key_value(h, "key", binaryData, sizeof(binaryData));
553     return SUCCESS;
554 }
555 
test_wrong_vb_get(EngineIface* h)556 static enum test_result test_wrong_vb_get(EngineIface* h) {
557     int numNotMyVBucket = get_int_stat(h, "ep_num_not_my_vbuckets");
558     checkeq(ENGINE_NOT_MY_VBUCKET,
559             verify_key(h, "key", Vbid(1)),
560             "Expected wrong bucket.");
561     wait_for_stat_change(h, "ep_num_not_my_vbuckets", numNotMyVBucket);
562     return SUCCESS;
563 }
564 
test_vb_get_pending(EngineIface* h)565 static enum test_result test_vb_get_pending(EngineIface* h) {
566     check(set_vbucket_state(h, Vbid(1), vbucket_state_pending),
567           "Failed to set vbucket state.");
568     const void* cookie = testHarness->create_cookie();
569     testHarness->set_ewouldblock_handling(cookie, false);
570 
571     checkeq(cb::engine_errc::would_block,
572             get(h, cookie, "key", Vbid(1)).first,
573             "Expected wouldblock.");
574     checkeq(1, get_int_stat(h, "vb_pending_ops_get"), "Expected 1 get");
575 
576     testHarness->destroy_cookie(cookie);
577     return SUCCESS;
578 }
579 
test_vb_get_replica(EngineIface* h)580 static enum test_result test_vb_get_replica(EngineIface* h) {
581     check(set_vbucket_state(h, Vbid(1), vbucket_state_replica),
582           "Failed to set vbucket state.");
583     int numNotMyVBucket = get_int_stat(h, "ep_num_not_my_vbuckets");
584     checkeq(ENGINE_NOT_MY_VBUCKET,
585             verify_key(h, "key", Vbid(1)),
586             "Expected not my bucket.");
587     wait_for_stat_change(h, "ep_num_not_my_vbuckets", numNotMyVBucket);
588     return SUCCESS;
589 }
590 
test_wrong_vb_set(EngineIface* h)591 static enum test_result test_wrong_vb_set(EngineIface* h) {
592     return test_wrong_vb_mutation(h, OPERATION_SET);
593 }
594 
test_wrong_vb_cas(EngineIface* h)595 static enum test_result test_wrong_vb_cas(EngineIface* h) {
596     return test_wrong_vb_mutation(h, OPERATION_CAS);
597 }
598 
test_wrong_vb_add(EngineIface* h)599 static enum test_result test_wrong_vb_add(EngineIface* h) {
600     return test_wrong_vb_mutation(h, OPERATION_ADD);
601 }
602 
test_wrong_vb_replace(EngineIface* h)603 static enum test_result test_wrong_vb_replace(EngineIface* h) {
604     return test_wrong_vb_mutation(h, OPERATION_REPLACE);
605 }
606 
test_wrong_vb_del(EngineIface* h)607 static enum test_result test_wrong_vb_del(EngineIface* h) {
608     int numNotMyVBucket = get_int_stat(h, "ep_num_not_my_vbuckets");
609     checkeq(ENGINE_NOT_MY_VBUCKET,
610             del(h, "key", 0, Vbid(1)),
611             "Expected wrong bucket.");
612     wait_for_stat_change(h, "ep_num_not_my_vbuckets", numNotMyVBucket);
613     return SUCCESS;
614 }
615 
616 /* Returns a string in the format "%Y-%m-%d %H:%M:%S" of the specified
617  * time point.
618  */
make_time_string(std::chrono::system_clock::time_point time_point)619 std::string make_time_string(std::chrono::system_clock::time_point time_point) {
620     time_t tt = std::chrono::system_clock::to_time_t(time_point);
621 #ifdef _MSC_VER
622     // Windows' gmtime() is already thread-safe.
623     struct tm* split = gmtime(&tt);
624 #else
625     struct tm local_storage;
626     struct tm* split = gmtime_r(&tt, &local_storage);
627 #endif
628     char timeStr[20];
629     strftime(timeStr, 20, "%Y-%m-%d %H:%M:%S", split);
630     return timeStr;
631 }
632 
test_expiry_pager_settings(EngineIface* h)633 static enum test_result test_expiry_pager_settings(EngineIface* h) {
634     cb_assert(!get_bool_stat(h, "ep_exp_pager_enabled"));
635     checkeq(3600,
636             get_int_stat(h, "ep_exp_pager_stime"),
637             "Expiry pager sleep time not expected");
638     set_param(h,
639               cb::mcbp::request::SetParamPayload::Type::Flush,
640               "exp_pager_stime",
641               "1");
642     checkeq(1,
643             get_int_stat(h, "ep_exp_pager_stime"),
644             "Expiry pager sleep time not updated");
645     cb_assert(!get_bool_stat(h, "ep_exp_pager_enabled"));
646     std::this_thread::sleep_for(std::chrono::seconds(1));
647     checkeq(0,
648             get_int_stat(h, "ep_num_expiry_pager_runs"),
649             "Expiry pager run count is not zero");
650 
651     set_param(h,
652               cb::mcbp::request::SetParamPayload::Type::Flush,
653               "exp_pager_enabled",
654               "true");
655     checkeq(1,
656             get_int_stat(h, "ep_exp_pager_stime"),
657             "Expiry pager sleep time not updated");
658     wait_for_stat_to_be_gte(h, "ep_num_expiry_pager_runs", 1);
659 
660     // Reload engine
661     testHarness->reload_engine(&h,
662                                testHarness->engine_path,
663                                testHarness->get_current_testcase()->cfg,
664                                true,
665                                false);
666 
667     wait_for_warmup_complete(h);
668     cb_assert(!get_bool_stat(h, "ep_exp_pager_enabled"));
669 
670     // Enable expiry pager again
671     set_param(h,
672               cb::mcbp::request::SetParamPayload::Type::Flush,
673               "exp_pager_enabled",
674               "true");
675 
676     checkeq(get_int_stat(h, "ep_exp_pager_initial_run_time"),
677             -1,
678             "Task time should be disable upon warmup");
679 
680     std::string err_msg;
681     // Update exp_pager_initial_run_time and ensure the update is successful
682     set_param(h,
683               cb::mcbp::request::SetParamPayload::Type::Flush,
684               "exp_pager_initial_run_time",
685               "3");
686     std::string expected_time = "03:00";
687     std::string str;
688     // [MB-21806] - Need to repeat the fetch as the set_param for
689     // "exp_pager_initial_run_time" schedules a task that sets the stats later
690     repeat_till_true([&]() {
691         str = get_str_stat(h, "ep_expiry_pager_task_time");
692         return 0 == str.substr(11, 5).compare(expected_time);
693     });
694     err_msg.assign("Updated time incorrect, expect: " +
695                    expected_time + ", actual: " + str.substr(11, 5));
696     checkeq(0, str.substr(11, 5).compare(expected_time), err_msg.c_str());
697 
698     // Update exp_pager_stime by 30 minutes and ensure that the update is successful
699     const std::chrono::minutes update_by{30};
700     std::string targetTaskTime1{make_time_string(std::chrono::system_clock::now() +
701                                                  update_by)};
702 
703     set_param(h,
704               cb::mcbp::request::SetParamPayload::Type::Flush,
705               "exp_pager_stime",
706               std::to_string(update_by.count() * 60).c_str());
707     str = get_str_stat(h, "ep_expiry_pager_task_time");
708 
709     std::string targetTaskTime2{make_time_string(std::chrono::system_clock::now() +
710                                                  update_by)};
711 
712     // ep_expiry_pager_task_time should fall within the range of
713     // targetTaskTime1 and targetTaskTime2
714     err_msg.assign("Unexpected task time range, expect: " +
715                    targetTaskTime1 + " <= " + str + " <= " + targetTaskTime2);
716     checkle(targetTaskTime1, str, err_msg.c_str());
717     checkle(str, targetTaskTime2, err_msg.c_str());
718 
719     return SUCCESS;
720 }
721 
test_expiry_with_xattr(EngineIface* h)722 static enum test_result test_expiry_with_xattr(EngineIface* h) {
723     const char* key = "test_expiry";
724     cb::xattr::Blob blob;
725 
726     //Add a few XAttrs
727     blob.set("user", "{\"author\":\"bubba\"}");
728     blob.set("_sync", "{\"cas\":\"0xdeadbeefcafefeed\"}");
729     blob.set("meta", "{\"content-type\":\"text\"}");
730 
731     auto xattr_value = blob.finalize();
732 
733     //Now, append user data to the xattrs and store the data
734     std::string value_data("test_expiry_value");
735     std::vector<char> data;
736     std::copy(xattr_value.buf, xattr_value.buf + xattr_value.len,
737               std::back_inserter(data));
738     std::copy(value_data.c_str(), value_data.c_str() + value_data.length(),
739               std::back_inserter(data));
740 
741     const void* cookie = testHarness->create_cookie();
742 
743     checkeq(cb::engine_errc::success,
744             storeCasVb11(h,
745                          cookie,
746                          OPERATION_SET,
747                          key,
748                          reinterpret_cast<char*>(data.data()),
749                          data.size(),
750                          9258,
751                          0,
752                          Vbid(0),
753                          10,
754                          PROTOCOL_BINARY_DATATYPE_XATTR)
755                     .first,
756             "Failed to store xattr document");
757 
758     if (isPersistentBucket(h)) {
759         wait_for_flusher_to_settle(h);
760     }
761 
762     testHarness->time_travel(11);
763 
764     cb::EngineErrorMetadataPair errorMetaPair;
765 
766     check(get_meta(h, "test_expiry", errorMetaPair, cookie),
767           "Get meta command failed");
768     auto prev_revseqno = errorMetaPair.second.seqno;
769 
770     checkeq(PROTOCOL_BINARY_DATATYPE_XATTR,
771             errorMetaPair.second.datatype,
772             "Datatype is not XATTR");
773 
774     auto ret = get(h, cookie, key, Vbid(0), DocStateFilter::AliveOrDeleted);
775     checkeq(cb::engine_errc::success,
776             ret.first,
777             "Unable to get a deleted item");
778 
779     check(get_meta(h, "test_expiry", errorMetaPair, cookie),
780           "Get meta command failed");
781 
782     checkeq(errorMetaPair.second.seqno,
783             prev_revseqno + 1,
784             "rev seqno must have incremented by 1");
785 
786     /* Retrieve the item info and create a new blob out of the data */
787     item_info info;
788     checkeq(true,
789             h->get_item_info(ret.second.get(), &info),
790             "Unable to retrieve item info");
791 
792     cb::char_buffer value_buf{static_cast<char*>(info.value[0].iov_base),
793                               info.value[0].iov_len};
794 
795     cb::xattr::Blob new_blob(value_buf, false);
796 
797     /* Only system extended attributes need to be present at this point.
798      * Thus, check the blob length with the system size.
799      */
800     const auto systemsize = new_blob.finalize().len;
801 
802     checkeq(systemsize, new_blob.get_system_size(),
803             "The size of the blob doesn't match the size of system attributes");
804 
805     const std::string& cas_str{"{\"cas\":\"0xdeadbeefcafefeed\"}"};
806     const std::string& sync_str = to_string(blob.get("_sync"));
807 
808     checkeq(cas_str, sync_str , "system xattr is invalid");
809 
810     testHarness->destroy_cookie(cookie);
811 
812     return SUCCESS;
813 }
814 
test_expiry(EngineIface* h)815 static enum test_result test_expiry(EngineIface* h) {
816     const char *key = "test_expiry";
817     const char *data = "some test data here.";
818 
819     const void* cookie = testHarness->create_cookie();
820     auto ret = allocate(h,
821                         cookie,
822                         key,
823                         strlen(data),
824                         0,
825                         2,
826                         PROTOCOL_BINARY_RAW_BYTES,
827                         Vbid(0));
828     checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
829 
830     item_info info;
831     if (!h->get_item_info(ret.second.get(), &info)) {
832         abort();
833     }
834     memcpy(info.value[0].iov_base, data, strlen(data));
835 
836     uint64_t cas = 0;
837     auto rv = h->store(cookie,
838                        ret.second.get(),
839                        cas,
840                        OPERATION_SET,
841                        {},
842                        DocumentState::Alive);
843     checkeq(ENGINE_SUCCESS, rv, "Set failed.");
844     check_key_value(h, key, data, strlen(data));
845 
846     testHarness->time_travel(5);
847     checkeq(cb::engine_errc::no_such_key,
848             get(h, cookie, key, Vbid(0)).first,
849             "Item didn't expire");
850 
851     int expired_access = get_int_stat(h, "ep_expired_access");
852     int expired_pager = get_int_stat(h, "ep_expired_pager");
853     int active_expired = get_int_stat(h, "vb_active_expired");
854     checkeq(0, expired_pager, "Expected zero expired item by pager");
855     checkeq(1, expired_access, "Expected an expired item on access");
856     checkeq(1, active_expired, "Expected an expired active item");
857     checkeq(ENGINE_SUCCESS,
858             store(h, cookie, OPERATION_SET, key, data),
859             "Failed set.");
860 
861     // When run under full eviction, the total item stats are set from the
862     // flusher. So we need to wait for it to finish before checking the
863     // total number of items.
864     wait_for_flusher_to_settle(h);
865 
866     std::stringstream ss;
867     ss << "curr_items stat should be still 1 after ";
868     ss << "overwriting the key that was expired, but not purged yet";
869     checkeq(1, get_int_stat(h, "curr_items"), ss.str().c_str());
870 
871     testHarness->destroy_cookie(cookie);
872     return SUCCESS;
873 }
874 
test_expiry_loader(EngineIface* h)875 static enum test_result test_expiry_loader(EngineIface* h) {
876     if (!isWarmupEnabled(h)) {
877         return SKIPPED;
878     }
879     const char *key = "test_expiry_loader";
880     const char *data = "some test data here.";
881 
882     const void* cookie = testHarness->create_cookie();
883     auto ret = allocate(h,
884                         cookie,
885                         key,
886                         strlen(data),
887                         0,
888                         2,
889                         PROTOCOL_BINARY_RAW_BYTES,
890                         Vbid(0));
891     checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
892 
893     item_info info;
894     if (!h->get_item_info(ret.second.get(), &info)) {
895         abort();
896     }
897     memcpy(info.value[0].iov_base, data, strlen(data));
898 
899     uint64_t cas = 0;
900     auto rv = h->store(cookie,
901                        ret.second.get(),
902                        cas,
903                        OPERATION_SET,
904                        {},
905                        DocumentState::Alive);
906     checkeq(ENGINE_SUCCESS, rv, "Set failed.");
907     check_key_value(h, key, data, strlen(data));
908 
909     testHarness->time_travel(3);
910 
911     ret = get(h, cookie, key, Vbid(0));
912     checkeq(cb::engine_errc::no_such_key, ret.first, "Item didn't expire");
913 
914     // Restart the engine to ensure the above expired item is not loaded
915     testHarness->reload_engine(&h,
916                                testHarness->engine_path,
917                                testHarness->get_current_testcase()->cfg,
918                                true,
919                                false);
920 
921     wait_for_warmup_complete(h);
922     cb_assert(0 == get_int_stat(h, "ep_warmup_value_count", "warmup"));
923 
924     testHarness->destroy_cookie(cookie);
925 
926     return SUCCESS;
927 }
928 
test_expiration_on_compaction(EngineIface* h)929 static enum test_result test_expiration_on_compaction(EngineIface* h) {
930     if (get_bool_stat(h, "ep_exp_pager_enabled")) {
931         set_param(h,
932                   cb::mcbp::request::SetParamPayload::Type::Flush,
933                   "exp_pager_enabled",
934                   "false");
935     }
936 
937     checkeq(1,
938             get_int_stat(h, "vb_0:persistence:num_visits", "checkpoint"),
939             "Cursor moved before item load");
940 
941     for (int i = 0; i < 25; i++) {
942         std::stringstream ss;
943         ss << "key" << i;
944         checkeq(ENGINE_SUCCESS,
945                 store(h,
946                       NULL,
947                       OPERATION_SET,
948                       ss.str().c_str(),
949                       "somevalue",
950                       nullptr,
951                       0,
952                       Vbid(0),
953                       10,
954                       PROTOCOL_BINARY_RAW_BYTES),
955                 "Set failed.");
956     }
957 
958     // Throw an xattr document in (and later compressed)
959     cb::xattr::Blob builder;
960     builder.set("_ep", "{\"foo\":\"bar\"}");
961     builder.set("key", "{\"foo\":\"bar\"}");
962     builder.set("_sync", "{\"foo\":\"bar\"}");
963     builder.set("stuff", "{\"foo\":\"bar\"}");
964     builder.set("misc", "{\"foo\":\"bar\"}");
965     builder.set("things", "{\"foo\":\"bar\"}");
966     builder.set("that", "{\"foo\":\"bar\"}");
967 
968     auto blob = builder.finalize();
969     std::string data;
970     std::copy(blob.buf, blob.buf + blob.size(), std::back_inserter(data));
971     for (int i = 0; i < 12; i++) {
972         std::stringstream ss;
973         ss << "xattr_key" << i;
974 
975         checkeq(cb::engine_errc::success,
976                 storeCasVb11(h,
977                              nullptr,
978                              OPERATION_SET,
979                              ss.str().c_str(),
980                              data.data(),
981                              data.size(),
982                              0,
983                              0,
984                              Vbid(0),
985                              10,
986                              PROTOCOL_BINARY_DATATYPE_XATTR,
987                              DocumentState::Alive)
988                         .first,
989                 "Unable to store item");
990     }
991 
992     cb::compression::Buffer compressedDoc;
993     cb::compression::deflate(
994             cb::compression::Algorithm::Snappy, data, compressedDoc);
995 
996     for (int i = 0; i < 13; i++) {
997         std::stringstream ss;
998         ss << "compressed_xattr_key" << i;
999 
1000         checkeq(cb::engine_errc::success,
1001                 storeCasVb11(h,
1002                              nullptr,
1003                              OPERATION_SET,
1004                              ss.str().c_str(),
1005                              compressedDoc.data(),
1006                              compressedDoc.size(),
1007                              0,
1008                              0,
1009                              Vbid(0),
1010                              10,
1011                              PROTOCOL_BINARY_DATATYPE_XATTR |
1012                                      PROTOCOL_BINARY_DATATYPE_SNAPPY,
1013                              DocumentState::Alive)
1014                         .first,
1015                 "Unable to store item");
1016     }
1017 
1018     wait_for_flusher_to_settle(h);
1019     checkeq(50,
1020             get_int_stat(h, "curr_items"),
1021             "Unexpected number of items on database");
1022     checklt(1, get_int_stat(h, "vb_0:persistence:num_visits", "checkpoint"),
1023             "Cursor not moved even after flusher runs");
1024 
1025     testHarness->time_travel(15);
1026 
1027     // Compaction on VBucket
1028     compact_db(h, Vbid(0), Vbid(0), 0, 0, 0);
1029     wait_for_stat_to_be(h, "ep_pending_compactions", 0);
1030 
1031     checkeq(50,
1032             get_int_stat(h, "ep_expired_compactor"),
1033             "Unexpected expirations by compactor");
1034 
1035     return SUCCESS;
1036 }
1037 
test_expiration_on_warmup(EngineIface* h)1038 static enum test_result test_expiration_on_warmup(EngineIface* h) {
1039     if (!isWarmupEnabled(h)) {
1040         return SKIPPED;
1041     }
1042 
1043     const auto* cookie = testHarness->create_cookie();
1044     set_param(h,
1045               cb::mcbp::request::SetParamPayload::Type::Flush,
1046               "exp_pager_enabled",
1047               "false");
1048     int pager_runs = get_int_stat(h, "ep_num_expiry_pager_runs");
1049 
1050     const char *key = "KEY";
1051     const char *data = "VALUE";
1052 
1053     auto ret = allocate(h,
1054                         cookie,
1055                         key,
1056                         strlen(data),
1057                         0,
1058                         10,
1059                         PROTOCOL_BINARY_RAW_BYTES,
1060                         Vbid(0));
1061     checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
1062 
1063     item_info info;
1064     if (!h->get_item_info(ret.second.get(), &info)) {
1065         abort();
1066     }
1067     memcpy(info.value[0].iov_base, data, strlen(data));
1068 
1069     uint64_t cas = 0;
1070     auto rv = h->store(cookie,
1071                        ret.second.get(),
1072                        cas,
1073                        OPERATION_SET,
1074                        {},
1075                        DocumentState::Alive);
1076     checkeq(ENGINE_SUCCESS, rv, "Set failed.");
1077     check_key_value(h, key, data, strlen(data));
1078     ret.second.reset();
1079     wait_for_flusher_to_settle(h);
1080 
1081     checkeq(1, get_int_stat(h, "curr_items"), "Failed store item");
1082     testHarness->time_travel(15);
1083 
1084     checkeq(pager_runs,
1085             get_int_stat(h, "ep_num_expiry_pager_runs"),
1086             "Expiry pager shouldn't have run during this time");
1087 
1088     // Restart the engine to ensure the above item is expired
1089     testHarness->reload_engine(&h,
1090                                testHarness->engine_path,
1091                                testHarness->get_current_testcase()->cfg,
1092                                true,
1093                                false);
1094 
1095     wait_for_warmup_complete(h);
1096     check(get_bool_stat(h, "ep_exp_pager_enabled"),
1097           "Expiry pager should be enabled on warmup");
1098 
1099     // Wait for the expiry pager to run and expire our item.
1100     wait_for_stat_to_be_gte(h, "ep_expired_pager", 1, nullptr, /*secs*/ 10);
1101 
1102     // Note: previously we checked that curr_items was zero here (immediately
1103     // after waiting for ep_expired_pager == 1), however we cannot assume that
1104     // - items are actually expired asynchronously.
1105     // See EPStore::deleteExpiredItem - for non-temporary, expired items we
1106     // call processSoftDelete (soft-marking the item as deleted in the
1107     // hashtable), and then call queueDirty to queue a deletion, and then
1108     // increment the expired stat. Only when that delete is actually persisted
1109     // and the deleted callback is invoked -
1110     // PeristenceCallback::callback(int&) - is curr_items finally decremented.
1111     // Therefore we need to wait for the flusher to settle (i.e. delete
1112     // callback to be called) for the curr_items stat to be accurate.
1113     wait_for_flusher_to_settle(h);
1114 
1115     checkeq(0,
1116             get_int_stat(h, "curr_items"),
1117             "The item should have been expired.");
1118 
1119     testHarness->destroy_cookie(cookie);
1120     return SUCCESS;
1121 }
1122 
test_bug3454(EngineIface* h)1123 static enum test_result test_bug3454(EngineIface* h) {
1124     if (!isWarmupEnabled(h)) {
1125         return SKIPPED;
1126     }
1127 
1128     const char *key = "test_expiry_duplicate_warmup";
1129     const char *data = "some test data here.";
1130 
1131     const void* cookie = testHarness->create_cookie();
1132     auto ret = allocate(h,
1133                         cookie,
1134                         key,
1135                         strlen(data),
1136                         0,
1137                         5,
1138                         PROTOCOL_BINARY_RAW_BYTES,
1139                         Vbid(0));
1140     checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
1141 
1142     item_info info;
1143     if (!h->get_item_info(ret.second.get(), &info)) {
1144         abort();
1145     }
1146     memcpy(info.value[0].iov_base, data, strlen(data));
1147 
1148     uint64_t cas = 0;
1149     auto rv = h->store(cookie,
1150                        ret.second.get(),
1151                        cas,
1152                        OPERATION_SET,
1153                        {},
1154                        DocumentState::Alive);
1155     checkeq(ENGINE_SUCCESS, rv, "Set failed.");
1156     check_key_value(h, key, data, strlen(data));
1157     wait_for_flusher_to_settle(h);
1158 
1159     // Advance the ep_engine time by 10 sec for the above item to be expired.
1160     testHarness->time_travel(10);
1161     ret = get(h, cookie, key, Vbid(0));
1162     checkeq(cb::engine_errc::no_such_key, ret.first, "Item didn't expire");
1163 
1164     ret = allocate(h,
1165                    cookie,
1166                    key,
1167                    strlen(data),
1168                    0,
1169                    0,
1170                    PROTOCOL_BINARY_RAW_BYTES,
1171                    Vbid(0));
1172     checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
1173 
1174     if (!h->get_item_info(ret.second.get(), &info)) {
1175         abort();
1176     }
1177     memcpy(info.value[0].iov_base, data, strlen(data));
1178 
1179     cas = 0;
1180     // Add a new item with the same key.
1181     rv = h->store(cookie,
1182                   ret.second.get(),
1183                   cas,
1184                   OPERATION_ADD,
1185                   {},
1186                   DocumentState::Alive);
1187     checkeq(ENGINE_SUCCESS, rv, "Add failed.");
1188     check_key_value(h, key, data, strlen(data));
1189     ret.second.reset();
1190     wait_for_flusher_to_settle(h);
1191 
1192     checkeq(cb::engine_errc::success,
1193             get(h, cookie, key, Vbid(0)).first,
1194             "Item shouldn't expire");
1195 
1196     // Restart the engine to ensure the above unexpired new item is loaded
1197     testHarness->reload_engine(&h,
1198                                testHarness->engine_path,
1199                                testHarness->get_current_testcase()->cfg,
1200                                true,
1201                                false);
1202 
1203     wait_for_warmup_complete(h);
1204     cb_assert(1 == get_int_stat(h, "ep_warmup_value_count", "warmup"));
1205     cb_assert(0 == get_int_stat(h, "ep_warmup_dups", "warmup"));
1206 
1207     testHarness->destroy_cookie(cookie);
1208     return SUCCESS;
1209 }
1210 
test_bug3522(EngineIface* h)1211 static enum test_result test_bug3522(EngineIface* h) {
1212     if (!isWarmupEnabled(h)) {
1213         return SKIPPED;
1214     }
1215 
1216     const char *key = "test_expiry_no_items_warmup";
1217     const char *data = "some test data here.";
1218 
1219     const void* cookie = testHarness->create_cookie();
1220     auto ret = allocate(h,
1221                         cookie,
1222                         key,
1223                         strlen(data),
1224                         0,
1225                         0,
1226                         PROTOCOL_BINARY_RAW_BYTES,
1227                         Vbid(0));
1228     checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
1229 
1230     item_info info;
1231     if (!h->get_item_info(ret.second.get(), &info)) {
1232         abort();
1233     }
1234     memcpy(info.value[0].iov_base, data, strlen(data));
1235 
1236     uint64_t cas = 0;
1237     auto rv = h->store(cookie,
1238                        ret.second.get(),
1239                        cas,
1240                        OPERATION_SET,
1241                        {},
1242                        DocumentState::Alive);
1243     checkeq(ENGINE_SUCCESS, rv, "Set failed.");
1244     check_key_value(h, key, data, strlen(data));
1245     wait_for_flusher_to_settle(h);
1246 
1247     // Add a new item with the same key and 2 sec of expiration.
1248     const char *new_data = "new data here.";
1249     ret = allocate(h,
1250                    cookie,
1251                    key,
1252                    strlen(new_data),
1253                    0,
1254                    2,
1255                    PROTOCOL_BINARY_RAW_BYTES,
1256                    Vbid(0));
1257     checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
1258 
1259     if (!h->get_item_info(ret.second.get(), &info)) {
1260         abort();
1261     }
1262     memcpy(info.value[0].iov_base, new_data, strlen(new_data));
1263 
1264     int pager_runs = get_int_stat(h, "ep_num_expiry_pager_runs");
1265     cas = 0;
1266     rv = h->store(cookie,
1267                   ret.second.get(),
1268                   cas,
1269                   OPERATION_SET,
1270                   {},
1271                   DocumentState::Alive);
1272     checkeq(ENGINE_SUCCESS, rv, "Set failed.");
1273     check_key_value(h, key, new_data, strlen(new_data));
1274     ret.second.reset();
1275     testHarness->time_travel(3);
1276     wait_for_stat_change(h, "ep_num_expiry_pager_runs", pager_runs);
1277     wait_for_flusher_to_settle(h);
1278 
1279     // Restart the engine.
1280     testHarness->reload_engine(&h,
1281                                testHarness->engine_path,
1282                                testHarness->get_current_testcase()->cfg,
1283                                true,
1284                                false);
1285 
1286     wait_for_warmup_complete(h);
1287     // TODO: modify this for a better test case
1288     cb_assert(0 == get_int_stat(h, "ep_warmup_dups", "warmup"));
1289 
1290     testHarness->destroy_cookie(cookie);
1291     return SUCCESS;
1292 }
1293 
test_get_replica_active_state(EngineIface* h)1294 static enum test_result test_get_replica_active_state(EngineIface* h) {
1295     auto pkt = prepare_get_replica(h, vbucket_state_active);
1296     checkeq(ENGINE_NOT_MY_VBUCKET,
1297             h->unknown_command(NULL, *pkt, add_response),
1298             "Get Replica Failed");
1299 
1300     return SUCCESS;
1301 }
1302 
test_get_replica_pending_state(EngineIface* h)1303 static enum test_result test_get_replica_pending_state(EngineIface* h) {
1304     const void* cookie = testHarness->create_cookie();
1305     testHarness->set_ewouldblock_handling(cookie, false);
1306     auto pkt = prepare_get_replica(h, vbucket_state_pending);
1307     checkeq(ENGINE_NOT_MY_VBUCKET,
1308             h->unknown_command(cookie, *pkt, add_response),
1309             "Should have returned NOT_MY_VBUCKET for pending state");
1310     checkeq(1, get_int_stat(h, "ep_num_not_my_vbuckets"), "Expected 1 get");
1311     testHarness->destroy_cookie(cookie);
1312     return SUCCESS;
1313 }
1314 
test_get_replica_dead_state(EngineIface* h)1315 static enum test_result test_get_replica_dead_state(EngineIface* h) {
1316     auto pkt = prepare_get_replica(h, vbucket_state_dead);
1317     checkeq(ENGINE_NOT_MY_VBUCKET,
1318             h->unknown_command(NULL, *pkt, add_response),
1319             "Get Replica Failed");
1320     return SUCCESS;
1321 }
1322 
test_get_replica(EngineIface* h)1323 static enum test_result test_get_replica(EngineIface* h) {
1324     auto pkt = prepare_get_replica(h, vbucket_state_replica);
1325     checkeq(ENGINE_SUCCESS,
1326             h->unknown_command(NULL, *pkt, add_response),
1327             "Get Replica Failed");
1328     checkeq(cb::mcbp::Status::Success, last_status.load(),
1329             "Expected cb::mcbp::Status::Success response.");
1330     checkeq("replicadata"s, last_body, "Should have returned identical value");
1331     checkeq(1, get_int_stat(h, "vb_replica_ops_get"), "Expected 1 get");
1332 
1333     return SUCCESS;
1334 }
1335 
test_get_replica_non_resident(EngineIface* h)1336 static enum test_result test_get_replica_non_resident(EngineIface* h) {
1337     checkeq(ENGINE_SUCCESS,
1338             store(h, NULL, OPERATION_SET, "key", "value"),
1339             "Store Failed");
1340     wait_for_flusher_to_settle(h);
1341     wait_for_stat_to_be(h, "ep_total_persisted", 1);
1342 
1343     evict_key(h, "key", Vbid(0), "Ejected.");
1344     check(set_vbucket_state(h, Vbid(0), vbucket_state_replica),
1345           "Failed to set vbucket to replica");
1346 
1347     get_replica(h, "key", Vbid(0));
1348     checkeq(cb::mcbp::Status::Success, last_status.load(),
1349             "Expected success");
1350     checkeq(1, get_int_stat(h, "vb_replica_ops_get"), "Expected 1 get");
1351 
1352     return SUCCESS;
1353 }
1354 
test_get_replica_invalid_key(EngineIface* h)1355 static enum test_result test_get_replica_invalid_key(EngineIface* h) {
1356     bool makeinvalidkey = true;
1357     auto pkt = prepare_get_replica(h, vbucket_state_replica, makeinvalidkey);
1358     checkeq(ENGINE_NOT_MY_VBUCKET,
1359             h->unknown_command(NULL, *pkt, add_response),
1360             "Get Replica Failed");
1361     return SUCCESS;
1362 }
1363 
test_vb_del_pending(EngineIface* h)1364 static enum test_result test_vb_del_pending(EngineIface* h) {
1365     const void* cookie = testHarness->create_cookie();
1366     testHarness->set_ewouldblock_handling(cookie, false);
1367     check(set_vbucket_state(h, Vbid(1), vbucket_state_pending),
1368           "Failed to set vbucket state.");
1369     checkeq(ENGINE_EWOULDBLOCK,
1370             del(h, "key", 0, Vbid(1), cookie),
1371             "Expected woodblock.");
1372     testHarness->destroy_cookie(cookie);
1373     return SUCCESS;
1374 }
1375 
test_vb_del_replica(EngineIface* h)1376 static enum test_result test_vb_del_replica(EngineIface* h) {
1377     check(set_vbucket_state(h, Vbid(1), vbucket_state_replica),
1378           "Failed to set vbucket state.");
1379     int numNotMyVBucket = get_int_stat(h, "ep_num_not_my_vbuckets");
1380     checkeq(ENGINE_NOT_MY_VBUCKET,
1381             del(h, "key", 0, Vbid(1)),
1382             "Expected not my vbucket.");
1383     wait_for_stat_change(h, "ep_num_not_my_vbuckets", numNotMyVBucket);
1384     return SUCCESS;
1385 }
1386 
test_vbucket_get_miss(EngineIface* h)1387 static enum test_result test_vbucket_get_miss(EngineIface* h) {
1388     return verify_vbucket_missing(h, Vbid(1)) ? SUCCESS : FAIL;
1389 }
1390 
test_vbucket_get(EngineIface* h)1391 static enum test_result test_vbucket_get(EngineIface* h) {
1392     return verify_vbucket_state(h, Vbid(0), vbucket_state_active) ? SUCCESS
1393                                                                   : FAIL;
1394 }
1395 
test_vbucket_create(EngineIface* h)1396 static enum test_result test_vbucket_create(EngineIface* h) {
1397     if (!verify_vbucket_missing(h, Vbid(1))) {
1398         fprintf(stderr, "vbucket wasn't missing.\n");
1399         return FAIL;
1400     }
1401 
1402     if (!set_vbucket_state(h, Vbid(1), vbucket_state_active)) {
1403         fprintf(stderr, "set state failed.\n");
1404         return FAIL;
1405     }
1406 
1407     return verify_vbucket_state(h, Vbid(1), vbucket_state_active) ? SUCCESS
1408                                                                   : FAIL;
1409 }
1410 
test_takeover_stats_race_with_vb_create_DCP( EngineIface* h)1411 static enum test_result test_takeover_stats_race_with_vb_create_DCP(
1412         EngineIface* h) {
1413     check(set_vbucket_state(h, Vbid(1), vbucket_state_active),
1414           "Failed to set vbucket state information");
1415 
1416     checkeq(0,
1417             get_int_stat(h, "on_disk_deletes", "dcp-vbtakeover 1"),
1418             "Invalid number of on-disk deletes");
1419 
1420     return SUCCESS;
1421 }
1422 
test_takeover_stats_num_persisted_deletes( EngineIface* h)1423 static enum test_result test_takeover_stats_num_persisted_deletes(
1424         EngineIface* h) {
1425     /* set an item */
1426     std::string key("key");
1427     checkeq(ENGINE_SUCCESS,
1428             store(h, NULL, OPERATION_SET, key.c_str(), "data"),
1429             "Failed to store an item");
1430 
1431     /* delete the item */
1432     checkeq(ENGINE_SUCCESS,
1433             del(h, key.c_str(), 0, Vbid(0)),
1434             "Failed to delete the item");
1435 
1436     /* wait for persistence */
1437     wait_for_flusher_to_settle(h);
1438 
1439     /* check if persisted deletes stats is got correctly */
1440     checkeq(1,
1441             get_int_stat(h, "on_disk_deletes", "dcp-vbtakeover 0"),
1442             "Invalid number of on-disk deletes");
1443 
1444     return SUCCESS;
1445 }
1446 
test_vbucket_compact(EngineIface* h)1447 static enum test_result test_vbucket_compact(EngineIface* h) {
1448     const char* exp_key = "Carss";
1449     const char* exp_value = "pollute";
1450     const char* non_exp_key = "trees";
1451     const char* non_exp_value = "cleanse";
1452 
1453     // Set two keys - one to be expired and other to remain...
1454     // Set expiring key
1455     checkeq(ENGINE_SUCCESS,
1456             store(h, NULL, OPERATION_SET, exp_key, exp_value),
1457             "Failed to set expiring key");
1458     check_key_value(h, exp_key, exp_value, strlen(exp_value));
1459 
1460     // Set a non-expiring key...
1461     checkeq(ENGINE_SUCCESS,
1462             store(h, NULL, OPERATION_SET, non_exp_key, non_exp_value),
1463             "Failed to set non-expiring key");
1464     check_key_value(h, non_exp_key, non_exp_value, strlen(non_exp_value));
1465 
1466     // Touch expiring key with an expire time
1467     const int exp_time = 11;
1468     checkeq(ENGINE_SUCCESS,
1469             touch(h, exp_key, Vbid(0), exp_time),
1470             "Touch expiring key failed");
1471 
1472     // Move beyond expire time
1473     testHarness->time_travel(exp_time + 1);
1474 
1475     // Touch the expiring key to trigger the expiry
1476     const void* cookie = testHarness->create_cookie();
1477     checkeq(cb::engine_errc::no_such_key,
1478             get(h, cookie, exp_key, Vbid(0), DocStateFilter::Alive).first,
1479             "Expiration trigger via touch failed");
1480     testHarness->destroy_cookie(cookie);
1481 
1482     // Expect the item to now be expired
1483     checkeq(1,
1484             get_int_stat(h, "vb_active_expired"),
1485             "Incorrect number of expired keys");
1486     const int exp_purge_seqno =
1487             get_int_stat(h, "vb_0:high_seqno", "vbucket-seqno");
1488 
1489     // non_exp_key and its value should be intact...
1490     checkeq(ENGINE_SUCCESS,
1491             verify_key(h, non_exp_key),
1492             "key trees should be found.");
1493     // exp_key should have disappeared...
1494     ENGINE_ERROR_CODE val = verify_key(h, exp_key);
1495     checkeq(ENGINE_KEY_ENOENT, val, "Key Carss has not expired.");
1496 
1497     // Store a dummy item since we do not purge the item with highest seqno
1498     checkeq(ENGINE_SUCCESS,
1499             store(h, NULL, OPERATION_SET, "dummykey", "dummyvalue"),
1500             "Error setting dummy key");
1501     wait_for_flusher_to_settle(h);
1502 
1503     checkeq(0,
1504             get_int_stat(h, "vb_0:purge_seqno", "vbucket-seqno"),
1505             "purge_seqno not found to be zero before compaction");
1506 
1507     // Compaction on VBucket
1508     compact_db(
1509             h,
1510             Vbid(0) /* vbucket_id */,
1511             Vbid(0) /* db_file_id */,
1512             2 /* purge_before_ts */,
1513             exp_purge_seqno - 1 /* purge_before_seq */,
1514             1 /* drop deletes (forces purge irrespective purge_before_seq) */);
1515     wait_for_stat_to_be(h, "ep_pending_compactions", 0);
1516     checkeq(exp_purge_seqno,
1517             get_int_stat(h, "vb_0:purge_seqno", "vbucket-seqno"),
1518             "purge_seqno didn't match expected value");
1519 
1520     return SUCCESS;
1521 }
1522 
test_compaction_config(EngineIface* h)1523 static enum test_result test_compaction_config(EngineIface* h) {
1524     checkeq(10000,
1525             get_int_stat(h, "ep_compaction_write_queue_cap"),
1526             "Expected compaction queue cap to be 10000");
1527     set_param(h,
1528               cb::mcbp::request::SetParamPayload::Type::Flush,
1529               "compaction_write_queue_cap",
1530               "100000");
1531     checkeq(100000,
1532             get_int_stat(h, "ep_compaction_write_queue_cap"),
1533             "Expected compaction queue cap to be 100000");
1534     return SUCCESS;
1535 }
1536 
test_MB_33919(EngineIface* h)1537 static enum test_result test_MB_33919(EngineIface* h) {
1538     const char* expKey = "expiryKey";
1539     const char* otherKey = "otherKey";
1540     const char* value = "value";
1541 
1542     // Test runs in the future as we need to set times in the past without
1543     // falling foul of the expiry time being behind server start
1544     auto fiveDays =
1545             std::chrono::system_clock::now() + std::chrono::hours(5 * 24);
1546 
1547     // Calculate expiry and purgeBefore times
1548     auto threeDaysAgo = std::chrono::system_clock::to_time_t(
1549             fiveDays - std::chrono::hours(3 * 24));
1550     auto fourDaysAgo = std::chrono::system_clock::to_time_t(
1551             fiveDays - std::chrono::hours(4 * 24));
1552     // Time travel forwards by 5 days to give the illusion of 5 days uptime.
1553     testHarness->time_travel((86400 * 5));
1554 
1555     // Set expiring key, expiry 4 days ago
1556     checkeq(ENGINE_SUCCESS,
1557             store(h,
1558                   nullptr,
1559                   OPERATION_SET,
1560                   expKey,
1561                   value,
1562                   nullptr,
1563                   0,
1564                   Vbid(0),
1565                   fourDaysAgo),
1566             "Failed to set expiring key");
1567 
1568     // Force it to expire
1569     ENGINE_ERROR_CODE val = verify_key(h, expKey);
1570     checkeq(ENGINE_KEY_ENOENT, val, "expiryKey has not expired.");
1571 
1572     // And a second key (after expiry), compaction won't purge the high-seqno
1573     checkeq(ENGINE_SUCCESS,
1574             store(h, nullptr, OPERATION_SET, otherKey, value),
1575             "Failed to set non-expiring key");
1576 
1577     // Wait for the item to be expired
1578     wait_for_stat_to_be(h, "vb_active_expired", 1);
1579 
1580     wait_for_flusher_to_settle(h);
1581 
1582     checkeq(0,
1583             get_int_stat(h, "vb_0:purge_seqno", "vbucket-seqno"),
1584             "purge_seqno not found to be zero before compaction");
1585 
1586     // Compaction on VBucket
1587     compact_db(h, Vbid(0), Vbid(0), threeDaysAgo, 0, 0);
1588     wait_for_stat_to_be(h, "ep_pending_compactions", 0);
1589 
1590     // Purge-seqno should not of moved, without the fix it would
1591     checkeq(0,
1592             get_int_stat(h, "vb_0:purge_seqno", "vbucket-seqno"),
1593             "purge_seqno not found to be zero before compaction");
1594 
1595     return SUCCESS;
1596 }
1597 
1598 struct comp_thread_ctx {
1599     EngineIface* h;
1600     Vbid vbid;
1601     Vbid db_file_id;
1602 };
1603 
1604 void makeCouchstoreFileInaccessible(const std::string& dbname);
1605 extern "C" {
compaction_thread(void *arg)1606     static void compaction_thread(void *arg) {
1607         struct comp_thread_ctx *ctx = static_cast<comp_thread_ctx *>(arg);
1608         compact_db(ctx->h, ctx->vbid, ctx->db_file_id, 0, 0, 0);
1609     }
1610 }
1611 
test_multiple_vb_compactions(EngineIface* h)1612 static enum test_result test_multiple_vb_compactions(EngineIface* h) {
1613     for (uint16_t i = 0; i < 4; ++i) {
1614         if (!set_vbucket_state(h, Vbid(i), vbucket_state_active)) {
1615             fprintf(stderr, "set state failed for vbucket %d.\n", i);
1616             return FAIL;
1617         }
1618         check(verify_vbucket_state(h, Vbid(i), vbucket_state_active),
1619               "VBucket state not active");
1620     }
1621 
1622     std::vector<std::string> keys;
1623     for (int j = 0; j < 100; ++j) {
1624         std::stringstream ss;
1625         ss << "key" << j;
1626         std::string key(ss.str());
1627         keys.push_back(key);
1628     }
1629 
1630     int count = 0;
1631     std::vector<std::string>::iterator it;
1632     for (it = keys.begin(); it != keys.end(); ++it) {
1633         Vbid vbid = Vbid(count % 4);
1634         checkeq(ENGINE_SUCCESS,
1635                 store(h,
1636                       NULL,
1637                       OPERATION_SET,
1638                       it->c_str(),
1639                       it->c_str(),
1640                       nullptr,
1641                       0,
1642                       vbid),
1643                 "Failed to store a value");
1644         ++count;
1645     }
1646 
1647     // Compact multiple vbuckets.
1648     const int n_threads = 4;
1649     cb_thread_t threads[n_threads];
1650     struct comp_thread_ctx ctx[n_threads];
1651 
1652     const int num_shards =
1653             get_int_stat(h, "ep_workload:num_shards", "workload");
1654 
1655     for (int i = 0; i < n_threads; i++) {
1656         ctx[i].h = h;
1657         ctx[i].vbid = static_cast<Vbid>(i);
1658         ctx[i].db_file_id = Vbid(ctx[i].vbid.get() % num_shards);
1659         int r = cb_create_thread(&threads[i], compaction_thread, &ctx[i], 0);
1660         cb_assert(r == 0);
1661     }
1662 
1663     for (int i = 0; i < n_threads; i++) {
1664         int r = cb_join_thread(threads[i]);
1665         cb_assert(r == 0);
1666     }
1667 
1668     wait_for_stat_to_be(h, "ep_pending_compactions", 0);
1669 
1670     return SUCCESS;
1671 }
1672 
test_multi_vb_compactions_with_workload( EngineIface* h)1673 static enum test_result test_multi_vb_compactions_with_workload(
1674         EngineIface* h) {
1675     for (uint16_t i = 0; i < 4; ++i) {
1676         if (!set_vbucket_state(h, Vbid(i), vbucket_state_active)) {
1677             fprintf(stderr, "set state failed for vbucket %d.\n", i);
1678             return FAIL;
1679         }
1680         check(verify_vbucket_state(h, Vbid(i), vbucket_state_active),
1681               "VBucket state not active");
1682     }
1683 
1684     std::vector<std::string> keys;
1685     for (int j = 0; j < 100; ++j) {
1686         std::stringstream ss;
1687         ss << "key" << j;
1688         std::string key(ss.str());
1689         keys.push_back(key);
1690     }
1691 
1692     int count = 0;
1693     std::vector<std::string>::iterator it;
1694     for (it = keys.begin(); it != keys.end(); ++it) {
1695         Vbid vbid = Vbid(count % 4);
1696         checkeq(ENGINE_SUCCESS,
1697                 store(h,
1698                       NULL,
1699                       OPERATION_SET,
1700                       it->c_str(),
1701                       it->c_str(),
1702                       nullptr,
1703                       0,
1704                       vbid),
1705                 "Failed to store a value");
1706         ++count;
1707     }
1708     wait_for_flusher_to_settle(h);
1709 
1710     for (int i = 0; i < 2; ++i) {
1711         count = 0;
1712         for (it = keys.begin(); it != keys.end(); ++it) {
1713             Vbid vbid = Vbid(count % 4);
1714             checkeq(cb::engine_errc::success,
1715                     get(h, NULL, it->c_str(), vbid).first,
1716                     "Unable to get stored item");
1717             ++count;
1718         }
1719     }
1720     wait_for_stat_to_be(h, "ep_workload_pattern", std::string{"read_heavy"});
1721 
1722     // Compact multiple vbuckets.
1723     const int n_threads = 4;
1724     cb_thread_t threads[n_threads];
1725     struct comp_thread_ctx ctx[n_threads];
1726 
1727     for (int i = 0; i < n_threads; i++) {
1728         ctx[i].h = h;
1729         ctx[i].vbid = static_cast<Vbid>(i);
1730         int r = cb_create_thread(&threads[i], compaction_thread, &ctx[i], 0);
1731         cb_assert(r == 0);
1732     }
1733 
1734     for (int i = 0; i < n_threads; i++) {
1735         int r = cb_join_thread(threads[i]);
1736         cb_assert(r == 0);
1737     }
1738 
1739     wait_for_stat_to_be(h, "ep_pending_compactions", 0);
1740 
1741     return SUCCESS;
1742 }
1743 
vbucket_destroy(EngineIface* h, const char* value = NULL)1744 static enum test_result vbucket_destroy(EngineIface* h,
1745                                         const char* value = NULL) {
1746     check(set_vbucket_state(h, Vbid(1), vbucket_state_active),
1747           "Failed to set vbucket state.");
1748 
1749     checkeq(ENGINE_NOT_MY_VBUCKET,
1750             vbucketDelete(h, Vbid(2), value),
1751             "Expected NMVB");
1752 
1753     check(set_vbucket_state(h, Vbid(1), vbucket_state_dead),
1754           "Failed set set vbucket 1 state.");
1755 
1756     checkeq(ENGINE_SUCCESS,
1757             vbucketDelete(h, Vbid(1), value),
1758             "Expected success");
1759     checkeq(cb::mcbp::Status::Success, last_status.load(),
1760             "Expected failure deleting non-existent bucket.");
1761 
1762     check(verify_vbucket_missing(h, Vbid(1)),
1763           "vbucket 1 was not missing after deleting it.");
1764 
1765     return SUCCESS;
1766 }
1767 
test_vbucket_destroy_stats(EngineIface* h)1768 static enum test_result test_vbucket_destroy_stats(EngineIface* h) {
1769     int cacheSize = get_int_stat(h, "ep_total_cache_size");
1770     int overhead = get_int_stat(h, "ep_overhead");
1771     int nonResident = get_int_stat(h, "ep_num_non_resident");
1772 
1773     check(set_vbucket_state(h, Vbid(1), vbucket_state_active),
1774           "Failed to set vbucket state.");
1775 
1776     std::vector<std::string> keys;
1777     for (int j = 0; j < 2000; ++j) {
1778         std::stringstream ss;
1779         ss << "key" << j;
1780         std::string key(ss.str());
1781         keys.push_back(key);
1782     }
1783 
1784     int itemsRemoved = get_int_stat(h, "ep_items_rm_from_checkpoints");
1785     std::vector<std::string>::iterator it;
1786     for (it = keys.begin(); it != keys.end(); ++it) {
1787         checkeq(ENGINE_SUCCESS,
1788                 store(h,
1789                       NULL,
1790                       OPERATION_SET,
1791                       it->c_str(),
1792                       it->c_str(),
1793                       nullptr,
1794                       0,
1795                       Vbid(1)),
1796                 "Failed to store a value");
1797     }
1798     wait_for_flusher_to_settle(h);
1799     testHarness->time_travel(65);
1800     wait_for_stat_change(h, "ep_items_rm_from_checkpoints", itemsRemoved);
1801 
1802     check(set_vbucket_state(h, Vbid(1), vbucket_state_dead),
1803           "Failed set set vbucket 1 state.");
1804 
1805     int vbucketDel = get_int_stat(h, "ep_vbucket_del");
1806     checkeq(ENGINE_SUCCESS, vbucketDelete(h, Vbid(1)), "Expected success");
1807     checkeq(cb::mcbp::Status::Success,
1808             last_status.load(),
1809             "Expected failure deleting non-existent bucket.");
1810 
1811     check(verify_vbucket_missing(h, Vbid(1)),
1812           "vbucket 1 was not missing after deleting it.");
1813 
1814     wait_for_stat_change(h, "ep_vbucket_del", vbucketDel);
1815 
1816     wait_for_stat_to_be(h, "ep_total_cache_size", cacheSize);
1817     wait_for_stat_to_be(h, "ep_overhead", overhead);
1818     wait_for_stat_to_be(h, "ep_num_non_resident", nonResident);
1819 
1820     return SUCCESS;
1821 }
1822 
vbucket_destroy_restart(EngineIface* h, const char* value = NULL)1823 static enum test_result vbucket_destroy_restart(EngineIface* h,
1824                                                 const char* value = NULL) {
1825     if (!isWarmupEnabled(h)) {
1826         return SKIPPED;
1827     }
1828 
1829     check(set_vbucket_state(h, Vbid(1), vbucket_state_active),
1830           "Failed to set vbucket state.");
1831 
1832     // Store a value so the restart will try to resurrect it.
1833     checkeq(ENGINE_SUCCESS,
1834             store(h,
1835                   NULL,
1836                   OPERATION_SET,
1837                   "key",
1838                   "somevalue",
1839                   nullptr,
1840                   0,
1841                   Vbid(1)),
1842             "Failed to set a value");
1843     check_key_value(h, "key", "somevalue", 9, Vbid(1));
1844 
1845     // Reload to get a flush forced.
1846     testHarness->reload_engine(&h,
1847                                testHarness->engine_path,
1848                                testHarness->get_current_testcase()->cfg,
1849                                true,
1850                                false);
1851 
1852     wait_for_warmup_complete(h);
1853 
1854     check(verify_vbucket_state(h, Vbid(1), vbucket_state_active),
1855           "Bucket state was what it was initially, after restart.");
1856     check(set_vbucket_state(h, Vbid(1), vbucket_state_active),
1857           "Failed to set vbucket state.");
1858     check_key_value(h, "key", "somevalue", 9, Vbid(1));
1859 
1860     check(set_vbucket_state(h, Vbid(1), vbucket_state_dead),
1861           "Failed set set vbucket 1 state.");
1862 
1863     checkeq(ENGINE_SUCCESS,
1864             vbucketDelete(h, Vbid(1), value),
1865             "Expected success");
1866     checkeq(cb::mcbp::Status::Success, last_status.load(),
1867             "Expected failure deleting non-existent bucket.");
1868 
1869     check(verify_vbucket_missing(h, Vbid(1)),
1870           "vbucket 1 was not missing after deleting it.");
1871 
1872     testHarness->reload_engine(&h,
1873                                testHarness->engine_path,
1874                                testHarness->get_current_testcase()->cfg,
1875                                true,
1876                                false);
1877 
1878     wait_for_warmup_complete(h);
1879 
1880     if (verify_vbucket_state(h, Vbid(1), vbucket_state_pending, true)) {
1881         std::cerr << "Bucket came up in pending state after delete." << std::endl;
1882         abort();
1883     }
1884 
1885     check(verify_vbucket_missing(h, Vbid(1)),
1886           "vbucket 1 was not missing after restart.");
1887 
1888     return SUCCESS;
1889 }
1890 
test_async_vbucket_destroy(EngineIface* h)1891 static enum test_result test_async_vbucket_destroy(EngineIface* h) {
1892     return vbucket_destroy(h);
1893 }
1894 
test_sync_vbucket_destroy(EngineIface* h)1895 static enum test_result test_sync_vbucket_destroy(EngineIface* h) {
1896     return vbucket_destroy(h, "async=0");
1897 }
1898 
test_async_vbucket_destroy_restart(EngineIface* h)1899 static enum test_result test_async_vbucket_destroy_restart(EngineIface* h) {
1900     return vbucket_destroy_restart(h);
1901 }
1902 
test_sync_vbucket_destroy_restart(EngineIface* h)1903 static enum test_result test_sync_vbucket_destroy_restart(EngineIface* h) {
1904     return vbucket_destroy_restart(h, "async=0");
1905 }
1906 
test_vb_set_pending(EngineIface* h)1907 static enum test_result test_vb_set_pending(EngineIface* h) {
1908     return test_pending_vb_mutation(h, OPERATION_SET);
1909 }
1910 
test_vb_add_pending(EngineIface* h)1911 static enum test_result test_vb_add_pending(EngineIface* h) {
1912     return test_pending_vb_mutation(h, OPERATION_ADD);
1913 }
1914 
test_vb_cas_pending(EngineIface* h)1915 static enum test_result test_vb_cas_pending(EngineIface* h) {
1916     return test_pending_vb_mutation(h, OPERATION_CAS);
1917 }
1918 
test_vb_set_replica(EngineIface* h)1919 static enum test_result test_vb_set_replica(EngineIface* h) {
1920     return test_replica_vb_mutation(h, OPERATION_SET);
1921 }
1922 
test_vb_replace_replica(EngineIface* h)1923 static enum test_result test_vb_replace_replica(EngineIface* h) {
1924     return test_replica_vb_mutation(h, OPERATION_REPLACE);
1925 }
1926 
test_vb_replace_pending(EngineIface* h)1927 static enum test_result test_vb_replace_pending(EngineIface* h) {
1928     return test_pending_vb_mutation(h, OPERATION_REPLACE);
1929 }
1930 
test_vb_add_replica(EngineIface* h)1931 static enum test_result test_vb_add_replica(EngineIface* h) {
1932     return test_replica_vb_mutation(h, OPERATION_ADD);
1933 }
1934 
test_vb_cas_replica(EngineIface* h)1935 static enum test_result test_vb_cas_replica(EngineIface* h) {
1936     return test_replica_vb_mutation(h, OPERATION_CAS);
1937 }
1938 
test_stats_seqno(EngineIface* h)1939 static enum test_result test_stats_seqno(EngineIface* h) {
1940     check(set_vbucket_state(h, Vbid(1), vbucket_state_active),
1941           "Failed to set vbucket state.");
1942 
1943     int num_keys = 100;
1944     for (int ii = 0; ii < num_keys; ++ii) {
1945         std::stringstream ss;
1946         ss << "key" << ii;
1947         checkeq(ENGINE_SUCCESS,
1948                 store(h,
1949                       NULL,
1950                       OPERATION_SET,
1951                       ss.str().c_str(),
1952                       "value",
1953                       NULL,
1954                       0,
1955                       Vbid(0)),
1956                 "Failed to store an item.");
1957     }
1958     wait_for_flusher_to_settle(h);
1959 
1960     checkeq(100,
1961             get_int_stat(h, "vb_0:high_seqno", "vbucket-seqno"),
1962             "Invalid seqno");
1963 
1964     if (isPersistentBucket(h)) {
1965         checkeq(100,
1966                 get_int_stat(h, "vb_0:last_persisted_seqno", "vbucket-seqno"),
1967                 "Unexpected last_persisted_seqno");
1968     }
1969     checkeq(0,
1970             get_int_stat(h, "vb_1:high_seqno", "vbucket-seqno"),
1971             "Invalid seqno");
1972     checkeq(0,
1973             get_int_stat(h, "vb_1:high_seqno", "vbucket-seqno 1"),
1974             "Invalid seqno");
1975     if (isPersistentBucket(h)) {
1976         checkeq(0,
1977                 get_int_stat(h, "vb_1:last_persisted_seqno", "vbucket-seqno 1"),
1978                 "Invalid last_persisted_seqno");
1979     }
1980 
1981     uint64_t vb_uuid = get_ull_stat(h, "vb_1:0:id", "failovers");
1982 
1983     auto seqno_stats = get_all_stats(h, "vbucket-seqno 1");
1984     checkeq(vb_uuid, uint64_t(std::stoull(seqno_stats.at("vb_1:uuid"))),
1985             "Invalid uuid");
1986 
1987     // Check invalid vbucket
1988     checkeq(ENGINE_NOT_MY_VBUCKET,
1989             get_stats(h, "vbucket-seqno 2"_ccb, {}, add_stats),
1990             "Expected not my vbucket");
1991 
1992     // Check bad vbucket parameter (not numeric)
1993     checkeq(ENGINE_EINVAL,
1994             get_stats(h, "vbucket-seqno tt2"_ccb, {}, add_stats),
1995             "Expected invalid");
1996 
1997     // Check extra spaces at the end
1998     checkeq(ENGINE_EINVAL,
1999             get_stats(h, "vbucket-seqno    "_ccb, {}, add_stats),
2000             "Expected invalid");
2001 
2002     return SUCCESS;
2003 }
2004 
test_stats_diskinfo(EngineIface* h)2005 static enum test_result test_stats_diskinfo(EngineIface* h) {
2006     check(set_vbucket_state(h, Vbid(1), vbucket_state_active),
2007           "Failed to set vbucket state.");
2008 
2009     int num_keys = 100;
2010     for (int ii = 0; ii < num_keys; ++ii) {
2011         std::stringstream ss;
2012         ss << "key" << ii;
2013         checkeq(ENGINE_SUCCESS,
2014                 store(h,
2015                       NULL,
2016                       OPERATION_SET,
2017                       ss.str().c_str(),
2018                       "value",
2019                       NULL,
2020                       0,
2021                       Vbid(1)),
2022                 "Failed to store an item.");
2023     }
2024     wait_for_flusher_to_settle(h);
2025 
2026     size_t file_size = get_int_stat(h, "ep_db_file_size", "diskinfo");
2027     size_t data_size = get_int_stat(h, "ep_db_data_size", "diskinfo");
2028     checklt(size_t{0}, file_size, "DB file size should be greater than 0");
2029     checklt(size_t{0}, data_size, "DB data size should be greater than 0");
2030     checkge(file_size, data_size, "DB file size should be >= DB data size");
2031     checkgt(get_int_stat(h, "vb_1:data_size", "diskinfo detail"), 0,
2032           "VB 1 data size should be greater than 0");
2033 
2034     checkeq(ENGINE_EINVAL,
2035             get_stats(h, "diskinfo "_ccb, {}, add_stats),
2036             "Expected invalid");
2037 
2038     checkeq(ENGINE_EINVAL,
2039             get_stats(h, "diskinfo detai"_ccb, {}, add_stats),
2040             "Expected invalid");
2041 
2042     checkeq(ENGINE_EINVAL,
2043             get_stats(h, "diskinfo detaillll"_ccb, {}, add_stats),
2044             "Expected invalid");
2045 
2046     return SUCCESS;
2047 }
2048 
test_uuid_stats(EngineIface* h)2049 static enum test_result test_uuid_stats(EngineIface* h) {
2050     vals.clear();
2051     checkeq(ENGINE_SUCCESS,
2052             get_stats(h, "uuid"_ccb, {}, add_stats),
2053             "Failed to get stats.");
2054     checkeq(cb::const_char_buffer("foobar"),
2055             static_cast<cb::const_char_buffer>(vals["uuid"]), "Incorrect uuid");
2056     return SUCCESS;
2057 }
2058 
test_item_stats(EngineIface* h)2059 static enum test_result test_item_stats(EngineIface* h) {
2060     checkeq(ENGINE_SUCCESS,
2061             store(h, NULL, OPERATION_SET, "key", "somevalue"),
2062             "Failed set.");
2063     wait_for_flusher_to_settle(h);
2064     checkeq(ENGINE_SUCCESS,
2065             store(h, NULL, OPERATION_SET, "key", "somevalueX"),
2066             "Failed set.");
2067     wait_for_flusher_to_settle(h);
2068     checkeq(ENGINE_SUCCESS,
2069             store(h, NULL, OPERATION_SET, "key1", "somevalueY"),
2070             "Failed set.");
2071     wait_for_flusher_to_settle(h);
2072 
2073     check_key_value(h, "key", "somevalueX", 10);
2074     check_key_value(h, "key1", "somevalueY", 10);
2075 
2076     checkeq(ENGINE_SUCCESS,
2077             del(h, "key1", 0, Vbid(0)),
2078             "Failed remove with value.");
2079     wait_for_flusher_to_settle(h);
2080 
2081     checkeq(ENGINE_SUCCESS,
2082             store(h, NULL, OPERATION_SET, "key1", "someothervalue"),
2083             "Failed set.");
2084     wait_for_flusher_to_settle(h);
2085 
2086     check_key_value(h, "key1", "someothervalue", 14);
2087 
2088     checkeq(3, get_int_stat(h, "vb_active_ops_create"), "Expected 3 creations");
2089     checkeq(1, get_int_stat(h, "vb_active_ops_update"), "Expected 1 updation");
2090     checkeq(1, get_int_stat(h, "vb_active_ops_delete"), "Expected 1 deletion");
2091     checkeq(3, get_int_stat(h, "vb_active_ops_get"), "Expected 3 gets");
2092 
2093     return SUCCESS;
2094 }
2095 
test_stats(EngineIface* h)2096 static enum test_result test_stats(EngineIface* h) {
2097     vals.clear();
2098     checkeq(ENGINE_SUCCESS,
2099             get_stats(h, {}, {}, add_stats),
2100             "Failed to get stats.");
2101     checklt(size_t{10}, vals.size(), "Kind of expected more stats than that.");
2102 
2103     return SUCCESS;
2104 }
2105 
test_mem_stats(EngineIface* h)2106 static enum test_result test_mem_stats(EngineIface* h) {
2107     char value[2048];
2108     memset(value, 'b', sizeof(value));
2109     strcpy(value + sizeof(value) - 4, "\r\n");
2110     int itemsRemoved = get_int_stat(h, "ep_items_rm_from_checkpoints");
2111     wait_for_persisted_value(h, "key", value);
2112     testHarness->time_travel(65);
2113     if (isPersistentBucket(h)) {
2114         wait_for_stat_change(h, "ep_items_rm_from_checkpoints", itemsRemoved);
2115     }
2116 
2117     if (isActiveCompressionEnabled(h)) {
2118         wait_for_item_compressor_to_settle(h);
2119     }
2120 
2121     int mem_used = get_int_stat(h, "mem_used");
2122     int cache_size = get_int_stat(h, "ep_total_cache_size");
2123     int overhead = get_int_stat(h, "ep_overhead");
2124     int value_size = get_int_stat(h, "ep_value_size");
2125     checkgt((mem_used - overhead), cache_size,
2126             "ep_kv_size should be greater than the hashtable cache size due to "
2127             "the checkpoint overhead");
2128 
2129     if (isPersistentBucket(h)) {
2130         evict_key(h, "key", Vbid(0), "Ejected.");
2131 
2132         checkge(cache_size, get_int_stat(h, "ep_total_cache_size"),
2133                 "Evict a value shouldn't increase the total cache size");
2134         checkgt(mem_used, get_int_stat(h, "mem_used"),
2135               "Expected mem_used to decrease when an item is evicted");
2136 
2137         check_key_value(h,
2138                         "key",
2139                         value,
2140                         strlen(value),
2141                         Vbid(0)); // Load an item from disk again.
2142 
2143         if (isActiveCompressionEnabled(h)) {
2144             wait_for_item_compressor_to_settle(h);
2145         }
2146         checkeq(value_size, get_int_stat(h, "ep_value_size"),
2147                 "Expected ep_value_size to remain the same after item is "
2148                 "loaded from disk");
2149     }
2150 
2151     return SUCCESS;
2152 }
2153 
test_io_stats(EngineIface* h)2154 static enum test_result test_io_stats(EngineIface* h) {
2155     int exp_write_bytes;
2156     std::string backend = get_str_stat(h, "ep_backend");
2157     if (backend == "couchdb") {
2158         exp_write_bytes = 22; /* TBD: Do not hard code the value */
2159     } else if (backend == "rocksdb") {
2160         // TODO RDB:
2161         return SKIPPED_UNDER_ROCKSDB;
2162     }
2163     else {
2164         return SKIPPED;
2165     }
2166 
2167     std::string collections = get_str_stat(h, "ep_collections_enabled");
2168     if (collections == "true") {
2169         // 1 byte of meta data for the collection-ID
2170         exp_write_bytes += 1;
2171     }
2172 
2173     reset_stats(h);
2174 
2175     checkeq(0,
2176             get_int_stat(h, "rw_0:io_bg_fetch_docs_read", "kvstore"),
2177             "Expected reset stats to set io_bg_fetch_docs_read to zero");
2178     checkeq(0,
2179             get_int_stat(h, "rw_0:io_num_write", "kvstore"),
2180             "Expected reset stats to set io_num_write to zero");
2181     checkeq(0,
2182             get_int_stat(h, "rw_0:io_bg_fetch_doc_bytes", "kvstore"),
2183             "Expected reset stats to set io_bg_fetch_doc_bytes to zero");
2184     checkeq(0,
2185             get_int_stat(h, "rw_0:io_document_write_bytes", "kvstore"),
2186             "Expected reset stats to set io_document_write_bytes to zero");
2187 
2188     const std::string key("a");
2189     const std::string value("b\r\n");
2190     wait_for_persisted_value(h, key.c_str(), value.c_str());
2191     checkeq(0,
2192             get_int_stat(h, "rw_0:io_bg_fetch_docs_read", "kvstore"),
2193             "Expected storing one value to not change the read counter");
2194     checkeq(0,
2195             get_int_stat(h, "rw_0:io_bg_fetch_doc_bytes", "kvstore"),
2196             "Expected storing one value to not change the bgfetch doc bytes");
2197     checkeq(1,
2198             get_int_stat(h, "rw_0:io_num_write", "kvstore"),
2199             "Expected storing the key to update the write counter");
2200     checkeq(exp_write_bytes,
2201             get_int_stat(h, "rw_0:io_document_write_bytes", "kvstore"),
2202             "Expected storing the key to update the write bytes");
2203 
2204     // Exact write amplification varies, bur expect it to be at least 10.0x
2205     // given we are writing a single byte key and single byte value.
2206     checkge(get_stat<float>(
2207                     h, "rw_0:io_flusher_write_amplification", "kvstore"),
2208             10.0f,
2209             "Expected storing the key to update Flusher Write Amplification");
2210 
2211     evict_key(h, key.c_str(), Vbid(0), "Ejected.");
2212 
2213     check_key_value(h, "a", value.c_str(), value.size(), Vbid(0));
2214 
2215     std::stringstream numReadStatStr;
2216     std::stringstream readBytesStatStr;
2217 
2218     if (backend == "couchdb") {
2219         numReadStatStr << "ro_" << 0 << ":io_bg_fetch_docs_read";
2220         readBytesStatStr << "ro_" << 0 << ":io_bg_fetch_doc_bytes";
2221     } else {
2222         cb_assert(false);
2223     }
2224 
2225     checkeq(1,
2226             get_int_stat(h, numReadStatStr.str().c_str(), "kvstore"),
2227             "Expected reading the value back in to update the read counter");
2228 
2229     uint64_t exp_read_bytes = key.size() + value.size() +
2230                               MetaData::getMetaDataSize(MetaData::Version::V1);
2231     if (collections == "true") {
2232         // 1 byte of meta data for the collection-ID
2233         exp_read_bytes += 1;
2234     }
2235     checkeq(exp_read_bytes,
2236             get_stat<uint64_t>(h, readBytesStatStr.str().c_str(), "kvstore"),
2237             "Expected reading the value back in to update the read bytes");
2238 
2239     // For read amplification, exact value depends on couchstore file layout,
2240     // but generally see a value of 2 here.
2241     checkge(get_float_stat(h, "ep_bg_fetch_avg_read_amplification"),
2242             2.0f,
2243             "Expected sensible bgFetch read amplification value");
2244 
2245     checkeq(1,
2246             get_int_stat(h, "rw_0:io_num_write", "kvstore"),
2247             "Expected reading the value back in to not update the write "
2248             "counter");
2249     checkeq(exp_write_bytes,
2250             get_int_stat(h, "rw_0:io_document_write_bytes", "kvstore"),
2251             "Expected reading the value back in to not update the write bytes");
2252 
2253     return SUCCESS;
2254 }
2255 
test_vb_file_stats(EngineIface* h)2256 static enum test_result test_vb_file_stats(EngineIface* h) {
2257     wait_for_flusher_to_settle(h);
2258     wait_for_stat_change(h, "ep_db_data_size", 0);
2259 
2260     int old_data_size = get_int_stat(h, "ep_db_data_size");
2261     int old_file_size = get_int_stat(h, "ep_db_file_size");
2262     checkne(0, old_file_size, "Expected a non-zero value for ep_db_file_size");
2263 
2264     // Write a value and test ...
2265     wait_for_persisted_value(h, "a", "b\r\n");
2266     checklt(old_data_size, get_int_stat(h, "ep_db_data_size"),
2267             "Expected the DB data size to increase");
2268     checklt(old_file_size, get_int_stat(h, "ep_db_file_size"),
2269             "Expected the DB file size to increase");
2270 
2271     checklt(0, get_int_stat(h, "vb_0:db_data_size", "vbucket-details 0"),
2272             "Expected the vbucket DB data size to non-zero");
2273     checklt(0, get_int_stat(h, "vb_0:db_file_size", "vbucket-details 0"),
2274             "Expected the vbucket DB file size to non-zero");
2275     return SUCCESS;
2276 }
2277 
test_vb_file_stats_after_warmup(EngineIface* h)2278 static enum test_result test_vb_file_stats_after_warmup(EngineIface* h) {
2279     if (!isWarmupEnabled(h)) {
2280         return SKIPPED;
2281     }
2282 
2283     for (int i = 0; i < 100; ++i) {
2284         std::stringstream key;
2285         key << "key-" << i;
2286         checkeq(ENGINE_SUCCESS,
2287                 store(h,
2288                       NULL,
2289                       OPERATION_SET,
2290                       key.str().c_str(),
2291                       "somevalue"),
2292                 "Error setting.");
2293     }
2294     wait_for_flusher_to_settle(h);
2295 
2296     int fileSize = get_int_stat(h, "vb_0:db_file_size", "vbucket-details 0");
2297     int spaceUsed = get_int_stat(h, "vb_0:db_data_size", "vbucket-details 0");
2298 
2299     // Restart the engine.
2300     testHarness->reload_engine(&h,
2301                                testHarness->engine_path,
2302                                testHarness->get_current_testcase()->cfg,
2303                                true,
2304                                false);
2305 
2306     wait_for_warmup_complete(h);
2307 
2308     int newFileSize = get_int_stat(h, "vb_0:db_file_size", "vbucket-details 0");
2309     int newSpaceUsed =
2310             get_int_stat(h, "vb_0:db_data_size", "vbucket-details 0");
2311 
2312     checkle(static_cast<float>(0.9 * fileSize),
2313             static_cast<float>(newFileSize),
2314             "Unexpected fileSize for vbucket");
2315     checkle(static_cast<float>(0.9 * spaceUsed),
2316             static_cast<float>(newSpaceUsed),
2317             "Unexpected spaceUsed for vbucket");
2318 
2319     return SUCCESS;
2320 }
2321 
test_bg_stats(EngineIface* h)2322 static enum test_result test_bg_stats(EngineIface* h) {
2323     reset_stats(h);
2324     wait_for_persisted_value(h, "a", "b\r\n");
2325     evict_key(h, "a", Vbid(0), "Ejected.");
2326     testHarness->time_travel(43);
2327     check_key_value(h, "a", "b\r\n", 3, Vbid(0));
2328 
2329     auto stats = get_all_stats(h);
2330     checkeq(1, std::stoi(stats.at("ep_bg_num_samples")),
2331                "Expected one sample");
2332 
2333     const char* bg_keys[] = { "ep_bg_min_wait",
2334                               "ep_bg_max_wait",
2335                               "ep_bg_wait_avg",
2336                               "ep_bg_min_load",
2337                               "ep_bg_max_load",
2338                               "ep_bg_load_avg"};
2339     for (const auto* key : bg_keys) {
2340         check(stats.find(key) != stats.end(),
2341               (std::string("Found no ") + key).c_str());
2342     }
2343 
2344     evict_key(h, "a", Vbid(0), "Ejected.");
2345     check_key_value(h, "a", "b\r\n", 3, Vbid(0));
2346     checkeq(2, get_int_stat(h, "ep_bg_num_samples"), "Expected one sample");
2347 
2348     reset_stats(h);
2349     checkeq(0,
2350             get_int_stat(h, "ep_bg_fetched"),
2351             "ep_bg_fetched is not reset to 0");
2352     return SUCCESS;
2353 }
2354 
test_bg_meta_stats(EngineIface* h)2355 static enum test_result test_bg_meta_stats(EngineIface* h) {
2356     reset_stats(h);
2357 
2358     wait_for_persisted_value(h, "k1", "v1");
2359     wait_for_persisted_value(h, "k2", "v2");
2360 
2361     evict_key(h, "k1", Vbid(0), "Ejected.");
2362     checkeq(ENGINE_SUCCESS,
2363             del(h, "k2", 0, Vbid(0)),
2364             "Failed remove with value.");
2365     wait_for_flusher_to_settle(h);
2366 
2367     checkeq(0, get_int_stat(h, "ep_bg_fetched"), "Expected bg_fetched to be 0");
2368     checkeq(0,
2369             get_int_stat(h, "ep_bg_meta_fetched"),
2370             "Expected bg_meta_fetched to be 0");
2371 
2372     check(get_meta(h, "k2"), "Get meta failed");
2373     checkeq(0, get_int_stat(h, "ep_bg_fetched"), "Expected bg_fetched to be 0");
2374     checkeq(1,
2375             get_int_stat(h, "ep_bg_meta_fetched"),
2376             "Expected bg_meta_fetched to be 1");
2377 
2378     checkeq(cb::engine_errc::success,
2379             get(h, NULL, "k1", Vbid(0)).first,
2380             "Missing key");
2381     checkeq(1, get_int_stat(h, "ep_bg_fetched"), "Expected bg_fetched to be 1");
2382     checkeq(1,
2383             get_int_stat(h, "ep_bg_meta_fetched"),
2384             "Expected bg_meta_fetched to be 1");
2385 
2386     // store new key with some random metadata
2387     const size_t keylen = strlen("k3");
2388     ItemMetaData itemMeta;
2389     itemMeta.revSeqno = 10;
2390     itemMeta.cas = 0xdeadbeef;
2391     itemMeta.exptime = 0;
2392     itemMeta.flags = 0xdeadbeef;
2393 
2394     checkeq(ENGINE_SUCCESS,
2395             add_with_meta(h, "k3", keylen, NULL, 0, Vbid(0), &itemMeta),
2396             "Expected to add item");
2397     checkeq(cb::mcbp::Status::Success, last_status.load(), "Set meta failed");
2398 
2399     check(get_meta(h, "k2"), "Get meta failed");
2400     checkeq(1, get_int_stat(h, "ep_bg_fetched"), "Expected bg_fetched to be 1");
2401 
2402     int expected_ep_bg_meta_fetched =
2403             get_bool_stat(h, "ep_bfilter_enabled") ? 1 : 2;
2404     checkeq(expected_ep_bg_meta_fetched,
2405             get_int_stat(h, "ep_bg_meta_fetched"),
2406             "bg_meta_fetched");
2407     return SUCCESS;
2408 }
2409 
test_key_stats(EngineIface* h)2410 static enum test_result test_key_stats(EngineIface* h) {
2411     check(set_vbucket_state(h, Vbid(1), vbucket_state_active),
2412           "Failed set vbucket 1 state.");
2413 
2414     // set (k1,v1) in vbucket 0
2415     checkeq(ENGINE_SUCCESS,
2416             store(h, NULL, OPERATION_SET, "k1", "v1"),
2417             "Failed to store an item.");
2418 
2419     // set (k2,v2) in vbucket 1
2420     checkeq(ENGINE_SUCCESS,
2421             store(h, NULL, OPERATION_SET, "k2", "v2", nullptr, 0, Vbid(1)),
2422             "Failed to store an item.");
2423 
2424     const void* cookie = testHarness->create_cookie();
2425 
2426     // stat for key "k1" and vbucket "0"
2427     const char *statkey1 = "key k1 0";
2428     checkeq(ENGINE_SUCCESS,
2429             h->get_stats(cookie, {statkey1, strlen(statkey1)}, {}, add_stats),
2430             "Failed to get stats.");
2431     check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2432     check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2433     check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2434     check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2435     check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2436 
2437     // stat for key "k2" and vbucket "1"
2438     const char *statkey2 = "key k2 1";
2439     checkeq(ENGINE_SUCCESS,
2440             h->get_stats(cookie, {statkey2, strlen(statkey2)}, {}, add_stats),
2441             "Failed to get stats.");
2442     check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2443     check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2444     check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2445     check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2446     check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2447 
2448     testHarness->destroy_cookie(cookie);
2449     return SUCCESS;
2450 }
2451 
test_vkey_stats(EngineIface* h)2452 static enum test_result test_vkey_stats(EngineIface* h) {
2453     check(set_vbucket_state(h, Vbid(1), vbucket_state_active),
2454           "Failed set vbucket 1 state.");
2455     check(set_vbucket_state(h, Vbid(2), vbucket_state_active),
2456           "Failed set vbucket 2 state.");
2457     check(set_vbucket_state(h, Vbid(3), vbucket_state_active),
2458           "Failed set vbucket 3 state.");
2459     check(set_vbucket_state(h, Vbid(4), vbucket_state_active),
2460           "Failed set vbucket 4 state.");
2461 
2462     wait_for_persisted_value(h, "k1", "v1");
2463     wait_for_persisted_value(h, "k2", "v2", Vbid(1));
2464     wait_for_persisted_value(h, "k3", "v3", Vbid(2));
2465     wait_for_persisted_value(h, "k4", "v4", Vbid(3));
2466     wait_for_persisted_value(h, "k5", "v5", Vbid(4));
2467 
2468     check(set_vbucket_state(h, Vbid(2), vbucket_state_replica),
2469           "Failed to set VB2 state.");
2470     check(set_vbucket_state(h, Vbid(3), vbucket_state_pending),
2471           "Failed to set VB3 state.");
2472     check(set_vbucket_state(h, Vbid(4), vbucket_state_dead),
2473           "Failed to set VB4 state.");
2474 
2475     const void* cookie = testHarness->create_cookie();
2476 
2477     // stat for key "k1" and vbucket "0"
2478     const char *statkey1 = "vkey k1 0";
2479     checkeq(ENGINE_SUCCESS,
2480             h->get_stats(cookie, {statkey1, strlen(statkey1)}, {}, add_stats),
2481             "Failed to get stats.");
2482     check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2483     check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2484     check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2485     check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2486     check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2487     check(vals.find("key_valid") != vals.end(), "Found no key_valid");
2488 
2489     // stat for key "k2" and vbucket "1"
2490     const char *statkey2 = "vkey k2 1";
2491     checkeq(ENGINE_SUCCESS,
2492             h->get_stats(cookie, {statkey2, strlen(statkey2)}, {}, add_stats),
2493             "Failed to get stats.");
2494     check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2495     check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2496     check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2497     check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2498     check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2499     check(vals.find("key_valid") != vals.end(), "Found no key_valid");
2500 
2501     // stat for key "k3" and vbucket "2"
2502     const char *statkey3 = "vkey k3 2";
2503     checkeq(ENGINE_SUCCESS,
2504             h->get_stats(cookie, {statkey3, strlen(statkey3)}, {}, add_stats),
2505             "Failed to get stats.");
2506     check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2507     check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2508     check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2509     check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2510     check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2511     check(vals.find("key_valid") != vals.end(), "Found no key_valid");
2512 
2513     // stat for key "k4" and vbucket "3"
2514     const char *statkey4 = "vkey k4 3";
2515     checkeq(ENGINE_SUCCESS,
2516             h->get_stats(cookie, {statkey4, strlen(statkey4)}, {}, add_stats),
2517             "Failed to get stats.");
2518     check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2519     check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2520     check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2521     check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2522     check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2523     check(vals.find("key_valid") != vals.end(), "Found no key_valid");
2524 
2525     // stat for key "k5" and vbucket "4"
2526     const char *statkey5 = "vkey k5 4";
2527     checkeq(ENGINE_SUCCESS,
2528             h->get_stats(cookie, {statkey5, strlen(statkey5)}, {}, add_stats),
2529             "Failed to get stats.");
2530     check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2531     check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2532     check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2533     check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2534     check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2535     check(vals.find("key_valid") != vals.end(), "Found no key_valid");
2536 
2537     testHarness->destroy_cookie(cookie);
2538     return SUCCESS;
2539 }
2540 
test_warmup_conf(EngineIface* h)2541 static enum test_result test_warmup_conf(EngineIface* h) {
2542     if (!isWarmupEnabled(h)) {
2543         return SKIPPED;
2544     }
2545 
2546     checkeq(100,
2547             get_int_stat(h, "ep_warmup_min_items_threshold"),
2548             "Incorrect initial warmup min items threshold.");
2549     checkeq(100,
2550             get_int_stat(h, "ep_warmup_min_memory_threshold"),
2551             "Incorrect initial warmup min memory threshold.");
2552 
2553     check(!set_param(h,
2554                      cb::mcbp::request::SetParamPayload::Type::Flush,
2555                      "warmup_min_items_threshold",
2556                      "a"),
2557           "Set warmup_min_items_threshold should have failed");
2558     check(!set_param(h,
2559                      cb::mcbp::request::SetParamPayload::Type::Flush,
2560                      "warmup_min_items_threshold",
2561                      "a"),
2562           "Set warmup_min_memory_threshold should have failed");
2563 
2564     check(set_param(h,
2565                     cb::mcbp::request::SetParamPayload::Type::Flush,
2566                     "warmup_min_items_threshold",
2567                     "80"),
2568           "Set warmup_min_items_threshold should have worked");
2569     check(set_param(h,
2570                     cb::mcbp::request::SetParamPayload::Type::Flush,
2571                     "warmup_min_memory_threshold",
2572                     "80"),
2573           "Set warmup_min_memory_threshold should have worked");
2574 
2575     checkeq(80,
2576             get_int_stat(h, "ep_warmup_min_items_threshold"),
2577             "Incorrect smaller warmup min items threshold.");
2578     checkeq(80,
2579             get_int_stat(h, "ep_warmup_min_memory_threshold"),
2580             "Incorrect smaller warmup min memory threshold.");
2581 
2582     for (int i = 0; i < 100; ++i) {
2583         std::stringstream key;
2584         key << "key-" << i;
2585         checkeq(ENGINE_SUCCESS,
2586                 store(h,
2587                       NULL,
2588                       OPERATION_SET,
2589                       key.str().c_str(),
2590                       "somevalue"),
2591                 "Error setting.");
2592     }
2593 
2594     // Restart the server.
2595     std::string config(testHarness->get_current_testcase()->cfg);
2596     config = config + "warmup_min_memory_threshold=0";
2597     testHarness->reload_engine(
2598             &h, testHarness->engine_path, config.c_str(), true, false);
2599 
2600     wait_for_warmup_complete(h);
2601 
2602     const std::string eviction_policy =
2603             get_str_stat(h, "ep_item_eviction_policy");
2604     if (eviction_policy == "value_only") {
2605         checkeq(100,
2606                 get_int_stat(h, "ep_warmup_key_count", "warmup"),
2607                 "Expected 100 keys loaded after warmup");
2608     } else { // Full eviction mode
2609         checkeq(0,
2610                 get_int_stat(h, "ep_warmup_key_count", "warmup"),
2611                 "Expected 0 keys loaded after warmup");
2612     }
2613 
2614     checkeq(0,
2615             get_int_stat(h, "ep_warmup_value_count", "warmup"),
2616             "Expected 0 values loaded after warmup");
2617 
2618     return SUCCESS;
2619 }
2620 
2621 // Test that all the configuration parameters associated with the ItemPager,
2622 // can be set.
test_itempager_conf(EngineIface* h)2623 static enum test_result test_itempager_conf(EngineIface* h) {
2624     check(set_param(h,
2625                     cb::mcbp::request::SetParamPayload::Type::Flush,
2626                     "pager_active_vb_pcnt",
2627                     "50"),
2628           "Setting pager_active_vb_pcnt should have worked");
2629     checkeq(50,
2630             get_int_stat(h, "ep_pager_active_vb_pcnt"),
2631             "pager_active_vb_pcnt did not get set to the correct value");
2632 
2633     check(set_param(h,
2634                     cb::mcbp::request::SetParamPayload::Type::Flush,
2635                     "pager_sleep_time_ms",
2636                     "1000"),
2637           "Setting pager_sleep_time_ms should have worked");
2638     checkeq(1000,
2639             get_int_stat(h, "ep_pager_sleep_time_ms"),
2640             "pager_sleep_time_ms did not get set to the correct value");
2641 
2642     check(set_param(h,
2643                     cb::mcbp::request::SetParamPayload::Type::Flush,
2644                     "item_eviction_age_percentage",
2645                     "100"),
2646           "Set item_eviction_age_percentage should have worked");
2647     checkeq(100,
2648             get_int_stat(h, "ep_item_eviction_age_percentage"),
2649             "item_eviction_age_percentage did not get set to the correct "
2650             "value");
2651 
2652     check(set_param(h,
2653                     cb::mcbp::request::SetParamPayload::Type::Flush,
2654                     "item_eviction_freq_counter_age_threshold",
2655                     "10"),
2656           "Set item_eviction_freq_counter_age_threshold should have worked");
2657     checkeq(10,
2658             get_int_stat(h, "ep_item_eviction_freq_counter_age_threshold"),
2659             "item_eviction_freq_counter_age_threshold did not get set to the "
2660             "correct value");
2661 
2662     check(set_param(h,
2663                     cb::mcbp::request::SetParamPayload::Type::Flush,
2664                     "item_freq_decayer_chunk_duration",
2665                     "1000"),
2666           "Set item_freq_decayer_chunk_duration should have worked");
2667     checkeq(1000,
2668             get_int_stat(h, "ep_item_freq_decayer_chunk_duration"),
2669             "item_freq_decayer_chunk_duration did not get set to the correct "
2670             "value");
2671 
2672     check(set_param(h,
2673                     cb::mcbp::request::SetParamPayload::Type::Flush,
2674                     "item_freq_decayer_percent",
2675                     "100"),
2676           "Set item_freq_decayer_percent should have worked");
2677     checkeq(100,
2678             get_int_stat(h, "ep_item_freq_decayer_percent"),
2679             "item_freq_decayer_percent did not get set to the correct value");
2680 
2681     return SUCCESS;
2682 }
2683 
test_bloomfilter_conf(EngineIface* h)2684 static enum test_result test_bloomfilter_conf(EngineIface* h) {
2685     if (get_bool_stat(h, "ep_bfilter_enabled") == false) {
2686         check(set_param(h,
2687                         cb::mcbp::request::SetParamPayload::Type::Flush,
2688                         "bfilter_enabled",
2689                         "true"),
2690               "Set bloomfilter_enabled should have worked");
2691     }
2692     check(get_bool_stat(h, "ep_bfilter_enabled"),
2693           "Bloom filter wasn't enabled");
2694 
2695     checkeq(0.1f,
2696             get_float_stat(h, "ep_bfilter_residency_threshold"),
2697             "Incorrect initial bfilter_residency_threshold.");
2698 
2699     check(set_param(h,
2700                     cb::mcbp::request::SetParamPayload::Type::Flush,
2701                     "bfilter_enabled",
2702                     "false"),
2703           "Set bloomfilter_enabled should have worked.");
2704     check(set_param(h,
2705                     cb::mcbp::request::SetParamPayload::Type::Flush,
2706                     "bfilter_residency_threshold",
2707                     "0.15"),
2708           "Set bfilter_residency_threshold should have worked.");
2709 
2710     checkeq(false, get_bool_stat(h, "ep_bfilter_enabled"),
2711             "Bloom filter should have been disabled.");
2712     checkeq(0.15f,
2713             get_float_stat(h, "ep_bfilter_residency_threshold"),
2714             "Incorrect bfilter_residency_threshold.");
2715 
2716     return SUCCESS;
2717 }
2718 
test_bloomfilters(EngineIface* h)2719 static enum test_result test_bloomfilters(EngineIface* h) {
2720     if (get_bool_stat(h, "ep_bfilter_enabled") == false) {
2721         check(set_param(h,
2722                         cb::mcbp::request::SetParamPayload::Type::Flush,
2723                         "bfilter_enabled",
2724                         "true"),
2725               "Set bloomfilter_enabled should have worked");
2726     }
2727     check(get_bool_stat(h, "ep_bfilter_enabled"),
2728           "Bloom filter wasn't enabled");
2729 
2730     // Key is only present if bgOperations is non-zero.
2731     int num_read_attempts = get_int_stat_or_default(h, 0, "ep_bg_num_samples");
2732 
2733     // Ensure vbucket's bloom filter is enabled
2734     checkeq("ENABLED"s,
2735             get_str_stat(h, "vb_0:bloom_filter", "vbucket-details 0"),
2736             "Vbucket 0's bloom filter wasn't enabled upon setup!");
2737 
2738     int i;
2739 
2740     // Insert 10 items.
2741     for (i = 0; i < 10; ++i) {
2742         std::stringstream key;
2743         key << "key-" << i;
2744         checkeq(ENGINE_SUCCESS,
2745                 store(h,
2746                       NULL,
2747                       OPERATION_SET,
2748                       key.str().c_str(),
2749                       "somevalue"),
2750                 "Error setting.");
2751     }
2752     wait_for_flusher_to_settle(h);
2753 
2754     // Evict all 10 items.
2755     for (i = 0; i < 10; ++i) {
2756         std::stringstream key;
2757         key << "key-" << i;
2758         evict_key(h, key.str().c_str(), Vbid(0), "Ejected.");
2759     }
2760     wait_for_flusher_to_settle(h);
2761 
2762     // Ensure 10 items are non-resident.
2763     cb_assert(10 == get_int_stat(h, "ep_num_non_resident"));
2764 
2765     // Issue delete on first 5 items.
2766     for (i = 0; i < 5; ++i) {
2767         std::stringstream key;
2768         key << "key-" << i;
2769         checkeq(ENGINE_SUCCESS,
2770                 del(h, key.str().c_str(), 0, Vbid(0)),
2771                 "Failed remove with value.");
2772     }
2773     wait_for_flusher_to_settle(h);
2774 
2775     // Ensure that there are 5 non-resident items
2776     cb_assert(5 == get_int_stat(h, "ep_num_non_resident"));
2777     cb_assert(5 == get_int_stat(h, "curr_items"));
2778 
2779     checkeq(ENGINE_SUCCESS,
2780             get_stats(h, {}, {}, add_stats),
2781             "Failed to get stats.");
2782     std::string eviction_policy = vals.find("ep_item_eviction_policy")->second;
2783 
2784     useconds_t sleepTime = 128;
2785 
2786     if (eviction_policy == "value_only") {  // VALUE-ONLY EVICTION MODE
2787 
2788         checkeq(5,
2789                 get_int_stat(
2790                         h, "vb_0:bloom_filter_key_count", "vbucket-details 0"),
2791                 "Unexpected no. of keys in bloom filter");
2792 
2793         checkeq(num_read_attempts,
2794                 get_int_stat_or_default(h, 0, "ep_bg_num_samples"),
2795                 "Expected bgFetch attempts to remain unchanged");
2796 
2797         for (i = 0; i < 5; ++i) {
2798             std::stringstream key;
2799             key << "key-" << i;
2800             check(get_meta(h, key.str().c_str()), "Get meta failed");
2801         }
2802 
2803         // GetMeta would cause bgFetches as bloomfilter contains
2804         // the deleted items.
2805         checkeq(num_read_attempts + 5,
2806                 get_int_stat(h, "ep_bg_num_samples"),
2807                 "Expected bgFetch attempts to increase by five");
2808 
2809         // Run compaction, with drop_deletes
2810         compact_db(h, Vbid(0), Vbid(0), 15, 15, 1);
2811         while (get_int_stat(h, "ep_pending_compactions") != 0) {
2812             decayingSleep(&sleepTime);
2813         }
2814 
2815         for (i = 0; i < 5; ++i) {
2816             std::stringstream key;
2817             key << "key-" << i;
2818             check(get_meta(h, key.str().c_str()), "Get meta failed");
2819         }
2820         checkeq(num_read_attempts + 5,
2821                 get_int_stat(h, "ep_bg_num_samples"),
2822                 "Expected bgFetch attempts to stay as before");
2823 
2824     } else {                                // FULL EVICTION MODE
2825 
2826         checkeq(10,
2827                 get_int_stat(
2828                         h, "vb_0:bloom_filter_key_count", "vbucket-details 0"),
2829                 "Unexpected no. of keys in bloom filter");
2830 
2831         // Because of issuing deletes on non-resident items
2832         checkeq(num_read_attempts + 5,
2833                 get_int_stat(h, "ep_bg_num_samples"),
2834                 "Expected bgFetch attempts to increase by five, after deletes");
2835 
2836         // Run compaction, with drop_deletes, to exclude deleted items
2837         // from bloomfilter.
2838         compact_db(h, Vbid(0), Vbid(0), 15, 15, 1);
2839         while (get_int_stat(h, "ep_pending_compactions") != 0) {
2840             decayingSleep(&sleepTime);
2841         }
2842 
2843         for (i = 0; i < 5; i++) {
2844             std::stringstream key;
2845             key << "key-" << i;
2846             checkeq(cb::engine_errc::no_such_key,
2847                     get(h, NULL, key.str(), Vbid(0)).first,
2848                     "Unable to get stored item");
2849         }
2850         // + 6 because last delete is not purged by the compactor
2851         checkeq(num_read_attempts + 6,
2852                 get_int_stat(h, "ep_bg_num_samples"),
2853                 "Expected bgFetch attempts to stay as before");
2854     }
2855 
2856     return SUCCESS;
2857 }
2858 
test_bloomfilters_with_store_apis(EngineIface* h)2859 static enum test_result test_bloomfilters_with_store_apis(EngineIface* h) {
2860     if (get_bool_stat(h, "ep_bfilter_enabled") == false) {
2861         check(set_param(h,
2862                         cb::mcbp::request::SetParamPayload::Type::Flush,
2863                         "bfilter_enabled",
2864                         "true"),
2865               "Set bloomfilter_enabled should have worked");
2866     }
2867     check(get_bool_stat(h, "ep_bfilter_enabled"),
2868           "Bloom filter wasn't enabled");
2869 
2870     int num_read_attempts = get_int_stat_or_default(h, 0, "ep_bg_num_samples");
2871 
2872     // Ensure vbucket's bloom filter is enabled
2873     checkeq("ENABLED"s,
2874             get_str_stat(h, "vb_0:bloom_filter", "vbucket-details 0"),
2875             "Vbucket 0's bloom filter wasn't enabled upon setup!");
2876 
2877     for (int i = 0; i < 1000; i++) {
2878         std::stringstream key;
2879         key << "key-" << i;
2880         check(!get_meta(h, key.str().c_str()), "Get meta should fail.");
2881     }
2882 
2883     checkeq(num_read_attempts,
2884             get_int_stat_or_default(h, 0, "ep_bg_num_samples"),
2885             "Expected no bgFetch attempts");
2886 
2887     checkeq(ENGINE_SUCCESS,
2888             get_stats(h, {}, {}, add_stats),
2889             "Failed to get stats.");
2890     std::string eviction_policy = vals.find("ep_item_eviction_policy")->second;
2891 
2892     if (eviction_policy == "full_eviction") {  // FULL EVICTION MODE
2893         // Set with Meta
2894         int j;
2895         for (j = 0; j < 10; j++) {
2896             // init some random metadata
2897             ItemMetaData itm_meta;
2898             itm_meta.revSeqno = 10;
2899             itm_meta.cas = 0xdeadbeef;
2900             itm_meta.exptime = time(NULL) + 300;
2901             itm_meta.flags = 0xdeadbeef;
2902 
2903             const std::string key = "swm-" + std::to_string(j);
2904             checkeq(ENGINE_SUCCESS,
2905                     set_with_meta(h,
2906                                   key.data(),
2907                                   key.size(),
2908                                   "somevalue",
2909                                   9,
2910                                   Vbid(0),
2911                                   &itm_meta,
2912                                   0),
2913                     "Expected to store item");
2914         }
2915 
2916         checkeq(num_read_attempts,
2917                 get_int_stat_or_default(h, 0, "ep_bg_num_samples"),
2918                 "Expected no bgFetch attempts");
2919 
2920         // Add
2921         for (j = 0; j < 10; j++) {
2922             std::stringstream key;
2923             key << "add-" << j;
2924 
2925             checkeq(ENGINE_SUCCESS,
2926                     store(h,
2927                           NULL,
2928                           OPERATION_ADD,
2929                           key.str().c_str(),
2930                           "newvalue"),
2931                     "Failed to add value again.");
2932         }
2933 
2934         checkeq(num_read_attempts,
2935                 get_int_stat_or_default(h, 0, "ep_bg_num_samples"),
2936                 "Expected no bgFetch attempts");
2937 
2938         // Delete
2939         for (j = 0; j < 10; j++) {
2940             std::stringstream key;
2941             key << "del-" << j;
2942             checkeq(ENGINE_KEY_ENOENT,
2943                     del(h, key.str().c_str(), 0, Vbid(0)),
2944                     "Failed remove with value.");
2945         }
2946 
2947         checkeq(num_read_attempts,
2948                 get_int_stat_or_default(h, 0, "ep_bg_num_samples"),
2949                 "Expected no bgFetch attempts");
2950     }
2951 
2952     return SUCCESS;
2953 }
2954 
test_bloomfilter_delete_plus_set_scenario( EngineIface* h)2955 static enum test_result test_bloomfilter_delete_plus_set_scenario(
2956         EngineIface* h) {
2957     if (get_bool_stat(h, "ep_bfilter_enabled") == false) {
2958         check(set_param(h,
2959                         cb::mcbp::request::SetParamPayload::Type::Flush,
2960                         "bfilter_enabled",
2961                         "true"),
2962               "Set bloomfilter_enabled should have worked");
2963     }
2964     check(get_bool_stat(h, "ep_bfilter_enabled"),
2965           "Bloom filter wasn't enabled");
2966 
2967     // Ensure vbucket's bloom filter is enabled
2968     checkeq("ENABLED"s,
2969             get_str_stat(h, "vb_0:bloom_filter", "vbucket-details 0"),
2970             "Vbucket 0's bloom filter wasn't enabled upon setup!");
2971 
2972     checkeq(ENGINE_SUCCESS,
2973             store(h, NULL, OPERATION_SET, "k1", "v1"),
2974             "Failed to fail to store an item.");
2975 
2976     wait_for_flusher_to_settle(h);
2977     int num_writes = get_int_stat(h, "rw_0:io_num_write", "kvstore");
2978     int num_persisted = get_int_stat(h, "ep_total_persisted");
2979     cb_assert(num_writes == 1 && num_persisted == 1);
2980 
2981     checkeq(ENGINE_SUCCESS,
2982             del(h, "k1", 0, Vbid(0)),
2983             "Failed remove with value.");
2984     stop_persistence(h);
2985     checkeq(ENGINE_SUCCESS,
2986             store(h, NULL, OPERATION_SET, "k1", "v2", nullptr, 0, Vbid(0)),
2987             "Failed to fail to store an item.");
2988     int key_count =
2989             get_int_stat(h, "vb_0:bloom_filter_key_count", "vbucket-details 0");
2990 
2991     if (key_count == 0) {
2992         checkge(2, get_int_stat(h, "rw_0:io_num_write", "kvstore"),
2993               "Unexpected number of writes");
2994         start_persistence(h);
2995         wait_for_flusher_to_settle(h);
2996         checkeq(0,
2997                 get_int_stat(
2998                         h, "vb_0:bloom_filter_key_count", "vbucket-details 0"),
2999                 "Unexpected number of keys in bloomfilter");
3000     } else {
3001         cb_assert(key_count == 1);
3002         checkeq(2,
3003                 get_int_stat(h, "rw_0:io_num_write", "kvstore"),
3004                 "Unexpected number of writes");
3005         start_persistence(h);
3006         wait_for_flusher_to_settle(h);
3007         checkeq(1,
3008                 get_int_stat(
3009                         h, "vb_0:bloom_filter_key_count", "vbucket-details 0"),
3010                 "Unexpected number of keys in bloomfilter");
3011     }
3012 
3013     return SUCCESS;
3014 }
3015 
test_datatype(EngineIface* h)3016 static enum test_result test_datatype(EngineIface* h) {
3017     const void* cookie = testHarness->create_cookie();
3018     testHarness->set_datatype_support(cookie, true);
3019 
3020     item *itm = NULL;
3021     const std::string key("{\"foo\":\"bar\"}");
3022     const protocol_binary_datatype_t datatype = PROTOCOL_BINARY_DATATYPE_JSON;
3023     uint64_t cas = 0;
3024     std::string value("x");
3025     checkeq(ENGINE_SUCCESS,
3026             storeCasOut(h, nullptr, Vbid(0), key, value, datatype, itm, cas),
3027             "Expected set to succeed");
3028 
3029     auto ret = get(h, cookie, key, Vbid(0));
3030     checkeq(cb::engine_errc::success, ret.first, "Unable to get stored item");
3031 
3032     item_info info;
3033     h->get_item_info(ret.second.get(), &info);
3034     checkeq(PROTOCOL_BINARY_DATATYPE_JSON, info.datatype, "Invalid datatype");
3035 
3036     const char* key1 = "foo";
3037     const char* val1 = "{\"foo1\":\"bar1\"}";
3038     ItemMetaData itm_meta;
3039     itm_meta.revSeqno = 10;
3040     itm_meta.cas = info.cas;
3041     itm_meta.exptime = info.exptime;
3042     itm_meta.flags = info.flags;
3043     checkeq(ENGINE_SUCCESS,
3044             set_with_meta(h,
3045                           key1,
3046                           strlen(key1),
3047                           val1,
3048                           strlen(val1),
3049                           Vbid(0),
3050                           &itm_meta,
3051                           last_cas,
3052                           0,
3053                           info.datatype,
3054                           cookie),
3055             "Expected to store item");
3056 
3057     ret = get(h, cookie, key1, Vbid(0));
3058     checkeq(cb::engine_errc::success, ret.first, "Unable to get stored item");
3059 
3060     h->get_item_info(ret.second.get(), &info);
3061     checkeq(PROTOCOL_BINARY_DATATYPE_JSON,
3062             info.datatype,
3063             "Invalid datatype, when setWithMeta");
3064 
3065     testHarness->destroy_cookie(cookie);
3066     return SUCCESS;
3067 }
3068 
test_datatype_with_unknown_command(EngineIface* h)3069 static enum test_result test_datatype_with_unknown_command(EngineIface* h) {
3070     const void* cookie = testHarness->create_cookie();
3071     testHarness->set_datatype_support(cookie, true);
3072     const char* key = "foo";
3073     const char* val = "{\"foo\":\"bar\"}";
3074     uint8_t datatype = PROTOCOL_BINARY_DATATYPE_JSON;
3075 
3076     ItemMetaData itm_meta;
3077     itm_meta.revSeqno = 10;
3078     itm_meta.cas = 0x1;
3079     itm_meta.exptime = 0;
3080     itm_meta.flags = 0;
3081 
3082     //SET_WITH_META
3083     checkeq(ENGINE_SUCCESS,
3084             set_with_meta(h,
3085                           key,
3086                           strlen(key),
3087                           val,
3088                           strlen(val),
3089                           Vbid(0),
3090                           &itm_meta,
3091                           0,
3092                           0,
3093                           datatype,
3094                           cookie),
3095             "Expected to store item");
3096 
3097     auto ret = get(h, cookie, key, Vbid(0));
3098     checkeq(cb::engine_errc::success, ret.first, "Unable to get stored item");
3099 
3100     item_info info;
3101     h->get_item_info(ret.second.get(), &info);
3102     checkeq(PROTOCOL_BINARY_DATATYPE_JSON,
3103             info.datatype,
3104             "Invalid datatype, when setWithMeta");
3105 
3106     //SET_RETURN_META
3107     checkeq(ENGINE_SUCCESS,
3108             set_ret_meta(h,
3109                          "foo1",
3110                          4,
3111                          val,
3112                          strlen(val),
3113                          Vbid(0),
3114                          0,
3115                          0,
3116                          0,
3117                          datatype,
3118                          cookie),
3119             "Expected success");
3120     checkeq(cb::mcbp::Status::Success, last_status.load(),
3121             "Expected set returing meta to succeed");
3122     checkeq(PROTOCOL_BINARY_DATATYPE_JSON,
3123             last_datatype.load(),
3124             "Invalid datatype, when set_return_meta");
3125 
3126     testHarness->destroy_cookie(cookie);
3127     return SUCCESS;
3128 }
3129 
test_session_cas_validation(EngineIface* h)3130 static enum test_result test_session_cas_validation(EngineIface* h) {
3131     // Testing cb::mcbp::ClientOpcode::SetVbucket..
3132     char ext[4];
3133     vbucket_state_t state = vbucket_state_active;
3134     uint32_t val = static_cast<uint32_t>(state);
3135     val = htonl(val);
3136     memcpy(ext, (char*)&val, sizeof(val));
3137 
3138     uint64_t cas = 0x0101010101010101;
3139     auto pkt = createPacket(
3140             cb::mcbp::ClientOpcode::SetVbucket, Vbid(0), cas, {ext, 4});
3141     checkeq(ENGINE_KEY_EEXISTS,
3142             h->unknown_command(NULL, *pkt, add_response),
3143             "SET_VBUCKET command failed");
3144 
3145     cas = 0x0102030405060708;
3146     pkt = createPacket(
3147             cb::mcbp::ClientOpcode::SetVbucket, Vbid(0), cas, {ext, 4});
3148     checkeq(ENGINE_SUCCESS,
3149             h->unknown_command(NULL, *pkt, add_response),
3150             "SET_VBUCKET command failed");
3151     cb_assert(last_status == cb::mcbp::Status::Success);
3152 
3153     return SUCCESS;
3154 }
3155 
test_access_scanner_settings(EngineIface* h)3156 static enum test_result test_access_scanner_settings(EngineIface* h) {
3157     if (!isWarmupEnabled(h)) {
3158         // Access scanner n/a without warmup.
3159         return SKIPPED;
3160     }
3161 
3162     // Create a unique access log path by combining with the db path.
3163     checkeq(ENGINE_SUCCESS,
3164             get_stats(h, {}, {}, add_stats),
3165             "Failed to get stats.");
3166     std::string dbname = vals.find("ep_dbname")->second;
3167 
3168     const auto alog_path = std::string("alog_path=") + dbname +
3169                            cb::io::DirectorySeparator + "access.log";
3170     std::string newconfig =
3171             std::string(testHarness->get_current_testcase()->cfg) + alog_path;
3172 
3173     testHarness->reload_engine(
3174             &h, testHarness->engine_path, newconfig.c_str(), true, false);
3175 
3176     wait_for_warmup_complete(h);
3177 
3178     std::string err_msg;
3179     // Check access scanner is enabled and alog_task_time is at default
3180     checkeq(true,
3181             get_bool_stat(h, "ep_access_scanner_enabled"),
3182             "Expected access scanner to be enabled");
3183     cb_assert(get_int_stat(h, "ep_alog_task_time") == 2);
3184 
3185     // Ensure access_scanner_task_time is what its expected to be.
3186     // Need to wait until the AccessScanner task has been setup.
3187     wait_for_stat_change(
3188             h, "ep_access_scanner_task_time", std::string{"NOT_SCHEDULED"});
3189 
3190     std::string str = get_str_stat(h, "ep_access_scanner_task_time");
3191     std::string expected_time = "02:00";
3192     err_msg.assign("Initial time incorrect, expect: " +
3193                    expected_time + ", actual: " + str.substr(11, 5));
3194     checkeq(0, str.substr(11, 5).compare(expected_time), err_msg.c_str());
3195 
3196     // Update alog_task_time and ensure the update is successful
3197     expected_time = "05:00";
3198 
3199     // [MB-24422] we need to set this multiple times as the change listeners
3200     //  may not have been initialized at the time of call
3201     repeat_till_true([&]() {
3202         set_param(h,
3203                   cb::mcbp::request::SetParamPayload::Type::Flush,
3204                   "alog_task_time",
3205                   "5");
3206         str = get_str_stat(h, "ep_access_scanner_task_time");
3207         return (0 == str.substr(11, 5).compare(expected_time));
3208     });
3209 
3210     err_msg.assign("Updated time incorrect, expect: " +
3211                    expected_time + ", actual: " + str.substr(11, 5));
3212     checkeq(0, str.substr(11, 5).compare(expected_time), err_msg.c_str());
3213 
3214     // Update alog_sleep_time by 10 mins and ensure the update is successful.
3215     const std::chrono::minutes update_by{10};
3216     std::string targetTaskTime1{make_time_string(std::chrono::system_clock::now() +
3217                                                  update_by)};
3218 
3219     set_param(h,
3220               cb::mcbp::request::SetParamPayload::Type::Flush,
3221               "alog_sleep_time",
3222               std::to_string(update_by.count()).c_str());
3223     str = get_str_stat(h, "ep_access_scanner_task_time");
3224 
3225     // Recalculate now() + 10mins as upper bound on when the task should be
3226     // scheduled.
3227     std::string targetTaskTime2{make_time_string(std::chrono::system_clock::now() +
3228                                                  update_by)};
3229 
3230     // ep_access_scanner_task_time should fall within the range of
3231     // targetTaskTime1 and targetTaskTime2
3232     err_msg.assign("Unexpected task time range, expect: " +
3233                    targetTaskTime1 + " <= " + str + " <= " + targetTaskTime2);
3234     checkle(targetTaskTime1, str, err_msg.c_str());
3235     checkle(str, targetTaskTime2, err_msg.c_str());
3236 
3237     return SUCCESS;
3238 }
3239 
test_access_scanner(EngineIface* h)3240 static enum test_result test_access_scanner(EngineIface* h) {
3241     if (!isWarmupEnabled(h)) {
3242         // Access scanner not applicable without warmup.
3243         return SKIPPED;
3244     }
3245 
3246     // Create a unique access log path by combining with the db path.
3247     checkeq(ENGINE_SUCCESS,
3248             get_stats(h, {}, {}, add_stats),
3249             "Failed to get stats.");
3250     const auto dbname = vals.find("ep_dbname")->second;
3251 
3252     const auto alog_path = std::string("alog_path=") + dbname +
3253                            cb::io::DirectorySeparator + "access.log";
3254 
3255     /* We do not want the access scanner task to be running while we initiate it
3256        explicitly below. Hence set the alog_task_time to about 1 ~ 2 hours
3257        from now */
3258     const time_t now = time(nullptr);