1 /* -*- MODE: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3  *     Copyright 2010 Couchbase, Inc
4  *
5  *   Licensed under the Apache License, Version 2.0 (the "License");
6  *   you may not use this file except in compliance with the License.
7  *   You may obtain a copy of the License at
8  *
9  *       http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *   Unless required by applicable law or agreed to in writing, software
12  *   distributed under the License is distributed on an "AS IS" BASIS,
13  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *   See the License for the specific language governing permissions and
15  *   limitations under the License.
16  */
17 
18 // Usage: (to run just a single test case)
19 // make engine_tests EP_TEST_NUM=3
20 
21 #include "config.h"
22 
23 #include <stdio.h>
24 #include <stdlib.h>
25 #include <string.h>
26 
27 #include <chrono>
28 #include <condition_variable>
29 #include <cstdlib>
30 #include <iomanip>
31 #include <iostream>
32 #include <map>
33 #include <mutex>
34 #include <regex>
35 #include <set>
36 #include <sstream>
37 #include <string>
38 #include <thread>
39 #include <unordered_map>
40 #include <unordered_set>
41 #include <vector>
42 
43 #include "atomic.h"
44 #include "couch-kvstore/couch-kvstore-metadata.h"
45 #include "ep_test_apis.h"
46 
47 #include "ep_testsuite_common.h"
48 #include "locks.h"
49 #include <libcouchstore/couch_db.h>
50 #include <memcached/engine.h>
51 #include <memcached/engine_testapp.h>
52 #include <platform/cb_malloc.h>
53 #include <platform/dirutils.h>
54 #include <JSON_checker.h>
55 #include <memcached/types.h>
56 #include <string_utilities.h>
57 #include <xattr/blob.h>
58 #include <xattr/utils.h>
59 
60 #ifdef linux
61 /* /usr/include/netinet/in.h defines macros from ntohs() to _bswap_nn to
62  * optimize the conversion functions, but the prototypes generate warnings
63  * from gcc. The conversion methods isn't the bottleneck for my app, so
64  * just remove the warnings by undef'ing the optimization ..
65  */
66 #undef ntohs
67 #undef ntohl
68 #undef htons
69 #undef htonl
70 #endif
71 
72 // ptr_fun don't like the extern "C" thing for unlock cookie.. cast it
73 // away ;)
74 typedef void (*UNLOCK_COOKIE_T)(const void *cookie);
75 
76 class ThreadData {
77 public:
ThreadData(ENGINE_HANDLE *eh, ENGINE_HANDLE_V1 *ehv1, int e=0)78     ThreadData(ENGINE_HANDLE *eh, ENGINE_HANDLE_V1 *ehv1,
79                int e=0) : h(eh), h1(ehv1), extra(e) {}
80     ENGINE_HANDLE    *h;
81     ENGINE_HANDLE_V1 *h1;
82     int               extra;
83 };
84 
85 enum class BucketType { EP, Ephemeral };
86 
check_observe_seqno(bool failover, BucketType bucket_type, uint8_t format_type, uint16_t vb_id, uint64_t vb_uuid, uint64_t last_persisted_seqno, uint64_t current_seqno, uint64_t failover_vbuuid = 0, uint64_t failover_seqno = 0)87 static void check_observe_seqno(bool failover,
88                                 BucketType bucket_type,
89                                 uint8_t format_type,
90                                 uint16_t vb_id,
91                                 uint64_t vb_uuid,
92                                 uint64_t last_persisted_seqno,
93                                 uint64_t current_seqno,
94                                 uint64_t failover_vbuuid = 0,
95                                 uint64_t failover_seqno = 0) {
96     uint8_t recv_format_type;
97     uint16_t recv_vb_id;
98     uint64_t recv_vb_uuid;
99     uint64_t recv_last_persisted_seqno;
100     uint64_t recv_current_seqno;
101     uint64_t recv_failover_vbuuid;
102     uint64_t recv_failover_seqno;
103 
104     memcpy(&recv_format_type, last_body.data(), sizeof(uint8_t));
105     checkeq(format_type, recv_format_type, "Wrong format type in result");
106     memcpy(&recv_vb_id, last_body.data() + 1, sizeof(uint16_t));
107     checkeq(vb_id, ntohs(recv_vb_id), "Wrong vbucket id in result");
108     memcpy(&recv_vb_uuid, last_body.data() + 3, sizeof(uint64_t));
109     checkeq(vb_uuid, ntohll(recv_vb_uuid), "Wrong vbucket uuid in result");
110     memcpy(&recv_last_persisted_seqno, last_body.data() + 11, sizeof(uint64_t));
111 
112     switch (bucket_type) {
113     case BucketType::EP:
114         // Should get the "real" persisted seqno:
115         checkeq(last_persisted_seqno,
116                 ntohll(recv_last_persisted_seqno),
117                 "Wrong persisted seqno in result (EP)");
118         break;
119     case BucketType::Ephemeral:
120         // For ephemeral, this should always be zero, as there is no
121         // persistence.
122         checkeq(uint64_t(0),
123                 ntohll(recv_last_persisted_seqno),
124                 "Wrong persisted seqno in result (Ephemeral)");
125         break;
126     }
127 
128     memcpy(&recv_current_seqno, last_body.data() + 19, sizeof(uint64_t));
129     checkeq(current_seqno, ntohll(recv_current_seqno), "Wrong current seqno in result");
130 
131     if (failover) {
132         memcpy(&recv_failover_vbuuid, last_body.data() + 27, sizeof(uint64_t));
133         checkeq(failover_vbuuid, ntohll(recv_failover_vbuuid),
134                 "Wrong failover uuid in result");
135         memcpy(&recv_failover_seqno, last_body.data() + 35, sizeof(uint64_t));
136         checkeq(failover_seqno, ntohll(recv_failover_seqno),
137                 "Wrong failover seqno in result");
138     }
139 }
140 
test_replace_with_eviction(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)141 static enum test_result test_replace_with_eviction(ENGINE_HANDLE *h,
142                                                    ENGINE_HANDLE_V1 *h1) {
143     checkeq(ENGINE_SUCCESS,
144             store(h, h1, NULL, OPERATION_SET, "key", "somevalue"),
145             "Failed to set value.");
146     wait_for_flusher_to_settle(h, h1);
147     evict_key(h, h1, "key");
148     int numBgFetched = get_int_stat(h, h1, "ep_bg_fetched");
149 
150     checkeq(ENGINE_SUCCESS,
151             store(h, h1, NULL, OPERATION_REPLACE, "key", "somevalue1"),
152             "Failed to replace existing value.");
153 
154     checkeq(ENGINE_SUCCESS,
155             get_stats(h, {}, add_stats),
156             "Failed to get stats.");
157     std::string eviction_policy = vals.find("ep_item_eviction_policy")->second;
158     if (eviction_policy == "full_eviction") {
159         numBgFetched++;
160     }
161 
162     checkeq(numBgFetched,
163             get_int_stat(h, h1, "ep_bg_fetched"),
164             "Bg fetched value didn't match");
165 
166     check_key_value(h, h1, "key", "somevalue1", 10);
167     return SUCCESS;
168 }
169 
test_wrong_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, ENGINE_STORE_OPERATION op)170 static enum test_result test_wrong_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
171                                                ENGINE_STORE_OPERATION op) {
172     int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
173     uint64_t cas = 11;
174     if (op == OPERATION_ADD) {
175         // Add operation with cas != 0 doesn't make sense
176         cas = 0;
177     }
178     checkeq(ENGINE_NOT_MY_VBUCKET,
179             store(h, h1, NULL, op, "key", "somevalue", nullptr, cas, 1),
180             "Expected not_my_vbucket");
181     wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
182     return SUCCESS;
183 }
184 
test_pending_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, ENGINE_STORE_OPERATION op)185 static enum test_result test_pending_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
186                                                  ENGINE_STORE_OPERATION op) {
187     const void *cookie = testHarness.create_cookie();
188     testHarness.set_ewouldblock_handling(cookie, false);
189     check(set_vbucket_state(h, h1, 1, vbucket_state_pending),
190           "Failed to set vbucket state.");
191     check(verify_vbucket_state(h, h1, 1, vbucket_state_pending),
192           "Bucket state was not set to pending.");
193     uint64_t cas = 11;
194     if (op == OPERATION_ADD) {
195         // Add operation with cas != 0 doesn't make sense..
196         cas = 0;
197     }
198     checkeq(ENGINE_EWOULDBLOCK,
199             store(h, h1, cookie, op, "key", "somevalue", nullptr, cas, 1),
200             "Expected ewouldblock");
201     testHarness.destroy_cookie(cookie);
202     return SUCCESS;
203 }
204 
test_replica_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, ENGINE_STORE_OPERATION op)205 static enum test_result test_replica_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
206                                                  ENGINE_STORE_OPERATION op) {
207     check(set_vbucket_state(h, h1, 1, vbucket_state_replica),
208           "Failed to set vbucket state.");
209     check(verify_vbucket_state(h, h1, 1, vbucket_state_replica),
210           "Bucket state was not set to replica.");
211     int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
212 
213     uint64_t cas = 11;
214     if (op == OPERATION_ADD) {
215         // performing add with a CAS != 0 doesn't make sense...
216         cas = 0;
217     }
218     checkeq(ENGINE_NOT_MY_VBUCKET,
219             store(h, h1, NULL, op, "key", "somevalue", nullptr, cas, 1),
220             "Expected not my vbucket");
221     wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
222     return SUCCESS;
223 }
224 
225 //
226 // ----------------------------------------------------------------------
227 // The actual tests are below.
228 // ----------------------------------------------------------------------
229 //
230 
checkCurrItemsAfterShutdown(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, int numItems2Load, bool shutdownForce)231 static int checkCurrItemsAfterShutdown(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
232                                        int numItems2Load, bool shutdownForce) {
233     if (!isWarmupEnabled(h, h1)) {
234         return SKIPPED;
235     }
236 
237     std::vector<std::string> keys;
238     for (int index = 0; index < numItems2Load; ++index) {
239         std::stringstream s;
240         s << "keys_2_load-" << index;
241         std::string key(s.str());
242         keys.push_back(key);
243     }
244 
245     // Check preconditions.
246     checkeq(0, get_int_stat(h, h1, "ep_total_persisted"),
247             "Expected ep_total_persisted equals 0");
248     checkeq(0, get_int_stat(h, h1, "curr_items"),
249             "Expected curr_items equals 0");
250 
251     // stop flusher before loading new items
252     protocol_binary_request_header *pkt = createPacket(PROTOCOL_BINARY_CMD_STOP_PERSISTENCE);
253     checkeq(ENGINE_SUCCESS,
254             h1->unknown_command(h, NULL, pkt, add_response, testHarness.doc_namespace),
255             "CMD_STOP_PERSISTENCE failed!");
256     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS,
257             last_status.load(),
258             "Failed to stop persistence!");
259     cb_free(pkt);
260 
261     std::vector<std::string>::iterator itr;
262     for (itr = keys.begin(); itr != keys.end(); ++itr) {
263         checkeq(ENGINE_SUCCESS,
264                 store(h, h1, NULL, OPERATION_SET, itr->c_str(), "oracle"),
265                 "Failed to store a value");
266     }
267 
268     checkeq(0, get_int_stat(h, h1, "ep_total_persisted"),
269             "Incorrect ep_total_persisted, expected 0");
270 
271     // Can only check curr_items in value_only eviction; full-eviction
272     // relies on persistence to complete (via flusher) to update count.
273     const auto evictionPolicy = get_str_stat(h, h1, "ep_item_eviction_policy");
274     if (evictionPolicy == "value_only") {
275         checkeq(numItems2Load,
276                 get_int_stat(h, h1, "curr_items"),
277                 "Expected curr_items to reflect item count");
278     }
279 
280     // resume flusher before shutdown + warmup
281     pkt = createPacket(PROTOCOL_BINARY_CMD_START_PERSISTENCE);
282     checkeq(ENGINE_SUCCESS, h1->unknown_command(h, NULL, pkt, add_response, testHarness.doc_namespace),
283             "CMD_START_PERSISTENCE failed!");
284     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
285           "Failed to start persistence!");
286     cb_free(pkt);
287 
288     // shutdown engine force and restart
289     testHarness.reload_engine(&h, &h1,
290                               testHarness.engine_path,
291                               testHarness.get_current_testcase()->cfg,
292                               true, shutdownForce);
293     wait_for_warmup_complete(h, h1);
294     return get_int_stat(h, h1, "curr_items");
295 }
296 
test_flush_shutdown_force(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)297 static enum test_result test_flush_shutdown_force(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
298     if (!isWarmupEnabled(h, h1)) {
299         return SKIPPED;
300     }
301 
302     int numItems2load = 3000;
303     bool shutdownForce = true;
304     int currItems = checkCurrItemsAfterShutdown(h, h1, numItems2load, shutdownForce);
305     check (currItems <= numItems2load,
306            "Number of curr items should be <= 3000, unless previous "
307            "shutdown force had to wait for the flusher");
308     return SUCCESS;
309 }
310 
test_flush_shutdown_noforce(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)311 static enum test_result test_flush_shutdown_noforce(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
312     if (!isWarmupEnabled(h, h1)) {
313         return SKIPPED;
314     }
315 
316     int numItems2load = 3000;
317     bool shutdownForce = false;
318     int currItems = checkCurrItemsAfterShutdown(h, h1, numItems2load, shutdownForce);
319     check (currItems == numItems2load,
320            "Number of curr items should be equal to 3000, unless previous "
321            "shutdown did not wait for the flusher");
322     return SUCCESS;
323 }
324 
test_shutdown_snapshot_range(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)325 static enum test_result test_shutdown_snapshot_range(ENGINE_HANDLE *h,
326                                                      ENGINE_HANDLE_V1 *h1) {
327     if (!isWarmupEnabled(h, h1)) {
328         return SKIPPED;
329     }
330 
331     const int num_items = 100;
332     for (int j = 0; j < num_items; ++j) {
333         std::stringstream ss;
334         ss << "key" << j;
335         checkeq(ENGINE_SUCCESS,
336                 store(h, h1, NULL, OPERATION_SET, ss.str().c_str(), "data"),
337                 "Failed to store a value");
338     }
339 
340     wait_for_flusher_to_settle(h, h1);
341     int end = get_int_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno");
342 
343     /* change vb state to replica before restarting (as it happens in graceful
344        failover)*/
345     check(set_vbucket_state(h, h1, 0, vbucket_state_replica),
346           "Failed set vbucket 0 to replica state.");
347 
348     /* trigger persist vb state task */
349     check(set_param(h, h1, protocol_binary_engine_param_flush,
350                     "vb_state_persist_run", "0"),
351           "Failed to trigger vb state persist");
352 
353     /* restart the engine */
354     testHarness.reload_engine(&h, &h1,
355                               testHarness.engine_path,
356                               testHarness.get_current_testcase()->cfg,
357                               true, false);
358     wait_for_warmup_complete(h, h1);
359 
360     /* Check if snapshot range is persisted correctly */
361     checkeq(end, get_int_stat(h, h1, "vb_0:last_persisted_snap_start",
362                               "vbucket-seqno"),
363             "Wrong snapshot start persisted");
364     checkeq(end, get_int_stat(h, h1, "vb_0:last_persisted_snap_end",
365                                     "vbucket-seqno"),
366             "Wrong snapshot end persisted");
367 
368     return SUCCESS;
369 }
370 
test_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)371 static enum test_result test_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
372     if (!isWarmupEnabled(h, h1)) {
373         return SKIPPED;
374     }
375 
376     static const char val[] = "somevalue";
377     ENGINE_ERROR_CODE ret = store(h, h1, NULL, OPERATION_SET, "key", val);
378     checkeq(ENGINE_SUCCESS, ret, "Failed set.");
379 
380     testHarness.reload_engine(&h, &h1,
381                               testHarness.engine_path,
382                               testHarness.get_current_testcase()->cfg,
383                               true, false);
384     wait_for_warmup_complete(h, h1);
385     check_key_value(h, h1, "key", val, strlen(val));
386     return SUCCESS;
387 }
388 
test_specialKeys(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)389 static enum test_result test_specialKeys(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
390     ENGINE_ERROR_CODE ret;
391 
392     // Simplified Chinese "Couchbase"
393     static const char key0[] = "沙发数据库";
394     static const char val0[] = "some Chinese value";
395     check((ret = store(h, h1, NULL, OPERATION_SET, key0, val0)) ==
396                   ENGINE_SUCCESS,
397           "Failed set Chinese key");
398     check_key_value(h, h1, key0, val0, strlen(val0));
399 
400     // Traditional Chinese "Couchbase"
401     static const char key1[] = "沙發數據庫";
402     static const char val1[] = "some Traditional Chinese value";
403     check((ret = store(h, h1, NULL, OPERATION_SET, key1, val1)) ==
404                   ENGINE_SUCCESS,
405           "Failed set Traditional Chinese key");
406 
407     // Korean "couch potato"
408     static const char key2[] = "쇼파감자";
409     static const char val2[] = "some Korean value";
410     check((ret = store(h, h1, NULL, OPERATION_SET, key2, val2)) ==
411                   ENGINE_SUCCESS,
412           "Failed set Korean key");
413 
414     // Russian "couch potato"
415     static const char key3[] = "лодырь, лентяй";
416     static const char val3[] = "some Russian value";
417     check((ret = store(h, h1, NULL, OPERATION_SET, key3, val3)) ==
418                   ENGINE_SUCCESS,
419           "Failed set Russian key");
420 
421     // Japanese "couch potato"
422     static const char key4[] = "カウチポテト";
423     static const char val4[] = "some Japanese value";
424     check((ret = store(h, h1, NULL, OPERATION_SET, key4, val4)) ==
425                   ENGINE_SUCCESS,
426           "Failed set Japanese key");
427 
428     // Indian char key, and no idea what it is
429     static const char key5[] = "हरियानवी";
430     static const char val5[] = "some Indian value";
431     check((ret = store(h, h1, NULL, OPERATION_SET, key5, val5)) ==
432                   ENGINE_SUCCESS,
433           "Failed set Indian key");
434 
435     // Portuguese translation "couch potato"
436     static const char key6[] = "sedentário";
437     static const char val6[] = "some Portuguese value";
438     check((ret = store(h, h1, NULL, OPERATION_SET, key6, val6)) ==
439                   ENGINE_SUCCESS,
440           "Failed set Portuguese key");
441 
442     // Arabic translation "couch potato"
443     static const char key7[] = "الحافلةالبطاطة";
444     static const char val7[] = "some Arabic value";
445     check((ret = store(h, h1, NULL, OPERATION_SET, key7, val7)) ==
446                   ENGINE_SUCCESS,
447           "Failed set Arabic key");
448 
449     if (isWarmupEnabled(h, h1)) {
450         // Check that after warmup the keys are still present.
451         testHarness.reload_engine(&h, &h1,
452                                   testHarness.engine_path,
453                                   testHarness.get_current_testcase()->cfg,
454                                   true, false);
455         wait_for_warmup_complete(h, h1);
456         check_key_value(h, h1, key0, val0, strlen(val0));
457         check_key_value(h, h1, key1, val1, strlen(val1));
458         check_key_value(h, h1, key2, val2, strlen(val2));
459         check_key_value(h, h1, key3, val3, strlen(val3));
460         check_key_value(h, h1, key4, val4, strlen(val4));
461         check_key_value(h, h1, key5, val5, strlen(val5));
462         check_key_value(h, h1, key6, val6, strlen(val6));
463         check_key_value(h, h1, key7, val7, strlen(val7));
464     }
465     return SUCCESS;
466 }
467 
test_binKeys(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)468 static enum test_result test_binKeys(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
469     ENGINE_ERROR_CODE ret;
470 
471     // binary key with char values beyond 0x7F
472     static const char key0[] = "\xe0\xed\xf1\x6f\x7f\xf8\xfa";
473     static const char val0[] = "some value val8";
474     check((ret = store(h, h1, NULL, OPERATION_SET, key0, val0)) ==
475                   ENGINE_SUCCESS,
476           "Failed set binary key0");
477     check_key_value(h, h1, key0, val0, strlen(val0));
478 
479     // binary keys with char values beyond 0x7F
480     static const char key1[] = "\xf1\xfd\xfe\xff\xf0\xf8\xef";
481     static const char val1[] = "some value val9";
482     check((ret = store(h, h1, NULL, OPERATION_SET, key1, val1)) ==
483                   ENGINE_SUCCESS,
484           "Failed set binary key1");
485     check_key_value(h, h1, key1, val1, strlen(val1));
486 
487     // binary keys with special utf-8 BOM (Byte Order Mark) values 0xBB 0xBF
488     // 0xEF
489     static const char key2[] = "\xff\xfe\xbb\xbf\xef";
490     static const char val2[] = "some utf-8 bom value";
491     check((ret = store(h, h1, NULL, OPERATION_SET, key2, val2)) ==
492                   ENGINE_SUCCESS,
493           "Failed set binary utf-8 bom key");
494     check_key_value(h, h1, key2, val2, strlen(val2));
495 
496     // binary keys with special utf-16BE BOM values "U+FEFF"
497     static const char key3[] = "U+\xfe\xff\xefU+\xff\xfe";
498     static const char val3[] = "some utf-16 bom value";
499     check((ret = store(h, h1, NULL, OPERATION_SET, key3, val3)) ==
500                   ENGINE_SUCCESS,
501           "Failed set binary utf-16 bom key");
502     check_key_value(h, h1, key3, val3, strlen(val3));
503 
504     if (isWarmupEnabled(h, h1)) {
505         testHarness.reload_engine(&h, &h1,
506                                   testHarness.engine_path,
507                                   testHarness.get_current_testcase()->cfg,
508                                   true, false);
509         wait_for_warmup_complete(h, h1);
510         check_key_value(h, h1, key0, val0, strlen(val0));
511         check_key_value(h, h1, key1, val1, strlen(val1));
512         check_key_value(h, h1, key2, val2, strlen(val2));
513         check_key_value(h, h1, key3, val3, strlen(val3));
514     }
515     return SUCCESS;
516 }
517 
test_restart_bin_val(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)518 static enum test_result test_restart_bin_val(ENGINE_HANDLE *h,
519                                              ENGINE_HANDLE_V1 *h1) {
520     if (!isWarmupEnabled(h, h1)) {
521         return SKIPPED;
522     }
523 
524     char binaryData[] = "abcdefg\0gfedcba";
525     cb_assert(sizeof(binaryData) != strlen(binaryData));
526 
527     checkeq(cb::engine_errc::success,
528             storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
529                          binaryData, sizeof(binaryData), 82758, 0, 0).first,
530             "Failed set.");
531 
532     testHarness.reload_engine(&h, &h1,
533                               testHarness.engine_path,
534                               testHarness.get_current_testcase()->cfg,
535                               true, false);
536     wait_for_warmup_complete(h, h1);
537 
538     check_key_value(h, h1, "key", binaryData, sizeof(binaryData));
539     return SUCCESS;
540 }
541 
test_wrong_vb_get(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)542 static enum test_result test_wrong_vb_get(ENGINE_HANDLE *h,
543                                           ENGINE_HANDLE_V1 *h1) {
544     int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
545     checkeq(ENGINE_NOT_MY_VBUCKET, verify_key(h, h1, "key", 1),
546             "Expected wrong bucket.");
547     wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
548     return SUCCESS;
549 }
550 
test_vb_get_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)551 static enum test_result test_vb_get_pending(ENGINE_HANDLE *h,
552                                             ENGINE_HANDLE_V1 *h1) {
553     check(set_vbucket_state(h, h1, 1, vbucket_state_pending),
554           "Failed to set vbucket state.");
555     const void *cookie = testHarness.create_cookie();
556     testHarness.set_ewouldblock_handling(cookie, false);
557 
558     checkeq(cb::engine_errc::would_block,
559             get(h, h1, cookie, "key", 1).first,
560             "Expected woodblock.");
561 
562     testHarness.destroy_cookie(cookie);
563     return SUCCESS;
564 }
565 
test_vb_get_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)566 static enum test_result test_vb_get_replica(ENGINE_HANDLE *h,
567                                             ENGINE_HANDLE_V1 *h1) {
568     check(set_vbucket_state(h, h1, 1, vbucket_state_replica),
569           "Failed to set vbucket state.");
570     int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
571     checkeq(ENGINE_NOT_MY_VBUCKET,
572             verify_key(h, h1, "key", 1),
573             "Expected not my bucket.");
574     wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
575     return SUCCESS;
576 }
577 
test_wrong_vb_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)578 static enum test_result test_wrong_vb_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
579     return test_wrong_vb_mutation(h, h1, OPERATION_SET);
580 }
581 
test_wrong_vb_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)582 static enum test_result test_wrong_vb_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
583     return test_wrong_vb_mutation(h, h1, OPERATION_CAS);
584 }
585 
test_wrong_vb_add(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)586 static enum test_result test_wrong_vb_add(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
587     return test_wrong_vb_mutation(h, h1, OPERATION_ADD);
588 }
589 
test_wrong_vb_replace(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)590 static enum test_result test_wrong_vb_replace(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
591     return test_wrong_vb_mutation(h, h1, OPERATION_REPLACE);
592 }
593 
test_wrong_vb_del(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)594 static enum test_result test_wrong_vb_del(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
595     int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
596     checkeq(ENGINE_NOT_MY_VBUCKET, del(h, h1, "key", 0, 1),
597             "Expected wrong bucket.");
598     wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
599     return SUCCESS;
600 }
601 
602 /* Returns a string in the format "%Y-%m-%d %H:%M:%S" of the specified
603  * time point.
604  */
make_time_string(std::chrono::system_clock::time_point time_point)605 std::string make_time_string(std::chrono::system_clock::time_point time_point) {
606     time_t tt = std::chrono::system_clock::to_time_t(time_point);
607 #ifdef _MSC_VER
608     // Windows' gmtime() is already thread-safe.
609     struct tm* split = gmtime(&tt);
610 #else
611     struct tm local_storage;
612     struct tm* split = gmtime_r(&tt, &local_storage);
613 #endif
614     char timeStr[20];
615     strftime(timeStr, 20, "%Y-%m-%d %H:%M:%S", split);
616     return timeStr;
617 }
618 
test_expiry_pager_settings(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)619 static enum test_result test_expiry_pager_settings(ENGINE_HANDLE *h,
620                                                    ENGINE_HANDLE_V1 *h1) {
621 
622     cb_assert(!get_bool_stat(h, h1, "ep_exp_pager_enabled"));
623     checkeq(3600, get_int_stat(h, h1, "ep_exp_pager_stime"),
624             "Expiry pager sleep time not expected");
625     set_param(h, h1, protocol_binary_engine_param_flush,
626               "exp_pager_stime", "1");
627     checkeq(1, get_int_stat(h, h1, "ep_exp_pager_stime"),
628             "Expiry pager sleep time not updated");
629     cb_assert(!get_bool_stat(h, h1, "ep_exp_pager_enabled"));
630     sleep(1);
631     checkeq(0, get_int_stat(h, h1, "ep_num_expiry_pager_runs"),
632             "Expiry pager run count is not zero");
633 
634     set_param(h, h1, protocol_binary_engine_param_flush,
635               "exp_pager_enabled", "true");
636     checkeq(1, get_int_stat(h, h1, "ep_exp_pager_stime"),
637             "Expiry pager sleep time not updated");
638     wait_for_stat_to_be_gte(h, h1, "ep_num_expiry_pager_runs", 1);
639 
640     // Reload engine
641     testHarness.reload_engine(&h, &h1,
642                               testHarness.engine_path,
643                               testHarness.get_current_testcase()->cfg,
644                               true, false);
645     wait_for_warmup_complete(h, h1);
646     cb_assert(!get_bool_stat(h, h1, "ep_exp_pager_enabled"));
647 
648     // Enable expiry pager again
649     set_param(h, h1, protocol_binary_engine_param_flush,
650               "exp_pager_enabled", "true");
651 
652     checkeq(get_int_stat(h, h1, "ep_exp_pager_initial_run_time"), -1,
653             "Task time should be disable upon warmup");
654 
655     std::string err_msg;
656     // Update exp_pager_initial_run_time and ensure the update is successful
657     set_param(h, h1, protocol_binary_engine_param_flush,
658               "exp_pager_initial_run_time", "3");
659     std::string expected_time = "03:00";
660     std::string str;
661     // [MB-21806] - Need to repeat the fetch as the set_param for
662     // "exp_pager_initial_run_time" schedules a task that sets the stats later
663     repeat_till_true([&]() {
664         str = get_str_stat(h, h1, "ep_expiry_pager_task_time");
665         return 0 == str.substr(11, 5).compare(expected_time);
666     });
667     err_msg.assign("Updated time incorrect, expect: " +
668                    expected_time + ", actual: " + str.substr(11, 5));
669     checkeq(0, str.substr(11, 5).compare(expected_time), err_msg.c_str());
670 
671     // Update exp_pager_stime by 30 minutes and ensure that the update is successful
672     const std::chrono::minutes update_by{30};
673     std::string targetTaskTime1{make_time_string(std::chrono::system_clock::now() +
674                                                  update_by)};
675 
676     set_param(h, h1, protocol_binary_engine_param_flush, "exp_pager_stime",
677               std::to_string(update_by.count() * 60).c_str());
678     str = get_str_stat(h, h1, "ep_expiry_pager_task_time");
679 
680     std::string targetTaskTime2{make_time_string(std::chrono::system_clock::now() +
681                                                  update_by)};
682 
683     // ep_expiry_pager_task_time should fall within the range of
684     // targetTaskTime1 and targetTaskTime2
685     err_msg.assign("Unexpected task time range, expect: " +
686                    targetTaskTime1 + " <= " + str + " <= " + targetTaskTime2);
687     check(targetTaskTime1 <= str, err_msg.c_str());
688     check(str <= targetTaskTime2, err_msg.c_str());
689 
690     return SUCCESS;
691 }
692 
test_expiry_with_xattr(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1)693 static enum test_result test_expiry_with_xattr(ENGINE_HANDLE* h,
694                                                ENGINE_HANDLE_V1* h1) {
695     const char* key = "test_expiry";
696     cb::xattr::Blob blob;
697 
698     //Add a few XAttrs
699     blob.set("user", "{\"author\":\"bubba\"}");
700     blob.set("_sync", "{\"cas\":\"0xdeadbeefcafefeed\"}");
701     blob.set("meta", "{\"content-type\":\"text\"}");
702 
703     auto xattr_value = blob.finalize();
704 
705     //Now, append user data to the xattrs and store the data
706     std::string value_data("test_expiry_value");
707     std::vector<char> data;
708     std::copy(xattr_value.buf, xattr_value.buf + xattr_value.len,
709               std::back_inserter(data));
710     std::copy(value_data.c_str(), value_data.c_str() + value_data.length(),
711               std::back_inserter(data));
712 
713     const void* cookie = testHarness.create_cookie();
714 
715     checkeq(cb::engine_errc::success,
716             storeCasVb11(h, h1, cookie, OPERATION_SET, key,
717                          reinterpret_cast<char*>(data.data()),
718                          data.size(), 9258, 0, 0, 10,
719                          PROTOCOL_BINARY_DATATYPE_XATTR).first,
720             "Failed to store xattr document");
721 
722     if (isPersistentBucket(h, h1)) {
723         wait_for_flusher_to_settle(h, h1);
724     }
725 
726     testHarness.time_travel(11);
727 
728     cb::EngineErrorMetadataPair errorMetaPair;
729 
730     check(get_meta(h, h1, "test_expiry", errorMetaPair, cookie),
731           "Get meta command failed");
732     auto prev_revseqno = errorMetaPair.second.seqno;
733 
734     checkeq(PROTOCOL_BINARY_DATATYPE_XATTR,
735             errorMetaPair.second.datatype,
736             "Datatype is not XATTR");
737 
738     auto ret = get(h, h1, cookie, key, 0, DocStateFilter::AliveOrDeleted);
739     checkeq(cb::engine_errc::success,
740             ret.first,
741             "Unable to get a deleted item");
742 
743     check(get_meta(h, h1, "test_expiry", errorMetaPair, cookie),
744           "Get meta command failed");
745 
746     checkeq(errorMetaPair.second.seqno,
747             prev_revseqno + 1,
748             "rev seqno must have incremented by 1");
749 
750     /* Retrieve the item info and create a new blob out of the data */
751     item_info info;
752     checkeq(true,
753             h1->get_item_info(h, ret.second.get(), &info),
754             "Unable to retrieve item info");
755 
756     cb::char_buffer value_buf{static_cast<char*>(info.value[0].iov_base),
757                               info.value[0].iov_len};
758 
759     cb::xattr::Blob new_blob(value_buf, false);
760 
761     /* Only system extended attributes need to be present at this point.
762      * Thus, check the blob length with the system size.
763      */
764     const auto systemsize = new_blob.finalize().len;
765 
766     checkeq(systemsize, new_blob.get_system_size(),
767             "The size of the blob doesn't match the size of system attributes");
768 
769     const std::string& cas_str{"{\"cas\":\"0xdeadbeefcafefeed\"}"};
770     const std::string& sync_str = to_string(blob.get("_sync"));
771 
772     checkeq(cas_str, sync_str , "system xattr is invalid");
773 
774     testHarness.destroy_cookie(cookie);
775 
776     return SUCCESS;
777 }
778 
test_expiry(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)779 static enum test_result test_expiry(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
780     const char *key = "test_expiry";
781     const char *data = "some test data here.";
782 
783     const void* cookie = testHarness.create_cookie();
784     auto ret = allocate(h,
785                         h1,
786                         cookie,
787                         key,
788                         strlen(data),
789                         0,
790                         2,
791                         PROTOCOL_BINARY_RAW_BYTES,
792                         0);
793     checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
794 
795     item_info info;
796     if (!h1->get_item_info(h, ret.second.get(), &info)) {
797         abort();
798     }
799     memcpy(info.value[0].iov_base, data, strlen(data));
800 
801     uint64_t cas = 0;
802     auto rv = h1->store(h,
803                         cookie,
804                         ret.second.get(),
805                         cas,
806                         OPERATION_SET,
807                         DocumentState::Alive);
808     checkeq(ENGINE_SUCCESS, rv, "Set failed.");
809     check_key_value(h, h1, key, data, strlen(data));
810 
811     testHarness.time_travel(5);
812     checkeq(cb::engine_errc::no_such_key,
813             get(h, h1, cookie, key, 0).first,
814             "Item didn't expire");
815 
816     int expired_access = get_int_stat(h, h1, "ep_expired_access");
817     int expired_pager = get_int_stat(h, h1, "ep_expired_pager");
818     int active_expired = get_int_stat(h, h1, "vb_active_expired");
819     checkeq(0, expired_pager, "Expected zero expired item by pager");
820     checkeq(1, expired_access, "Expected an expired item on access");
821     checkeq(1, active_expired, "Expected an expired active item");
822     checkeq(ENGINE_SUCCESS,
823             store(h, h1, cookie, OPERATION_SET, key, data),
824             "Failed set.");
825 
826     // When run under full eviction, the total item stats are set from the
827     // flusher. So we need to wait for it to finish before checking the
828     // total number of items.
829     wait_for_flusher_to_settle(h, h1);
830 
831     std::stringstream ss;
832     ss << "curr_items stat should be still 1 after ";
833     ss << "overwriting the key that was expired, but not purged yet";
834     checkeq(1, get_int_stat(h, h1, "curr_items"), ss.str().c_str());
835 
836     testHarness.destroy_cookie(cookie);
837     return SUCCESS;
838 }
839 
test_expiry_loader(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)840 static enum test_result test_expiry_loader(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
841     if (!isWarmupEnabled(h, h1)) {
842         return SKIPPED;
843     }
844     const char *key = "test_expiry_loader";
845     const char *data = "some test data here.";
846 
847     const void* cookie = testHarness.create_cookie();
848     auto ret = allocate(h,
849                         h1,
850                         cookie,
851                         key,
852                         strlen(data),
853                         0,
854                         2,
855                         PROTOCOL_BINARY_RAW_BYTES,
856                         0);
857     checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
858 
859     item_info info;
860     if (!h1->get_item_info(h, ret.second.get(), &info)) {
861         abort();
862     }
863     memcpy(info.value[0].iov_base, data, strlen(data));
864 
865     uint64_t cas = 0;
866     auto rv = h1->store(h,
867                         cookie,
868                         ret.second.get(),
869                         cas,
870                         OPERATION_SET,
871                         DocumentState::Alive);
872     checkeq(ENGINE_SUCCESS, rv, "Set failed.");
873     check_key_value(h, h1, key, data, strlen(data));
874 
875     testHarness.time_travel(3);
876 
877     ret = get(h, h1, cookie, key, 0);
878     checkeq(cb::engine_errc::no_such_key, ret.first, "Item didn't expire");
879 
880     // Restart the engine to ensure the above expired item is not loaded
881     testHarness.reload_engine(&h, &h1,
882                               testHarness.engine_path,
883                               testHarness.get_current_testcase()->cfg,
884                               true, false);
885     wait_for_warmup_complete(h, h1);
886     cb_assert(0 == get_int_stat(h, h1, "ep_warmup_value_count", "warmup"));
887 
888     testHarness.destroy_cookie(cookie);
889 
890     return SUCCESS;
891 }
892 
test_expiration_on_compaction(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)893 static enum test_result test_expiration_on_compaction(ENGINE_HANDLE *h,
894                                                       ENGINE_HANDLE_V1 *h1) {
895     if (get_bool_stat(h, h1, "ep_exp_pager_enabled")) {
896         set_param(h, h1, protocol_binary_engine_param_flush,
897                   "exp_pager_enabled", "false");
898     }
899 
900     checkeq(1,
901             get_int_stat(h, h1, "vb_0:persistence:num_visits", "checkpoint"),
902             "Cursor moved before item load");
903 
904     for (int i = 0; i < 25; i++) {
905         std::stringstream ss;
906         ss << "key" << i;
907         checkeq(ENGINE_SUCCESS,
908                 store(h,
909                       h1,
910                       NULL,
911                       OPERATION_SET,
912                       ss.str().c_str(),
913                       "somevalue",
914                       nullptr,
915                       0,
916                       0,
917                       10,
918                       PROTOCOL_BINARY_RAW_BYTES),
919                 "Set failed.");
920     }
921 
922     // Throw an xattr document in (and later compressed)
923     cb::xattr::Blob builder;
924     builder.set("_ep", "{\"foo\":\"bar\"}");
925     builder.set("key", "{\"foo\":\"bar\"}");
926     builder.set("_sync", "{\"foo\":\"bar\"}");
927     builder.set("stuff", "{\"foo\":\"bar\"}");
928     builder.set("misc", "{\"foo\":\"bar\"}");
929     builder.set("things", "{\"foo\":\"bar\"}");
930     builder.set("that", "{\"foo\":\"bar\"}");
931 
932     auto blob = builder.finalize();
933     std::string data;
934     std::copy(blob.buf, blob.buf + blob.size(), std::back_inserter(data));
935     for (int i = 0; i < 12; i++) {
936         std::stringstream ss;
937         ss << "xattr_key" << i;
938 
939         checkeq(cb::engine_errc::success,
940                 storeCasVb11(h,
941                              h1,
942                              nullptr,
943                              OPERATION_SET,
944                              ss.str().c_str(),
945                              data.data(),
946                              data.size(),
947                              0,
948                              0,
949                              0,
950                              10,
951                              PROTOCOL_BINARY_DATATYPE_XATTR,
952                              DocumentState::Alive)
953                         .first,
954                 "Unable to store item");
955     }
956 
957     cb::compression::Buffer compressedDoc;
958     cb::compression::deflate(
959             cb::compression::Algorithm::Snappy, data, compressedDoc);
960 
961     for (int i = 0; i < 13; i++) {
962         std::stringstream ss;
963         ss << "compressed_xattr_key" << i;
964 
965         checkeq(cb::engine_errc::success,
966                 storeCasVb11(h,
967                              h1,
968                              nullptr,
969                              OPERATION_SET,
970                              ss.str().c_str(),
971                              compressedDoc.data(),
972                              compressedDoc.size(),
973                              0,
974                              0,
975                              0,
976                              10,
977                              PROTOCOL_BINARY_DATATYPE_XATTR |
978                                      PROTOCOL_BINARY_DATATYPE_SNAPPY,
979                              DocumentState::Alive)
980                         .first,
981                 "Unable to store item");
982     }
983 
984     wait_for_flusher_to_settle(h, h1);
985     checkeq(50, get_int_stat(h, h1, "curr_items"),
986             "Unexpected number of items on database");
987     check(1 < get_int_stat(h, h1, "vb_0:persistence:num_visits", "checkpoint"),
988           "Cursor not moved even after flusher runs");
989 
990     testHarness.time_travel(15);
991 
992     // Compaction on VBucket
993     compact_db(h, h1, 0, 0, 0, 0, 0);
994     wait_for_stat_to_be(h, h1, "ep_pending_compactions", 0);
995 
996     checkeq(50, get_int_stat(h, h1, "ep_expired_compactor"),
997             "Unexpected expirations by compactor");
998 
999     return SUCCESS;
1000 }
1001 
test_expiration_on_warmup(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1002 static enum test_result test_expiration_on_warmup(ENGINE_HANDLE *h,
1003                                                   ENGINE_HANDLE_V1 *h1) {
1004     if (!isWarmupEnabled(h, h1)) {
1005         return SKIPPED;
1006     }
1007 
1008     const auto* cookie = testHarness.create_cookie();
1009     set_param(h, h1, protocol_binary_engine_param_flush,
1010               "exp_pager_enabled", "false");
1011     int pager_runs = get_int_stat(h, h1, "ep_num_expiry_pager_runs");
1012 
1013     const char *key = "KEY";
1014     const char *data = "VALUE";
1015 
1016     auto ret = allocate(h,
1017                         h1,
1018                         cookie,
1019                         key,
1020                         strlen(data),
1021                         0,
1022                         10,
1023                         PROTOCOL_BINARY_RAW_BYTES,
1024                         0);
1025     checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
1026 
1027     item_info info;
1028     if (!h1->get_item_info(h, ret.second.get(), &info)) {
1029         abort();
1030     }
1031     memcpy(info.value[0].iov_base, data, strlen(data));
1032 
1033     uint64_t cas = 0;
1034     auto rv = h1->store(h,
1035                         cookie,
1036                         ret.second.get(),
1037                         cas,
1038                         OPERATION_SET,
1039                         DocumentState::Alive);
1040     checkeq(ENGINE_SUCCESS, rv, "Set failed.");
1041     check_key_value(h, h1, key, data, strlen(data));
1042     ret.second.reset();
1043     wait_for_flusher_to_settle(h, h1);
1044 
1045     checkeq(1, get_int_stat(h, h1, "curr_items"), "Failed store item");
1046     testHarness.time_travel(15);
1047 
1048     checkeq(pager_runs, get_int_stat(h, h1, "ep_num_expiry_pager_runs"),
1049             "Expiry pager shouldn't have run during this time");
1050 
1051     // Restart the engine to ensure the above item is expired
1052     testHarness.reload_engine(&h, &h1,
1053                               testHarness.engine_path,
1054                               testHarness.get_current_testcase()->cfg,
1055                               true, false);
1056     wait_for_warmup_complete(h, h1);
1057     check(get_bool_stat(h, h1, "ep_exp_pager_enabled"),
1058           "Expiry pager should be enabled on warmup");
1059 
1060     // Wait for the expiry pager to run and expire our item.
1061     wait_for_stat_to_be_gte(h, h1, "ep_expired_pager", 1, nullptr, /*secs*/10);
1062 
1063     // Note: previously we checked that curr_items was zero here (immediately
1064     // after waiting for ep_expired_pager == 1), however we cannot assume that
1065     // - items are actually expired asynchronously.
1066     // See EPStore::deleteExpiredItem - for non-temporary, expired items we
1067     // call processSoftDelete (soft-marking the item as deleted in the
1068     // hashtable), and then call queueDirty to queue a deletion, and then
1069     // increment the expired stat. Only when that delete is actually persisted
1070     // and the deleted callback is invoked -
1071     // PeristenceCallback::callback(int&) - is curr_items finally decremented.
1072     // Therefore we need to wait for the flusher to settle (i.e. delete
1073     // callback to be called) for the curr_items stat to be accurate.
1074     wait_for_flusher_to_settle(h, h1);
1075 
1076     checkeq(0, get_int_stat(h, h1, "curr_items"),
1077             "The item should have been expired.");
1078 
1079     testHarness.destroy_cookie(cookie);
1080     return SUCCESS;
1081 
1082 }
1083 
test_bug3454(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1084 static enum test_result test_bug3454(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1085     if (!isWarmupEnabled(h, h1)) {
1086         return SKIPPED;
1087     }
1088 
1089     const char *key = "test_expiry_duplicate_warmup";
1090     const char *data = "some test data here.";
1091 
1092     const void* cookie = testHarness.create_cookie();
1093     auto ret = allocate(h,
1094                         h1,
1095                         cookie,
1096                         key,
1097                         strlen(data),
1098                         0,
1099                         5,
1100                         PROTOCOL_BINARY_RAW_BYTES,
1101                         0);
1102     checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
1103 
1104     item_info info;
1105     if (!h1->get_item_info(h, ret.second.get(), &info)) {
1106         abort();
1107     }
1108     memcpy(info.value[0].iov_base, data, strlen(data));
1109 
1110     uint64_t cas = 0;
1111     auto rv = h1->store(h,
1112                         cookie,
1113                         ret.second.get(),
1114                         cas,
1115                         OPERATION_SET,
1116                         DocumentState::Alive);
1117     checkeq(ENGINE_SUCCESS, rv, "Set failed.");
1118     check_key_value(h, h1, key, data, strlen(data));
1119     wait_for_flusher_to_settle(h, h1);
1120 
1121     // Advance the ep_engine time by 10 sec for the above item to be expired.
1122     testHarness.time_travel(10);
1123     ret = get(h, h1, cookie, key, 0);
1124     checkeq(cb::engine_errc::no_such_key, ret.first, "Item didn't expire");
1125 
1126     ret = allocate(h,
1127                    h1,
1128                    cookie,
1129                    key,
1130                    strlen(data),
1131                    0,
1132                    0,
1133                    PROTOCOL_BINARY_RAW_BYTES,
1134                    0);
1135     checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
1136 
1137     if (!h1->get_item_info(h, ret.second.get(), &info)) {
1138         abort();
1139     }
1140     memcpy(info.value[0].iov_base, data, strlen(data));
1141 
1142     cas = 0;
1143     // Add a new item with the same key.
1144     rv = h1->store(h,
1145                    cookie,
1146                    ret.second.get(),
1147                    cas,
1148                    OPERATION_ADD,
1149                    DocumentState::Alive);
1150     checkeq(ENGINE_SUCCESS, rv, "Add failed.");
1151     check_key_value(h, h1, key, data, strlen(data));
1152     ret.second.reset();
1153     wait_for_flusher_to_settle(h, h1);
1154 
1155     checkeq(cb::engine_errc::success,
1156             get(h, h1, cookie, key, 0).first,
1157             "Item shouldn't expire");
1158 
1159     // Restart the engine to ensure the above unexpired new item is loaded
1160     testHarness.reload_engine(&h, &h1,
1161                               testHarness.engine_path,
1162                               testHarness.get_current_testcase()->cfg,
1163                               true, false);
1164     wait_for_warmup_complete(h, h1);
1165     cb_assert(1 == get_int_stat(h, h1, "ep_warmup_value_count", "warmup"));
1166     cb_assert(0 == get_int_stat(h, h1, "ep_warmup_dups", "warmup"));
1167 
1168     testHarness.destroy_cookie(cookie);
1169     return SUCCESS;
1170 }
1171 
test_bug3522(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1172 static enum test_result test_bug3522(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1173     if (!isWarmupEnabled(h, h1)) {
1174         return SKIPPED;
1175     }
1176 
1177     const char *key = "test_expiry_no_items_warmup";
1178     const char *data = "some test data here.";
1179 
1180     const void* cookie = testHarness.create_cookie();
1181     auto ret = allocate(h,
1182                         h1,
1183                         cookie,
1184                         key,
1185                         strlen(data),
1186                         0,
1187                         0,
1188                         PROTOCOL_BINARY_RAW_BYTES,
1189                         0);
1190     checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
1191 
1192     item_info info;
1193     if (!h1->get_item_info(h, ret.second.get(), &info)) {
1194         abort();
1195     }
1196     memcpy(info.value[0].iov_base, data, strlen(data));
1197 
1198     uint64_t cas = 0;
1199     auto rv = h1->store(h,
1200                         cookie,
1201                         ret.second.get(),
1202                         cas,
1203                         OPERATION_SET,
1204                         DocumentState::Alive);
1205     checkeq(ENGINE_SUCCESS, rv, "Set failed.");
1206     check_key_value(h, h1, key, data, strlen(data));
1207     wait_for_flusher_to_settle(h, h1);
1208 
1209     // Add a new item with the same key and 2 sec of expiration.
1210     const char *new_data = "new data here.";
1211     ret = allocate(h,
1212                    h1,
1213                    cookie,
1214                    key,
1215                    strlen(new_data),
1216                    0,
1217                    2,
1218                    PROTOCOL_BINARY_RAW_BYTES,
1219                    0);
1220     checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
1221 
1222     if (!h1->get_item_info(h, ret.second.get(), &info)) {
1223         abort();
1224     }
1225     memcpy(info.value[0].iov_base, new_data, strlen(new_data));
1226 
1227     int pager_runs = get_int_stat(h, h1, "ep_num_expiry_pager_runs");
1228     cas = 0;
1229     rv = h1->store(h,
1230                    cookie,
1231                    ret.second.get(),
1232                    cas,
1233                    OPERATION_SET,
1234                    DocumentState::Alive);
1235     checkeq(ENGINE_SUCCESS, rv, "Set failed.");
1236     check_key_value(h, h1, key, new_data, strlen(new_data));
1237     ret.second.reset();
1238     testHarness.time_travel(3);
1239     wait_for_stat_change(h, h1, "ep_num_expiry_pager_runs", pager_runs);
1240     wait_for_flusher_to_settle(h, h1);
1241 
1242     // Restart the engine.
1243     testHarness.reload_engine(&h, &h1,
1244                               testHarness.engine_path,
1245                               testHarness.get_current_testcase()->cfg,
1246                               true, false);
1247     wait_for_warmup_complete(h, h1);
1248     // TODO: modify this for a better test case
1249     cb_assert(0 == get_int_stat(h, h1, "ep_warmup_dups", "warmup"));
1250 
1251     testHarness.destroy_cookie(cookie);
1252     return SUCCESS;
1253 }
1254 
test_get_replica_active_state(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1255 static enum test_result test_get_replica_active_state(ENGINE_HANDLE *h,
1256                                                       ENGINE_HANDLE_V1 *h1) {
1257     protocol_binary_request_header *pkt;
1258     pkt = prepare_get_replica(h, h1, vbucket_state_active);
1259     checkeq(ENGINE_NOT_MY_VBUCKET,
1260             h1->unknown_command(
1261                     h, NULL, pkt, add_response, testHarness.doc_namespace),
1262             "Get Replica Failed");
1263 
1264     cb_free(pkt);
1265     return SUCCESS;
1266 }
1267 
test_get_replica_pending_state(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1268 static enum test_result test_get_replica_pending_state(ENGINE_HANDLE *h,
1269                                                        ENGINE_HANDLE_V1 *h1) {
1270     protocol_binary_request_header *pkt;
1271 
1272     const void *cookie = testHarness.create_cookie();
1273     testHarness.set_ewouldblock_handling(cookie, false);
1274     pkt = prepare_get_replica(h, h1, vbucket_state_pending);
1275     checkeq(ENGINE_NOT_MY_VBUCKET,
1276             h1->unknown_command(h, cookie, pkt, add_response, testHarness.doc_namespace),
1277             "Should have returned NOT_MY_VBUCKET for pending state");
1278     checkeq(1, get_int_stat(h, h1, "ep_num_not_my_vbuckets"), "Expected 1 get");
1279     testHarness.destroy_cookie(cookie);
1280     cb_free(pkt);
1281     return SUCCESS;
1282 }
1283 
test_get_replica_dead_state(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1284 static enum test_result test_get_replica_dead_state(ENGINE_HANDLE *h,
1285                                                     ENGINE_HANDLE_V1 *h1) {
1286     protocol_binary_request_header *pkt;
1287     pkt = prepare_get_replica(h, h1, vbucket_state_dead);
1288     checkeq(ENGINE_NOT_MY_VBUCKET,
1289             h1->unknown_command(
1290                     h, NULL, pkt, add_response, testHarness.doc_namespace),
1291             "Get Replica Failed");
1292     cb_free(pkt);
1293     return SUCCESS;
1294 }
1295 
test_get_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1296 static enum test_result test_get_replica(ENGINE_HANDLE *h,
1297                                          ENGINE_HANDLE_V1 *h1) {
1298     protocol_binary_request_header *pkt;
1299     pkt = prepare_get_replica(h, h1, vbucket_state_replica);
1300     checkeq(ENGINE_SUCCESS,
1301             h1->unknown_command(h, NULL, pkt, add_response, testHarness.doc_namespace),
1302             "Get Replica Failed");
1303     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1304             "Expected PROTOCOL_BINARY_RESPONSE_SUCCESS response.");
1305     checkeq(std::string("replicadata"), last_body,
1306             "Should have returned identical value");
1307 
1308     cb_free(pkt);
1309     return SUCCESS;
1310 }
1311 
test_get_replica_non_resident(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1312 static enum test_result test_get_replica_non_resident(ENGINE_HANDLE *h,
1313                                                       ENGINE_HANDLE_V1 *h1) {
1314     checkeq(ENGINE_SUCCESS,
1315             store(h, h1, NULL, OPERATION_SET, "key", "value"),
1316             "Store Failed");
1317     wait_for_flusher_to_settle(h, h1);
1318     wait_for_stat_to_be(h, h1, "ep_total_persisted", 1);
1319 
1320     evict_key(h, h1, "key", 0, "Ejected.");
1321     check(set_vbucket_state(h, h1, 0, vbucket_state_replica),
1322           "Failed to set vbucket to replica");
1323 
1324     get_replica(h, h1, "key", 0);
1325     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1326             "Expected success");
1327 
1328     return SUCCESS;
1329 }
1330 
test_get_replica_invalid_key(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1331 static enum test_result test_get_replica_invalid_key(ENGINE_HANDLE *h,
1332                                                      ENGINE_HANDLE_V1 *h1) {
1333     protocol_binary_request_header *pkt;
1334     bool makeinvalidkey = true;
1335     pkt = prepare_get_replica(h, h1, vbucket_state_replica, makeinvalidkey);
1336     checkeq(ENGINE_NOT_MY_VBUCKET,
1337             h1->unknown_command(
1338                     h, NULL, pkt, add_response, testHarness.doc_namespace),
1339             "Get Replica Failed");
1340     cb_free(pkt);
1341     return SUCCESS;
1342 }
1343 
test_vb_del_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1344 static enum test_result test_vb_del_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1345     const void *cookie = testHarness.create_cookie();
1346     testHarness.set_ewouldblock_handling(cookie, false);
1347     check(set_vbucket_state(h, h1, 1, vbucket_state_pending),
1348           "Failed to set vbucket state.");
1349     checkeq(ENGINE_EWOULDBLOCK, del(h, h1, "key", 0, 1, cookie),
1350             "Expected woodblock.");
1351     testHarness.destroy_cookie(cookie);
1352     return SUCCESS;
1353 }
1354 
test_vb_del_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1355 static enum test_result test_vb_del_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1356     check(set_vbucket_state(h, h1, 1, vbucket_state_replica),
1357           "Failed to set vbucket state.");
1358     int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1359     checkeq(ENGINE_NOT_MY_VBUCKET, del(h, h1, "key", 0, 1),
1360             "Expected not my vbucket.");
1361     wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1362     return SUCCESS;
1363 }
1364 
test_vbucket_get_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1365 static enum test_result test_vbucket_get_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1366     return verify_vbucket_missing(h, h1, 1) ? SUCCESS : FAIL;
1367 }
1368 
test_vbucket_get(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1369 static enum test_result test_vbucket_get(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1370     return verify_vbucket_state(h, h1, 0, vbucket_state_active) ? SUCCESS : FAIL;
1371 }
1372 
test_vbucket_create(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1373 static enum test_result test_vbucket_create(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1374     if (!verify_vbucket_missing(h, h1, 1)) {
1375         fprintf(stderr, "vbucket wasn't missing.\n");
1376         return FAIL;
1377     }
1378 
1379     if (!set_vbucket_state(h, h1, 1, vbucket_state_active)) {
1380         fprintf(stderr, "set state failed.\n");
1381         return FAIL;
1382     }
1383 
1384     return verify_vbucket_state(h, h1, 1, vbucket_state_active) ? SUCCESS : FAIL;
1385 }
1386 
test_takeover_stats_race_with_vb_create_DCP( ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1387 static enum test_result test_takeover_stats_race_with_vb_create_DCP(
1388                                     ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1389     check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1390           "Failed to set vbucket state information");
1391 
1392     checkeq(0,
1393             get_int_stat(h, h1, "on_disk_deletes", "dcp-vbtakeover 1"),
1394             "Invalid number of on-disk deletes");
1395 
1396     return SUCCESS;
1397 }
1398 
test_takeover_stats_num_persisted_deletes( ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1399 static enum test_result test_takeover_stats_num_persisted_deletes(
1400                                     ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1401     /* set an item */
1402     std::string key("key");
1403     checkeq(ENGINE_SUCCESS,
1404             store(h, h1, NULL, OPERATION_SET, key.c_str(), "data"),
1405             "Failed to store an item");
1406 
1407     /* delete the item */
1408     checkeq(ENGINE_SUCCESS, del(h, h1, key.c_str(), 0, 0),
1409             "Failed to delete the item");
1410 
1411     /* wait for persistence */
1412     wait_for_flusher_to_settle(h, h1);
1413 
1414     /* check if persisted deletes stats is got correctly */
1415     checkeq(1,
1416             get_int_stat(h, h1, "on_disk_deletes", "dcp-vbtakeover 0"),
1417             "Invalid number of on-disk deletes");
1418 
1419     return SUCCESS;
1420 }
1421 
test_vbucket_compact(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1422 static enum test_result test_vbucket_compact(ENGINE_HANDLE *h,
1423                                              ENGINE_HANDLE_V1 *h1) {
1424     const char* exp_key = "Carss";
1425     const char* exp_value = "pollute";
1426     const char* non_exp_key = "trees";
1427     const char* non_exp_value = "cleanse";
1428 
1429     // Set two keys - one to be expired and other to remain...
1430     // Set expiring key
1431     checkeq(ENGINE_SUCCESS,
1432             store(h, h1, NULL, OPERATION_SET, exp_key, exp_value),
1433             "Failed to set expiring key");
1434     check_key_value(h, h1, exp_key, exp_value, strlen(exp_value));
1435 
1436     // Set a non-expiring key...
1437     checkeq(ENGINE_SUCCESS,
1438             store(h, h1, NULL, OPERATION_SET, non_exp_key, non_exp_value),
1439             "Failed to set non-expiring key");
1440     check_key_value(h, h1, non_exp_key, non_exp_value, strlen(non_exp_value));
1441 
1442     // Touch expiring key with an expire time
1443     const int exp_time = 11;
1444     checkeq(ENGINE_SUCCESS,
1445             touch(h, h1, exp_key, 0, exp_time),
1446             "Touch expiring key failed");
1447 
1448     // Move beyond expire time
1449     testHarness.time_travel(exp_time + 1);
1450 
1451     // Wait for the item to be expired
1452     wait_for_stat_to_be(h, h1, "vb_active_expired", 1);
1453     const int exp_purge_seqno =
1454             get_int_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno");
1455 
1456     // non_exp_key and its value should be intact...
1457     checkeq(ENGINE_SUCCESS,
1458             verify_key(h, h1, non_exp_key),
1459             "key trees should be found.");
1460     // exp_key should have disappeared...
1461     ENGINE_ERROR_CODE val = verify_key(h, h1, exp_key);
1462     checkeq(ENGINE_KEY_ENOENT, val, "Key Carss has not expired.");
1463 
1464     // Store a dummy item since we do not purge the item with highest seqno
1465     checkeq(ENGINE_SUCCESS,
1466             store(h, h1, NULL, OPERATION_SET, "dummykey", "dummyvalue"),
1467             "Error setting dummy key");
1468     wait_for_flusher_to_settle(h, h1);
1469 
1470     checkeq(0, get_int_stat(h, h1, "vb_0:purge_seqno", "vbucket-seqno"),
1471             "purge_seqno not found to be zero before compaction");
1472 
1473     // Compaction on VBucket
1474     compact_db(
1475             h,
1476             h1,
1477             0 /* vbucket_id */,
1478             0 /* db_file_id */,
1479             2 /* purge_before_ts */,
1480             exp_purge_seqno - 1 /* purge_before_seq */,
1481             1 /* drop deletes (forces purge irrespective purge_before_seq) */);
1482     wait_for_stat_to_be(h, h1, "ep_pending_compactions", 0);
1483     checkeq(exp_purge_seqno,
1484             get_int_stat(h, h1, "vb_0:purge_seqno", "vbucket-seqno"),
1485             "purge_seqno didn't match expected value");
1486 
1487     return SUCCESS;
1488 }
1489 
test_MB_33919(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1)1490 static enum test_result test_MB_33919(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1) {
1491     const char* expKey = "expiryKey";
1492     const char* otherKey = "otherKey";
1493     const char* value = "value";
1494 
1495     // Test runs in the future as we need to set times in the past without
1496     // falling foul of the expiry time being behind server start
1497     auto fiveDays =
1498             std::chrono::system_clock::now() + std::chrono::hours(5 * 24);
1499 
1500     // Calculate expiry and purgeBefore times
1501     auto threeDaysAgo = std::chrono::system_clock::to_time_t(
1502             fiveDays - std::chrono::hours(3 * 24));
1503     auto fourDaysAgo = std::chrono::system_clock::to_time_t(
1504             fiveDays - std::chrono::hours(4 * 24));
1505     // Time travel forwards by 5 days to give the illusion of 5 days uptime.
1506     testHarness.time_travel((86400 * 5));
1507 
1508     // Set expiring key, expiry 4 days ago
1509     checkeq(ENGINE_SUCCESS,
1510             store(h,
1511                   h1,
1512                   nullptr,
1513                   OPERATION_SET,
1514                   expKey,
1515                   value,
1516                   nullptr,
1517                   0,
1518                   0,
1519                   fourDaysAgo),
1520             "Failed to set expiring key");
1521 
1522     // Force it to expire
1523     ENGINE_ERROR_CODE val = verify_key(h, h1, expKey);
1524     checkeq(ENGINE_KEY_ENOENT, val, "expiryKey has not expired.");
1525 
1526     // And a second key (after expiry), compaction won't purge the high-seqno
1527     checkeq(ENGINE_SUCCESS,
1528             store(h, h1, nullptr, OPERATION_SET, otherKey, value),
1529             "Failed to set non-expiring key");
1530 
1531     // Wait for the item to be expired
1532     wait_for_stat_to_be(h, h1, "vb_active_expired", 1);
1533 
1534     wait_for_flusher_to_settle(h, h1);
1535 
1536     checkeq(0,
1537             get_int_stat(h, h1, "vb_0:purge_seqno", "vbucket-seqno"),
1538             "purge_seqno not found to be zero before compaction");
1539 
1540     // Compaction on VBucket
1541     compact_db(
1542             h, h1, 0 /* vbucket_id */, 0 /* db_file_id */, threeDaysAgo, 0, 0);
1543     wait_for_stat_to_be(h, h1, "ep_pending_compactions", 0);
1544 
1545     // Purge-seqno should not of moved, without the fix it would
1546     checkeq(0,
1547             get_int_stat(h, h1, "vb_0:purge_seqno", "vbucket-seqno"),
1548             "purge_seqno not found to be zero before compaction");
1549 
1550     return SUCCESS;
1551 }
1552 
test_compaction_config(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1553 static enum test_result test_compaction_config(ENGINE_HANDLE *h,
1554                                                ENGINE_HANDLE_V1 *h1) {
1555 
1556     checkeq(10000,
1557             get_int_stat(h, h1, "ep_compaction_write_queue_cap"),
1558             "Expected compaction queue cap to be 10000");
1559     set_param(h, h1, protocol_binary_engine_param_flush,
1560               "compaction_write_queue_cap", "100000");
1561     checkeq(100000, get_int_stat(h, h1, "ep_compaction_write_queue_cap"),
1562             "Expected compaction queue cap to be 100000");
1563     return SUCCESS;
1564 }
1565 
1566 struct comp_thread_ctx {
1567     ENGINE_HANDLE *h;
1568     ENGINE_HANDLE_V1 *h1;
1569     uint16_t vbid;
1570     uint16_t db_file_id;
1571 };
1572 
1573 extern "C" {
compaction_thread(void *arg)1574     static void compaction_thread(void *arg) {
1575         struct comp_thread_ctx *ctx = static_cast<comp_thread_ctx *>(arg);
1576         compact_db(ctx->h, ctx->h1, ctx->vbid, ctx->db_file_id, 0, 0, 0);
1577     }
1578 }
1579 
test_multiple_vb_compactions(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1580 static enum test_result test_multiple_vb_compactions(ENGINE_HANDLE *h,
1581                                                      ENGINE_HANDLE_V1 *h1) {
1582     for (uint16_t i = 0; i < 4; ++i) {
1583         if (!set_vbucket_state(h, h1, i, vbucket_state_active)) {
1584             fprintf(stderr, "set state failed for vbucket %d.\n", i);
1585             return FAIL;
1586         }
1587         check(verify_vbucket_state(h, h1, i, vbucket_state_active),
1588               "VBucket state not active");
1589     }
1590 
1591     std::vector<std::string> keys;
1592     for (int j = 0; j < 20000; ++j) {
1593         std::stringstream ss;
1594         ss << "key" << j;
1595         std::string key(ss.str());
1596         keys.push_back(key);
1597     }
1598 
1599     int count = 0;
1600     std::vector<std::string>::iterator it;
1601     for (it = keys.begin(); it != keys.end(); ++it) {
1602         uint16_t vbid = count % 4;
1603         checkeq(ENGINE_SUCCESS,
1604                 store(h,
1605                       h1,
1606                       NULL,
1607                       OPERATION_SET,
1608                       it->c_str(),
1609                       it->c_str(),
1610                       nullptr,
1611                       0,
1612                       vbid),
1613                 "Failed to store a value");
1614         ++count;
1615     }
1616 
1617     // Compact multiple vbuckets.
1618     const int n_threads = 4;
1619     cb_thread_t threads[n_threads];
1620     struct comp_thread_ctx ctx[n_threads];
1621 
1622     const int num_shards = get_int_stat(h, h1, "ep_workload:num_shards",
1623                                         "workload");
1624 
1625     for (int i = 0; i < n_threads; i++) {
1626         ctx[i].h = h;
1627         ctx[i].h1 = h1;
1628         ctx[i].vbid = static_cast<uint16_t>(i);
1629         ctx[i].db_file_id = ctx[i].vbid % num_shards;
1630         int r = cb_create_thread(&threads[i], compaction_thread, &ctx[i], 0);
1631         cb_assert(r == 0);
1632     }
1633 
1634     for (int i = 0; i < n_threads; i++) {
1635         int r = cb_join_thread(threads[i]);
1636         cb_assert(r == 0);
1637     }
1638 
1639     wait_for_stat_to_be(h, h1, "ep_pending_compactions", 0);
1640 
1641     return SUCCESS;
1642 }
1643 
1644 static enum test_result
test_multi_vb_compactions_with_workload(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1645 test_multi_vb_compactions_with_workload(ENGINE_HANDLE *h,
1646                                         ENGINE_HANDLE_V1 *h1) {
1647     for (uint16_t i = 0; i < 4; ++i) {
1648         if (!set_vbucket_state(h, h1, i, vbucket_state_active)) {
1649             fprintf(stderr, "set state failed for vbucket %d.\n", i);
1650             return FAIL;
1651         }
1652         check(verify_vbucket_state(h, h1, i, vbucket_state_active),
1653               "VBucket state not active");
1654     }
1655 
1656     std::vector<std::string> keys;
1657     for (int j = 0; j < 10000; ++j) {
1658         std::stringstream ss;
1659         ss << "key" << j;
1660         std::string key(ss.str());
1661         keys.push_back(key);
1662     }
1663 
1664     int count = 0;
1665     std::vector<std::string>::iterator it;
1666     for (it = keys.begin(); it != keys.end(); ++it) {
1667         uint16_t vbid = count % 4;
1668         checkeq(ENGINE_SUCCESS,
1669                 store(h,
1670                       h1,
1671                       NULL,
1672                       OPERATION_SET,
1673                       it->c_str(),
1674                       it->c_str(),
1675                       nullptr,
1676                       0,
1677                       vbid),
1678                 "Failed to store a value");
1679         ++count;
1680     }
1681     wait_for_flusher_to_settle(h, h1);
1682 
1683     for (int i = 0; i < 2; ++i) {
1684         count = 0;
1685         for (it = keys.begin(); it != keys.end(); ++it) {
1686             uint16_t vbid = count % 4;
1687             checkeq(cb::engine_errc::success,
1688                     get(h, h1, NULL, it->c_str(), vbid).first,
1689                     "Unable to get stored item");
1690             ++count;
1691         }
1692     }
1693     wait_for_stat_to_be(h, h1, "ep_workload_pattern", std::string{"read_heavy"});
1694 
1695     // Compact multiple vbuckets.
1696     const int n_threads = 4;
1697     cb_thread_t threads[n_threads];
1698     struct comp_thread_ctx ctx[n_threads];
1699 
1700     for (int i = 0; i < n_threads; i++) {
1701         ctx[i].h = h;
1702         ctx[i].h1 = h1;
1703         ctx[i].vbid = static_cast<uint16_t>(i);
1704         int r = cb_create_thread(&threads[i], compaction_thread, &ctx[i], 0);
1705         cb_assert(r == 0);
1706     }
1707 
1708     for (int i = 0; i < n_threads; i++) {
1709         int r = cb_join_thread(threads[i]);
1710         cb_assert(r == 0);
1711     }
1712 
1713     wait_for_stat_to_be(h, h1, "ep_pending_compactions", 0);
1714 
1715     return SUCCESS;
1716 }
1717 
vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char* value = NULL)1718 static enum test_result vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1719                                              const char* value = NULL) {
1720     check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1721           "Failed to set vbucket state.");
1722 
1723     checkeq(ENGINE_NOT_MY_VBUCKET,
1724             vbucketDelete(h, h1, 2, value),
1725             "Expected NMVB");
1726 
1727     check(set_vbucket_state(h, h1, 1, vbucket_state_dead),
1728           "Failed set set vbucket 1 state.");
1729 
1730     checkeq(ENGINE_SUCCESS, vbucketDelete(h, h1, 1, value), "Expected success");
1731     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1732             "Expected failure deleting non-existent bucket.");
1733 
1734     check(verify_vbucket_missing(h, h1, 1),
1735           "vbucket 0 was not missing after deleting it.");
1736 
1737     return SUCCESS;
1738 }
1739 
test_vbucket_destroy_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1740 static enum test_result test_vbucket_destroy_stats(ENGINE_HANDLE *h,
1741                                                    ENGINE_HANDLE_V1 *h1) {
1742 
1743     int cacheSize = get_int_stat(h, h1, "ep_total_cache_size");
1744     int overhead = get_int_stat(h, h1, "ep_overhead");
1745     int nonResident = get_int_stat(h, h1, "ep_num_non_resident");
1746 
1747     check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1748           "Failed to set vbucket state.");
1749 
1750     std::vector<std::string> keys;
1751     for (int j = 0; j < 2000; ++j) {
1752         std::stringstream ss;
1753         ss << "key" << j;
1754         std::string key(ss.str());
1755         keys.push_back(key);
1756     }
1757 
1758     int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
1759     std::vector<std::string>::iterator it;
1760     for (it = keys.begin(); it != keys.end(); ++it) {
1761         checkeq(ENGINE_SUCCESS,
1762                 store(h,
1763                       h1,
1764                       NULL,
1765                       OPERATION_SET,
1766                       it->c_str(),
1767                       it->c_str(),
1768                       nullptr,
1769                       0,
1770                       1),
1771                 "Failed to store a value");
1772     }
1773     wait_for_flusher_to_settle(h, h1);
1774     testHarness.time_travel(65);
1775     wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
1776 
1777     check(set_vbucket_state(h, h1, 1, vbucket_state_dead),
1778           "Failed set set vbucket 1 state.");
1779 
1780     int vbucketDel = get_int_stat(h, h1, "ep_vbucket_del");
1781     checkeq(ENGINE_SUCCESS, vbucketDelete(h, h1, 1), "Expected success");
1782     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS,
1783             last_status.load(),
1784             "Expected failure deleting non-existent bucket.");
1785 
1786     check(verify_vbucket_missing(h, h1, 1),
1787           "vbucket 1 was not missing after deleting it.");
1788 
1789     wait_for_stat_change(h, h1, "ep_vbucket_del", vbucketDel);
1790 
1791     wait_for_stat_to_be(h, h1, "ep_total_cache_size", cacheSize);
1792     wait_for_stat_to_be(h, h1, "ep_overhead", overhead);
1793     wait_for_stat_to_be(h, h1, "ep_num_non_resident", nonResident);
1794 
1795     return SUCCESS;
1796 }
1797 
vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char* value = NULL)1798 static enum test_result vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1799                                                 const char* value = NULL) {
1800     if (!isWarmupEnabled(h, h1)) {
1801         return SKIPPED;
1802     }
1803 
1804     check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1805           "Failed to set vbucket state.");
1806 
1807     // Store a value so the restart will try to resurrect it.
1808     checkeq(ENGINE_SUCCESS,
1809             store(h,
1810                   h1,
1811                   NULL,
1812                   OPERATION_SET,
1813                   "key",
1814                   "somevalue",
1815                   nullptr,
1816                   0,
1817                   1),
1818             "Failed to set a value");
1819     check_key_value(h, h1, "key", "somevalue", 9, 1);
1820 
1821     // Reload to get a flush forced.
1822     testHarness.reload_engine(&h, &h1,
1823                               testHarness.engine_path,
1824                               testHarness.get_current_testcase()->cfg,
1825                               true, false);
1826     wait_for_warmup_complete(h, h1);
1827 
1828     check(verify_vbucket_state(h, h1, 1, vbucket_state_active),
1829           "Bucket state was what it was initially, after restart.");
1830     check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1831           "Failed to set vbucket state.");
1832     check_key_value(h, h1, "key", "somevalue", 9, 1);
1833 
1834     check(set_vbucket_state(h, h1, 1, vbucket_state_dead),
1835           "Failed set set vbucket 1 state.");
1836 
1837     checkeq(ENGINE_SUCCESS, vbucketDelete(h, h1, 1, value), "Expected success");
1838     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1839             "Expected failure deleting non-existent bucket.");
1840 
1841     check(verify_vbucket_missing(h, h1, 1),
1842           "vbucket 1 was not missing after deleting it.");
1843 
1844     testHarness.reload_engine(&h, &h1,
1845                               testHarness.engine_path,
1846                               testHarness.get_current_testcase()->cfg,
1847                               true, false);
1848     wait_for_warmup_complete(h, h1);
1849 
1850     if (verify_vbucket_state(h, h1, 1, vbucket_state_pending, true)) {
1851         std::cerr << "Bucket came up in pending state after delete." << std::endl;
1852         abort();
1853     }
1854 
1855     check(verify_vbucket_missing(h, h1, 1),
1856           "vbucket 1 was not missing after restart.");
1857 
1858     return SUCCESS;
1859 }
1860 
test_async_vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1861 static enum test_result test_async_vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1862     return vbucket_destroy(h, h1);
1863 }
1864 
test_sync_vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1865 static enum test_result test_sync_vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1866     return vbucket_destroy(h, h1, "async=0");
1867 }
1868 
test_async_vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1869 static enum test_result test_async_vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1870     return vbucket_destroy_restart(h, h1);
1871 }
1872 
test_sync_vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1873 static enum test_result test_sync_vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1874     return vbucket_destroy_restart(h, h1, "async=0");
1875 }
1876 
test_vb_set_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1877 static enum test_result test_vb_set_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1878     return test_pending_vb_mutation(h, h1, OPERATION_SET);
1879 }
1880 
test_vb_add_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1881 static enum test_result test_vb_add_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1882     return test_pending_vb_mutation(h, h1, OPERATION_ADD);
1883 }
1884 
test_vb_cas_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1885 static enum test_result test_vb_cas_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1886     return test_pending_vb_mutation(h, h1, OPERATION_CAS);
1887 }
1888 
test_vb_set_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1889 static enum test_result test_vb_set_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1890     return test_replica_vb_mutation(h, h1, OPERATION_SET);
1891 }
1892 
test_vb_replace_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1893 static enum test_result test_vb_replace_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1894     return test_replica_vb_mutation(h, h1, OPERATION_REPLACE);
1895 }
1896 
test_vb_replace_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1897 static enum test_result test_vb_replace_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1898     return test_pending_vb_mutation(h, h1, OPERATION_REPLACE);
1899 }
1900 
test_vb_add_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1901 static enum test_result test_vb_add_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1902     return test_replica_vb_mutation(h, h1, OPERATION_ADD);
1903 }
1904 
test_vb_cas_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1905 static enum test_result test_vb_cas_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1906     return test_replica_vb_mutation(h, h1, OPERATION_CAS);
1907 }
1908 
test_stats_seqno(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1909 static enum test_result test_stats_seqno(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1910     check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1911           "Failed to set vbucket state.");
1912 
1913     int num_keys = 100;
1914     for (int ii = 0; ii < num_keys; ++ii) {
1915         std::stringstream ss;
1916         ss << "key" << ii;
1917         checkeq(ENGINE_SUCCESS,
1918                 store(h, h1, NULL, OPERATION_SET, ss.str().c_str(),
1919                       "value", NULL, 0, 0),
1920                 "Failed to store an item.");
1921     }
1922     wait_for_flusher_to_settle(h, h1);
1923 
1924     checkeq(100, get_int_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno"),
1925             "Invalid seqno");
1926 
1927     if (isPersistentBucket(h, h1)) {
1928         checkeq(100,
1929                 get_int_stat(h, h1, "vb_0:last_persisted_seqno", "vbucket-seqno"),
1930                 "Unexpected last_persisted_seqno");
1931     }
1932     checkeq(0, get_int_stat(h, h1, "vb_1:high_seqno", "vbucket-seqno"),
1933             "Invalid seqno");
1934     checkeq(0, get_int_stat(h, h1, "vb_1:high_seqno", "vbucket-seqno 1"),
1935             "Invalid seqno");
1936     if (isPersistentBucket(h, h1)) {
1937         checkeq(0,
1938                 get_int_stat(h, h1, "vb_1:last_persisted_seqno", "vbucket-seqno 1"),
1939                 "Invalid last_persisted_seqno");
1940     }
1941 
1942     uint64_t vb_uuid = get_ull_stat(h, h1, "vb_1:0:id", "failovers");
1943 
1944     auto seqno_stats = get_all_stats(h, h1, "vbucket-seqno 1");
1945     checkeq(vb_uuid, uint64_t(std::stoull(seqno_stats.at("vb_1:uuid"))),
1946             "Invalid uuid");
1947 
1948     checkeq(size_t(7), seqno_stats.size(), "Expected seven stats");
1949 
1950     // Check invalid vbucket
1951     checkeq(ENGINE_NOT_MY_VBUCKET,
1952             get_stats(h, "vbucket-seqno 2"_ccb, add_stats),
1953             "Expected not my vbucket");
1954 
1955     // Check bad vbucket parameter (not numeric)
1956     checkeq(ENGINE_EINVAL,
1957             get_stats(h, "vbucket-seqno tt2"_ccb, add_stats),
1958             "Expected invalid");
1959 
1960     // Check extra spaces at the end
1961     checkeq(ENGINE_EINVAL,
1962             get_stats(h, "vbucket-seqno    "_ccb, add_stats),
1963             "Expected invalid");
1964 
1965     return SUCCESS;
1966 }
1967 
test_stats_diskinfo(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1968 static enum test_result test_stats_diskinfo(ENGINE_HANDLE *h,
1969                                             ENGINE_HANDLE_V1 *h1) {
1970     check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1971           "Failed to set vbucket state.");
1972 
1973     int num_keys = 100;
1974     for (int ii = 0; ii < num_keys; ++ii) {
1975         std::stringstream ss;
1976         ss << "key" << ii;
1977         checkeq(ENGINE_SUCCESS,
1978                 store(h, h1, NULL, OPERATION_SET, ss.str().c_str(),
1979                       "value", NULL, 0, 1),
1980                 "Failed to store an item.");
1981     }
1982     wait_for_flusher_to_settle(h, h1);
1983 
1984     size_t file_size = get_int_stat(h, h1, "ep_db_file_size", "diskinfo");
1985     size_t data_size = get_int_stat(h, h1, "ep_db_data_size", "diskinfo");
1986     check(file_size > 0, "DB file size should be greater than 0");
1987     check(data_size > 0, "DB data size should be greater than 0");
1988     check(file_size >= data_size, "DB file size should be >= DB data size");
1989     check(get_int_stat(h, h1, "vb_1:data_size", "diskinfo detail") > 0,
1990           "VB 1 data size should be greater than 0");
1991 
1992     checkeq(ENGINE_EINVAL,
1993             get_stats(h, "diskinfo "_ccb, add_stats),
1994             "Expected invalid");
1995 
1996     checkeq(ENGINE_EINVAL,
1997             get_stats(h, "diskinfo detai"_ccb, add_stats),
1998             "Expected invalid");
1999 
2000     checkeq(ENGINE_EINVAL,
2001             get_stats(h, "diskinfo detaillll"_ccb, add_stats),
2002             "Expected invalid");
2003 
2004     return SUCCESS;
2005 }
2006 
test_uuid_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2007 static enum test_result test_uuid_stats(ENGINE_HANDLE *h,
2008                                         ENGINE_HANDLE_V1 *h1)
2009 {
2010     vals.clear();
2011     checkeq(ENGINE_SUCCESS,
2012             get_stats(h, "uuid"_ccb, add_stats),
2013             "Failed to get stats.");
2014     check(vals["uuid"] == "foobar", "Incorrect uuid");
2015     return SUCCESS;
2016 }
2017 
test_item_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2018 static enum test_result test_item_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2019     checkeq(ENGINE_SUCCESS,
2020             store(h, h1, NULL, OPERATION_SET, "key", "somevalue"),
2021             "Failed set.");
2022     wait_for_flusher_to_settle(h, h1);
2023     checkeq(ENGINE_SUCCESS,
2024             store(h, h1, NULL, OPERATION_SET, "key", "somevalueX"),
2025             "Failed set.");
2026     wait_for_flusher_to_settle(h, h1);
2027     checkeq(ENGINE_SUCCESS,
2028             store(h, h1, NULL, OPERATION_SET, "key1", "somevalueY"),
2029             "Failed set.");
2030     wait_for_flusher_to_settle(h, h1);
2031 
2032     check_key_value(h, h1, "key", "somevalueX", 10);
2033     check_key_value(h, h1, "key1", "somevalueY", 10);
2034 
2035     checkeq(ENGINE_SUCCESS, del(h, h1, "key1", 0, 0),
2036             "Failed remove with value.");
2037     wait_for_flusher_to_settle(h, h1);
2038 
2039     checkeq(ENGINE_SUCCESS,
2040             store(h, h1, NULL, OPERATION_SET, "key1", "someothervalue"),
2041             "Failed set.");
2042     wait_for_flusher_to_settle(h, h1);
2043 
2044     check_key_value(h, h1, "key1", "someothervalue", 14);
2045 
2046     checkeq(3,
2047             get_int_stat(h, h1, "vb_active_ops_create"),
2048             "Expected 3 creations");
2049     checkeq(1,
2050             get_int_stat(h, h1, "vb_active_ops_update"),
2051             "Expected 1 updation");
2052     checkeq(1,
2053             get_int_stat(h, h1, "vb_active_ops_delete"),
2054             "Expected 1 deletion");
2055 
2056     return SUCCESS;
2057 }
2058 
test_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2059 static enum test_result test_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2060     vals.clear();
2061     checkeq(ENGINE_SUCCESS,
2062             get_stats(h, {}, add_stats),
2063             "Failed to get stats.");
2064     check(vals.size() > 10, "Kind of expected more stats than that.");
2065 
2066     return SUCCESS;
2067 }
2068 
test_mem_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2069 static enum test_result test_mem_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2070     char value[2048];
2071     memset(value, 'b', sizeof(value));
2072     strcpy(value + sizeof(value) - 4, "\r\n");
2073     int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
2074     wait_for_persisted_value(h, h1, "key", value);
2075     testHarness.time_travel(65);
2076     if (isPersistentBucket(h, h1)) {
2077         wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
2078     }
2079 
2080     if (isActiveCompressionEnabled(h, h1)) {
2081         wait_for_item_compressor_to_settle(h, h1);
2082     }
2083 
2084     int mem_used = get_int_stat(h, h1, "mem_used");
2085     int cache_size = get_int_stat(h, h1, "ep_total_cache_size");
2086     int overhead = get_int_stat(h, h1, "ep_overhead");
2087     int value_size = get_int_stat(h, h1, "ep_value_size");
2088     check((mem_used - overhead) > cache_size,
2089           "ep_kv_size should be greater than the hashtable cache size due to "
2090           "the checkpoint overhead");
2091 
2092     if (isPersistentBucket(h, h1)) {
2093         evict_key(h, h1, "key", 0, "Ejected.");
2094 
2095         check(get_int_stat(h, h1, "ep_total_cache_size") <= cache_size,
2096               "Evict a value shouldn't increase the total cache size");
2097         check(get_int_stat(h, h1, "mem_used") < mem_used,
2098               "Expected mem_used to decrease when an item is evicted");
2099 
2100         check_key_value(h, h1, "key", value, strlen(value), 0); // Load an item from disk again.
2101 
2102         if (isActiveCompressionEnabled(h, h1)) {
2103             wait_for_item_compressor_to_settle(h, h1);
2104         }
2105 
2106         check(get_int_stat(h, h1, "mem_used") >= mem_used,
2107               "Expected mem_used to remain the same after an item is loaded from disk");
2108         check(get_int_stat(h, h1, "ep_value_size") == value_size,
2109               "Expected ep_value_size to remain the same after item is "
2110               "loaded from disk");
2111     }
2112 
2113     return SUCCESS;
2114 }
2115 
test_io_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2116 static enum test_result test_io_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2117     int exp_write_bytes;
2118     std::string backend = get_str_stat(h, h1, "ep_backend");
2119     if (backend == "couchdb") {
2120         exp_write_bytes = 22; /* TBD: Do not hard code the value */
2121     } else if (backend == "rocksdb") {
2122         // TODO RDB:
2123         return SKIPPED_UNDER_ROCKSDB;
2124     }
2125     else {
2126         return SKIPPED;
2127     }
2128 
2129     reset_stats(h);
2130 
2131     checkeq(0,
2132             get_int_stat(h, h1, "rw_0:io_bg_fetch_docs_read", "kvstore"),
2133             "Expected reset stats to set io_bg_fetch_docs_read to zero");
2134     checkeq(0, get_int_stat(h, h1, "rw_0:io_num_write", "kvstore"),
2135             "Expected reset stats to set io_num_write to zero");
2136     checkeq(0,
2137             get_int_stat(h, h1, "rw_0:io_bg_fetch_doc_bytes", "kvstore"),
2138             "Expected reset stats to set io_bg_fetch_doc_bytes to zero");
2139     checkeq(0, get_int_stat(h, h1, "rw_0:io_write_bytes", "kvstore"),
2140             "Expected reset stats to set io_write_bytes to zero");
2141 
2142     const std::string key("a");
2143     const std::string value("b\r\n");
2144     wait_for_persisted_value(h, h1, key.c_str(), value.c_str());
2145     checkeq(0,
2146             get_int_stat(h, h1, "rw_0:io_bg_fetch_docs_read", "kvstore"),
2147             "Expected storing one value to not change the read counter");
2148     checkeq(0,
2149             get_int_stat(h, h1, "rw_0:io_bg_fetch_doc_bytes", "kvstore"),
2150             "Expected storing one value to not change the bgfetch doc bytes");
2151     checkeq(1, get_int_stat(h, h1, "rw_0:io_num_write", "kvstore"),
2152             "Expected storing the key to update the write counter");
2153     checkeq(exp_write_bytes,
2154             get_int_stat(h, h1, "rw_0:io_write_bytes", "kvstore"),
2155             "Expected storing the key to update the write bytes");
2156 
2157     evict_key(h, h1, key.c_str(), 0, "Ejected.");
2158 
2159     check_key_value(h, h1, "a", value.c_str(), value.size(), 0);
2160 
2161     std::stringstream numReadStatStr;
2162     std::stringstream readBytesStatStr;
2163 
2164     if (backend == "couchdb") {
2165         numReadStatStr << "ro_" << 0 << ":io_bg_fetch_docs_read";
2166         readBytesStatStr << "ro_" << 0 << ":io_bg_fetch_doc_bytes";
2167     } else {
2168         cb_assert(false);
2169     }
2170 
2171     checkeq(1,
2172             get_int_stat(h, h1, numReadStatStr.str().c_str(), "kvstore"),
2173             "Expected reading the value back in to update the read counter");
2174 
2175     const uint64_t exp_read_bytes =
2176             key.size() + value.size() +
2177             MetaData::getMetaDataSize(MetaData::Version::V1);
2178     checkeq(exp_read_bytes,
2179             get_stat<uint64_t>(
2180                     h, h1, readBytesStatStr.str().c_str(), "kvstore"),
2181             "Expected reading the value back in to update the read bytes");
2182 
2183     // For read amplification, exact value depends on couchstore file layout,
2184     // but generally see a value of 2 here.
2185     checkge(get_float_stat(h, h1, "ep_bg_fetch_avg_read_amplification"),
2186             2.0f,
2187             "Expected sensible bgFetch read amplification value");
2188 
2189     checkeq(1, get_int_stat(h, h1, "rw_0:io_num_write", "kvstore"),
2190             "Expected reading the value back in to not update the write counter");
2191     checkeq(exp_write_bytes,
2192             get_int_stat(h, h1, "rw_0:io_write_bytes", "kvstore"),
2193             "Expected reading the value back in to not update the write bytes");
2194 
2195     return SUCCESS;
2196 }
2197 
test_vb_file_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2198 static enum test_result test_vb_file_stats(ENGINE_HANDLE *h,
2199                                         ENGINE_HANDLE_V1 *h1) {
2200     wait_for_flusher_to_settle(h, h1);
2201     wait_for_stat_change(h, h1, "ep_db_data_size", 0);
2202 
2203     int old_data_size = get_int_stat(h, h1, "ep_db_data_size");
2204     int old_file_size = get_int_stat(h, h1, "ep_db_file_size");
2205     check(old_file_size != 0, "Expected a non-zero value for ep_db_file_size");
2206 
2207     // Write a value and test ...
2208     wait_for_persisted_value(h, h1, "a", "b\r\n");
2209     check(get_int_stat(h, h1, "ep_db_data_size") > old_data_size,
2210           "Expected the DB data size to increase");
2211     check(get_int_stat(h, h1, "ep_db_file_size") > old_file_size,
2212           "Expected the DB file size to increase");
2213 
2214     check(get_int_stat(h, h1, "vb_0:db_data_size", "vbucket-details 0") > 0,
2215           "Expected the vbucket DB data size to non-zero");
2216     check(get_int_stat(h, h1, "vb_0:db_file_size", "vbucket-details 0") > 0,
2217           "Expected the vbucket DB file size to non-zero");
2218     return SUCCESS;
2219 }
2220 
test_vb_file_stats_after_warmup(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2221 static enum test_result test_vb_file_stats_after_warmup(ENGINE_HANDLE *h,
2222                                                         ENGINE_HANDLE_V1 *h1) {
2223     if (!isWarmupEnabled(h, h1)) {
2224         return SKIPPED;
2225     }
2226 
2227     for (int i = 0; i < 100; ++i) {
2228         std::stringstream key;
2229         key << "key-" << i;
2230         checkeq(ENGINE_SUCCESS,
2231                 store(h,
2232                       h1,
2233                       NULL,
2234                       OPERATION_SET,
2235                       key.str().c_str(),
2236                       "somevalue"),
2237                 "Error setting.");
2238     }
2239     wait_for_flusher_to_settle(h, h1);
2240 
2241     int fileSize = get_int_stat(h, h1, "vb_0:db_file_size", "vbucket-details 0");
2242     int spaceUsed = get_int_stat(h, h1, "vb_0:db_data_size", "vbucket-details 0");
2243 
2244     // Restart the engine.
2245     testHarness.reload_engine(&h, &h1,
2246                               testHarness.engine_path,
2247                               testHarness.get_current_testcase()->cfg,
2248                               true, false);
2249     wait_for_warmup_complete(h, h1);
2250 
2251     int newFileSize = get_int_stat(h, h1, "vb_0:db_file_size", "vbucket-details 0");
2252     int newSpaceUsed = get_int_stat(h, h1, "vb_0:db_data_size", "vbucket-details 0");
2253 
2254     check((float)newFileSize >= 0.9 * fileSize, "Unexpected fileSize for vbucket");
2255     check((float)newSpaceUsed >= 0.9 * spaceUsed, "Unexpected spaceUsed for vbucket");
2256 
2257     return SUCCESS;
2258 }
2259 
test_bg_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2260 static enum test_result test_bg_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2261     reset_stats(h);
2262     wait_for_persisted_value(h, h1, "a", "b\r\n");
2263     evict_key(h, h1, "a", 0, "Ejected.");
2264     testHarness.time_travel(43);
2265     check_key_value(h, h1, "a", "b\r\n", 3, 0);
2266 
2267     auto stats = get_all_stats(h, h1);
2268     checkeq(1, std::stoi(stats.at("ep_bg_num_samples")),
2269                "Expected one sample");
2270 
2271     const char* bg_keys[] = { "ep_bg_min_wait",
2272                               "ep_bg_max_wait",
2273                               "ep_bg_wait_avg",
2274                               "ep_bg_min_load",
2275                               "ep_bg_max_load",
2276                               "ep_bg_load_avg"};
2277     for (const auto* key : bg_keys) {
2278         check(stats.find(key) != stats.end(),
2279               (std::string("Found no ") + key).c_str());
2280     }
2281 
2282     evict_key(h, h1, "a", 0, "Ejected.");
2283     check_key_value(h, h1, "a", "b\r\n", 3, 0);
2284     check(get_int_stat(h, h1, "ep_bg_num_samples") == 2,
2285           "Expected one sample");
2286 
2287     reset_stats(h);
2288     checkeq(0, get_int_stat(h, h1, "ep_bg_fetched"),
2289             "ep_bg_fetched is not reset to 0");
2290     return SUCCESS;
2291 }
2292 
test_bg_meta_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2293 static enum test_result test_bg_meta_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2294     reset_stats(h);
2295 
2296     wait_for_persisted_value(h, h1, "k1", "v1");
2297     wait_for_persisted_value(h, h1, "k2", "v2");
2298 
2299     evict_key(h, h1, "k1", 0, "Ejected.");
2300     checkeq(ENGINE_SUCCESS,
2301             del(h, h1, "k2", 0, 0), "Failed remove with value.");
2302     wait_for_flusher_to_settle(h, h1);
2303 
2304     checkeq(0, get_int_stat(h, h1, "ep_bg_fetched"), "Expected bg_fetched to be 0");
2305     checkeq(0, get_int_stat(h, h1, "ep_bg_meta_fetched"), "Expected bg_meta_fetched to be 0");
2306 
2307     check(get_meta(h, h1, "k2"), "Get meta failed");
2308     checkeq(0, get_int_stat(h, h1, "ep_bg_fetched"), "Expected bg_fetched to be 0");
2309     checkeq(1, get_int_stat(h, h1, "ep_bg_meta_fetched"), "Expected bg_meta_fetched to be 1");
2310 
2311     checkeq(cb::engine_errc::success,
2312             get(h, h1, NULL, "k1", 0).first,
2313             "Missing key");
2314     checkeq(1, get_int_stat(h, h1, "ep_bg_fetched"), "Expected bg_fetched to be 1");
2315     checkeq(1, get_int_stat(h, h1, "ep_bg_meta_fetched"), "Expected bg_meta_fetched to be 1");
2316 
2317     // store new key with some random metadata
2318     const size_t keylen = strlen("k3");
2319     ItemMetaData itemMeta;
2320     itemMeta.revSeqno = 10;
2321     itemMeta.cas = 0xdeadbeef;
2322     itemMeta.exptime = 0;
2323     itemMeta.flags = 0xdeadbeef;
2324 
2325     add_with_meta(h, h1, "k3", keylen, NULL, 0, 0, &itemMeta);
2326     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Set meta failed");
2327 
2328     check(get_meta(h, h1, "k2"), "Get meta failed");
2329     checkeq(1, get_int_stat(h, h1, "ep_bg_fetched"), "Expected bg_fetched to be 1");
2330     checkeq(1, get_int_stat(h, h1, "ep_bg_meta_fetched"),
2331             "Expected bg_meta_fetched to remain at 1");
2332 
2333     return SUCCESS;
2334 }
2335 
test_key_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2336 static enum test_result test_key_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2337     check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed set vbucket 1 state.");
2338 
2339     // set (k1,v1) in vbucket 0
2340     checkeq(ENGINE_SUCCESS,
2341             store(h, h1, NULL, OPERATION_SET, "k1", "v1"),
2342             "Failed to store an item.");
2343 
2344     // set (k2,v2) in vbucket 1
2345     checkeq(ENGINE_SUCCESS,
2346             store(h, h1, NULL, OPERATION_SET, "k2", "v2", nullptr, 0, 1),
2347             "Failed to store an item.");
2348 
2349     const void *cookie = testHarness.create_cookie();
2350 
2351     // stat for key "k1" and vbucket "0"
2352     const char *statkey1 = "key k1 0";
2353     checkeq(ENGINE_SUCCESS,
2354             h1->get_stats(h, cookie, {statkey1, strlen(statkey1)}, add_stats),
2355             "Failed to get stats.");
2356     check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2357     check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2358     check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2359     check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2360     check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2361 
2362     // stat for key "k2" and vbucket "1"
2363     const char *statkey2 = "key k2 1";
2364     checkeq(ENGINE_SUCCESS,
2365             h1->get_stats(h, cookie, {statkey2, strlen(statkey2)}, add_stats),
2366             "Failed to get stats.");
2367     check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2368     check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2369     check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2370     check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2371     check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2372 
2373     testHarness.destroy_cookie(cookie);
2374     return SUCCESS;
2375 }
2376 
test_vkey_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2377 static enum test_result test_vkey_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2378     check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed set vbucket 1 state.");
2379     check(set_vbucket_state(h, h1, 2, vbucket_state_active), "Failed set vbucket 2 state.");
2380     check(set_vbucket_state(h, h1, 3, vbucket_state_active), "Failed set vbucket 3 state.");
2381     check(set_vbucket_state(h, h1, 4, vbucket_state_active), "Failed set vbucket 4 state.");
2382 
2383     wait_for_persisted_value(h, h1, "k1", "v1");
2384     wait_for_persisted_value(h, h1, "k2", "v2", 1);
2385     wait_for_persisted_value(h, h1, "k3", "v3", 2);
2386     wait_for_persisted_value(h, h1, "k4", "v4", 3);
2387     wait_for_persisted_value(h, h1, "k5", "v5", 4);
2388 
2389     check(set_vbucket_state(h, h1, 2, vbucket_state_replica), "Failed to set VB2 state.");
2390     check(set_vbucket_state(h, h1, 3, vbucket_state_pending), "Failed to set VB3 state.");
2391     check(set_vbucket_state(h, h1, 4, vbucket_state_dead), "Failed to set VB4 state.");
2392 
2393     const void *cookie = testHarness.create_cookie();
2394 
2395     // stat for key "k1" and vbucket "0"
2396     const char *statkey1 = "vkey k1 0";
2397     checkeq(ENGINE_SUCCESS,
2398             h1->get_stats(h, cookie, {statkey1, strlen(statkey1)}, add_stats),
2399             "Failed to get stats.");
2400     check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2401     check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2402     check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2403     check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2404     check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2405     check(vals.find("key_valid") != vals.end(), "Found no key_valid");
2406 
2407     // stat for key "k2" and vbucket "1"
2408     const char *statkey2 = "vkey k2 1";
2409     checkeq(ENGINE_SUCCESS,
2410             h1->get_stats(h, cookie, {statkey2, strlen(statkey2)}, add_stats),
2411             "Failed to get stats.");
2412     check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2413     check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2414     check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2415     check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2416     check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2417     check(vals.find("key_valid") != vals.end(), "Found no key_valid");
2418 
2419     // stat for key "k3" and vbucket "2"
2420     const char *statkey3 = "vkey k3 2";
2421     checkeq(ENGINE_SUCCESS,
2422             h1->get_stats(h, cookie, {statkey3, strlen(statkey3)}, add_stats),
2423             "Failed to get stats.");
2424     check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2425     check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2426     check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2427     check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2428     check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2429     check(vals.find("key_valid") != vals.end(), "Found no key_valid");
2430 
2431     // stat for key "k4" and vbucket "3"
2432     const char *statkey4 = "vkey k4 3";
2433     checkeq(ENGINE_SUCCESS,
2434             h1->get_stats(h, cookie, {statkey4, strlen(statkey4)}, add_stats),
2435             "Failed to get stats.");
2436     check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2437     check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2438     check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2439     check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2440     check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2441     check(vals.find("key_valid") != vals.end(), "Found no key_valid");
2442 
2443     // stat for key "k5" and vbucket "4"
2444     const char *statkey5 = "vkey k5 4";
2445     checkeq(ENGINE_SUCCESS,
2446             h1->get_stats(h, cookie, {statkey5, strlen(statkey5)}, add_stats),
2447             "Failed to get stats.");
2448     check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2449     check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2450     check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2451     check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2452     check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2453     check(vals.find("key_valid") != vals.end(), "Found no key_valid");
2454 
2455     testHarness.destroy_cookie(cookie);
2456     return SUCCESS;
2457 }
2458 
test_warmup_conf(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2459 static enum test_result test_warmup_conf(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2460     if (!isWarmupEnabled(h, h1)) {
2461         return SKIPPED;
2462     }
2463 
2464     checkeq(100, get_int_stat(h, h1, "ep_warmup_min_items_threshold"),
2465             "Incorrect initial warmup min items threshold.");
2466     checkeq(100, get_int_stat(h, h1, "ep_warmup_min_memory_threshold"),
2467             "Incorrect initial warmup min memory threshold.");
2468 
2469     check(!set_param(h, h1, protocol_binary_engine_param_flush,
2470                      "warmup_min_items_threshold", "a"),
2471           "Set warmup_min_items_threshold should have failed");
2472     check(!set_param(h, h1, protocol_binary_engine_param_flush,
2473                      "warmup_min_items_threshold", "a"),
2474           "Set warmup_min_memory_threshold should have failed");
2475 
2476     check(set_param(h, h1, protocol_binary_engine_param_flush,
2477                     "warmup_min_items_threshold", "80"),
2478           "Set warmup_min_items_threshold should have worked");
2479     check(set_param(h, h1, protocol_binary_engine_param_flush,
2480                     "warmup_min_memory_threshold", "80"),
2481           "Set warmup_min_memory_threshold should have worked");
2482 
2483     checkeq(80, get_int_stat(h, h1, "ep_warmup_min_items_threshold"),
2484             "Incorrect smaller warmup min items threshold.");
2485     checkeq(80, get_int_stat(h, h1, "ep_warmup_min_memory_threshold"),
2486             "Incorrect smaller warmup min memory threshold.");
2487 
2488     for (int i = 0; i < 100; ++i) {
2489         std::stringstream key;
2490         key << "key-" << i;
2491         checkeq(ENGINE_SUCCESS,
2492                 store(h,
2493                       h1,
2494                       NULL,
2495                       OPERATION_SET,
2496                       key.str().c_str(),
2497                       "somevalue"),
2498                 "Error setting.");
2499     }
2500 
2501     // Restart the server.
2502     std::string config(testHarness.get_current_testcase()->cfg);
2503     config = config + "warmup_min_memory_threshold=0";
2504     testHarness.reload_engine(&h, &h1,
2505                               testHarness.engine_path,
2506                               config.c_str(),
2507                               true, false);
2508     wait_for_warmup_complete(h, h1);
2509 
2510     const std::string eviction_policy = get_str_stat(h, h1, "ep_item_eviction_policy");
2511     if (eviction_policy == "value_only") {
2512         checkeq(100, get_int_stat(h, h1, "ep_warmup_key_count", "warmup"),
2513                 "Expected 100 keys loaded after warmup");
2514     } else { // Full eviction mode
2515         checkeq(0, get_int_stat(h, h1, "ep_warmup_key_count", "warmup"),
2516                 "Expected 0 keys loaded after warmup");
2517     }
2518 
2519     checkeq(0, get_int_stat(h, h1, "ep_warmup_value_count", "warmup"),
2520             "Expected 0 values loaded after warmup");
2521 
2522     return SUCCESS;
2523 }
2524 
2525 // Test that all the configuration parameters associated with the ItemPager,
2526 // can be set.
test_itempager_conf(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1)2527 static enum test_result test_itempager_conf(ENGINE_HANDLE* h,
2528                                             ENGINE_HANDLE_V1* h1) {
2529     check(set_param(h,
2530                     h1,
2531                     protocol_binary_engine_param_flush,
2532                     "pager_active_vb_pcnt",
2533                     "50"),
2534           "Setting pager_active_vb_pcnt should have worked");
2535     checkeq(50,
2536             get_int_stat(h, h1, "ep_pager_active_vb_pcnt"),
2537             "pager_active_vb_pcnt did not get set to the correct value");
2538 
2539     check(set_param(h,
2540                     h1,
2541                     protocol_binary_engine_param_flush,
2542                     "pager_sleep_time_ms",
2543                     "1000"),
2544           "Setting pager_sleep_time_ms should have worked");
2545     checkeq(1000,
2546             get_int_stat(h, h1, "ep_pager_sleep_time_ms"),
2547             "pager_sleep_time_ms did not get set to the correct value");
2548 
2549     check(set_param(h,
2550                     h1,
2551                     protocol_binary_engine_param_flush,
2552                     "ht_eviction_policy",
2553                     "2-bit_lru"),
2554           "Setting ht_eviction_policy should have worked");
2555     checkeq(std::string("2-bit_lru"),
2556             get_str_stat(h, h1, "ep_ht_eviction_policy"),
2557             "ht_eviction_policy did not get set to the correct value");
2558 
2559     check(set_param(h,
2560                     h1,
2561                     protocol_binary_engine_param_flush,
2562                     "item_eviction_age_percentage",
2563                     "100"),
2564           "Set item_eviction_age_percentage should have worked");
2565     checkeq(100,
2566             get_int_stat(h, h1, "ep_item_eviction_age_percentage"),
2567             "item_eviction_age_percentage did not get set to the correct "
2568             "value");
2569 
2570     check(set_param(h,
2571                     h1,
2572                     protocol_binary_engine_param_flush,
2573                     "item_eviction_freq_counter_age_threshold",
2574                     "10"),
2575           "Set item_eviction_freq_counter_age_threshold should have worked");
2576     checkeq(10,
2577             get_int_stat(h, h1, "ep_item_eviction_freq_counter_age_threshold"),
2578             "item_eviction_freq_counter_age_threshold did not get set to the "
2579             "correct value");
2580 
2581     check(set_param(h,
2582                     h1,
2583                     protocol_binary_engine_param_flush,
2584                     "item_freq_decayer_chunk_duration",
2585                     "1000"),
2586           "Set item_freq_decayer_chunk_duration should have worked");
2587     checkeq(1000,
2588             get_int_stat(h, h1, "ep_item_freq_decayer_chunk_duration"),
2589             "item_freq_decayer_chunk_duration did not get set to the correct "
2590             "value");
2591 
2592     check(set_param(h,
2593                     h1,
2594                     protocol_binary_engine_param_flush,
2595                     "item_freq_decayer_percent",
2596                     "100"),
2597           "Set item_freq_decayer_percent should have worked");
2598     checkeq(100,
2599             get_int_stat(h, h1, "ep_item_freq_decayer_percent"),
2600             "item_freq_decayer_percent did not get set to the correct value");
2601 
2602     return SUCCESS;
2603 }
2604 
test_bloomfilter_conf(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2605 static enum test_result test_bloomfilter_conf(ENGINE_HANDLE *h,
2606                                               ENGINE_HANDLE_V1 *h1) {
2607 
2608     if (get_bool_stat(h, h1, "ep_bfilter_enabled") == false) {
2609         check(set_param(h, h1, protocol_binary_engine_param_flush,
2610                         "bfilter_enabled", "true"),
2611               "Set bloomfilter_enabled should have worked");
2612     }
2613     check(get_bool_stat(h, h1, "ep_bfilter_enabled"),
2614           "Bloom filter wasn't enabled");
2615 
2616     check(get_float_stat(h, h1, "ep_bfilter_residency_threshold") == (float)0.1,
2617           "Incorrect initial bfilter_residency_threshold.");
2618 
2619     check(set_param(h, h1, protocol_binary_engine_param_flush,
2620           "bfilter_enabled", "false"),
2621           "Set bloomfilter_enabled should have worked.");
2622     check(set_param(h, h1, protocol_binary_engine_param_flush,
2623           "bfilter_residency_threshold", "0.15"),
2624           "Set bfilter_residency_threshold should have worked.");
2625 
2626     check(get_bool_stat(h, h1, "ep_bfilter_enabled") == false,
2627           "Bloom filter should have been disabled.");
2628     check(get_float_stat(h, h1, "ep_bfilter_residency_threshold") == (float)0.15,
2629           "Incorrect bfilter_residency_threshold.");
2630 
2631     return SUCCESS;
2632 }
2633 
test_bloomfilters(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2634 static enum test_result test_bloomfilters(ENGINE_HANDLE *h,
2635                                           ENGINE_HANDLE_V1 *h1) {
2636 
2637     if (get_bool_stat(h, h1, "ep_bfilter_enabled") == false) {
2638         check(set_param(h, h1, protocol_binary_engine_param_flush,
2639                     "bfilter_enabled", "true"),
2640                 "Set bloomfilter_enabled should have worked");
2641     }
2642     check(get_bool_stat(h, h1, "ep_bfilter_enabled"),
2643             "Bloom filter wasn't enabled");
2644 
2645     // Key is only present if bgOperations is non-zero.
2646     int num_read_attempts = get_int_stat_or_default(h, h1, 0,
2647                                                     "ep_bg_num_samples");
2648 
2649     // Ensure vbucket's bloom filter is enabled
2650     checkeq(std::string("ENABLED"),
2651             get_str_stat(h, h1, "vb_0:bloom_filter", "vbucket-details 0"),
2652             "Vbucket 0's bloom filter wasn't enabled upon setup!");
2653 
2654     int i;
2655 
2656     // Insert 10 items.
2657     for (i = 0; i < 10; ++i) {
2658         std::stringstream key;
2659         key << "key-" << i;
2660         checkeq(ENGINE_SUCCESS,
2661                 store(h,
2662                       h1,
2663                       NULL,
2664                       OPERATION_SET,
2665                       key.str().c_str(),
2666                       "somevalue"),
2667                 "Error setting.");
2668     }
2669     wait_for_flusher_to_settle(h, h1);
2670 
2671     // Evict all 10 items.
2672     for (i = 0; i < 10; ++i) {
2673         std::stringstream key;
2674         key << "key-" << i;
2675         evict_key(h, h1, key.str().c_str(), 0, "Ejected.");
2676     }
2677     wait_for_flusher_to_settle(h, h1);
2678 
2679     // Ensure 10 items are non-resident.
2680     cb_assert(10 == get_int_stat(h, h1, "ep_num_non_resident"));
2681 
2682     // Issue delete on first 5 items.
2683     for (i = 0; i < 5; ++i) {
2684         std::stringstream key;
2685         key << "key-" << i;
2686         checkeq(ENGINE_SUCCESS,
2687                 del(h, h1, key.str().c_str(), 0, 0),
2688                 "Failed remove with value.");
2689     }
2690     wait_for_flusher_to_settle(h, h1);
2691 
2692     // Ensure that there are 5 non-resident items
2693     cb_assert(5 == get_int_stat(h, h1, "ep_num_non_resident"));
2694     cb_assert(5 == get_int_stat(h, h1, "curr_items"));
2695 
2696     checkeq(ENGINE_SUCCESS,
2697             get_stats(h, {}, add_stats),
2698             "Failed to get stats.");
2699     std::string eviction_policy = vals.find("ep_item_eviction_policy")->second;
2700 
2701     useconds_t sleepTime = 128;
2702 
2703     if (eviction_policy == "value_only") {  // VALUE-ONLY EVICTION MODE
2704 
2705         checkeq(5,
2706                 get_int_stat(h, h1, "vb_0:bloom_filter_key_count",
2707                              "vbucket-details 0"),
2708                 "Unexpected no. of keys in bloom filter");
2709 
2710         checkeq(num_read_attempts,
2711                 get_int_stat_or_default(h, h1, 0, "ep_bg_num_samples"),
2712                 "Expected bgFetch attempts to remain unchanged");
2713 
2714         for (i = 0; i < 5; ++i) {
2715             std::stringstream key;
2716             key << "key-" << i;
2717             check(get_meta(h, h1, key.str().c_str()), "Get meta failed");
2718         }
2719 
2720         // GetMeta would cause bgFetches as bloomfilter contains
2721         // the deleted items.
2722         checkeq(num_read_attempts + 5,
2723                 get_int_stat(h, h1, "ep_bg_num_samples"),
2724                 "Expected bgFetch attempts to increase by five");
2725 
2726         // Run compaction, with drop_deletes
2727         compact_db(h, h1, 0, 0, 15, 15, 1);
2728         while (get_int_stat(h, h1, "ep_pending_compactions") != 0) {
2729             decayingSleep(&sleepTime);
2730         }
2731 
2732         for (i = 0; i < 5; ++i) {
2733             std::stringstream key;
2734             key << "key-" << i;
2735             check(get_meta(h, h1, key.str().c_str()), "Get meta failed");
2736         }
2737         checkeq(num_read_attempts + 5,
2738                 get_int_stat(h, h1, "ep_bg_num_samples"),
2739                 "Expected bgFetch attempts to stay as before");
2740 
2741     } else {                                // FULL EVICTION MODE
2742 
2743         checkeq(10,
2744                 get_int_stat(h, h1, "vb_0:bloom_filter_key_count",
2745                              "vbucket-details 0"),
2746                 "Unexpected no. of keys in bloom filter");
2747 
2748 
2749         // Because of issuing deletes on non-resident items
2750         checkeq(num_read_attempts + 5,
2751                 get_int_stat(h, h1, "ep_bg_num_samples"),
2752                 "Expected bgFetch attempts to increase by five, after deletes");
2753 
2754         // Run compaction, with drop_deletes, to exclude deleted items
2755         // from bloomfilter.
2756         compact_db(h, h1, 0, 0, 15, 15, 1);
2757         while (get_int_stat(h, h1, "ep_pending_compactions") != 0) {
2758             decayingSleep(&sleepTime);
2759         }
2760 
2761         for (i = 0; i < 5; i++) {
2762             std::stringstream key;
2763             key << "key-" << i;
2764             checkeq(cb::engine_errc::no_such_key,
2765                     get(h, h1, NULL, key.str(), 0).first,
2766                     "Unable to get stored item");
2767         }
2768         // + 6 because last delete is not purged by the compactor
2769         checkeq(num_read_attempts + 6,
2770                 get_int_stat(h, h1, "ep_bg_num_samples"),
2771                 "Expected bgFetch attempts to stay as before");
2772     }
2773 
2774     return SUCCESS;
2775 }
2776 
test_bloomfilters_with_store_apis(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2777 static enum test_result test_bloomfilters_with_store_apis(ENGINE_HANDLE *h,
2778                                                           ENGINE_HANDLE_V1 *h1) {
2779     if (get_bool_stat(h, h1, "ep_bfilter_enabled") == false) {
2780         check(set_param(h, h1, protocol_binary_engine_param_flush,
2781                     "bfilter_enabled", "true"),
2782                 "Set bloomfilter_enabled should have worked");
2783     }
2784     check(get_bool_stat(h, h1, "ep_bfilter_enabled"),
2785             "Bloom filter wasn't enabled");
2786 
2787     int num_read_attempts = get_int_stat_or_default(h, h1, 0,
2788                                                     "ep_bg_num_samples");
2789 
2790     // Ensure vbucket's bloom filter is enabled
2791     checkeq(std::string("ENABLED"),
2792             get_str_stat(h, h1, "vb_0:bloom_filter", "vbucket-details 0"),
2793             "Vbucket 0's bloom filter wasn't enabled upon setup!");
2794 
2795     for (int i = 0; i < 1000; i++) {
2796         std::stringstream key;
2797         key << "key-" << i;
2798         check(!get_meta(h, h1, key.str().c_str()),
2799                 "Get meta should fail.");
2800     }
2801 
2802     checkeq(num_read_attempts,
2803             get_int_stat_or_default(h, h1, 0, "ep_bg_num_samples"),
2804             "Expected no bgFetch attempts");
2805 
2806     checkeq(ENGINE_SUCCESS,
2807             get_stats(h, {}, add_stats),
2808             "Failed to get stats.");
2809     std::string eviction_policy = vals.find("ep_item_eviction_policy")->second;
2810 
2811     if (eviction_policy == "full_eviction") {  // FULL EVICTION MODE
2812         // Set with Meta
2813         int j;
2814         for (j = 0; j < 10; j++) {
2815             uint64_t cas_for_set = last_cas;
2816             // init some random metadata
2817             ItemMetaData itm_meta;
2818             itm_meta.revSeqno = 10;
2819             itm_meta.cas = 0xdeadbeef;
2820             itm_meta.exptime = time(NULL) + 300;
2821             itm_meta.flags = 0xdeadbeef;
2822 
2823             std::stringstream key;
2824             key << "swm-" << j;
2825             set_with_meta(h, h1, key.str().c_str(), key.str().length(),
2826                           "somevalue", 9, 0, &itm_meta, cas_for_set);
2827         }
2828 
2829         checkeq(num_read_attempts,
2830                 get_int_stat_or_default(h, h1, 0, "ep_bg_num_samples"),
2831                 "Expected no bgFetch attempts");
2832 
2833         // Add
2834         for (j = 0; j < 10; j++) {
2835             std::stringstream key;
2836             key << "add-" << j;
2837 
2838             checkeq(ENGINE_SUCCESS,
2839                     store(h,
2840                           h1,
2841                           NULL,
2842                           OPERATION_ADD,
2843                           key.str().c_str(),
2844                           "newvalue"),
2845                     "Failed to add value again.");
2846         }
2847 
2848         checkeq(num_read_attempts,
2849                 get_int_stat_or_default(h, h1, 0, "ep_bg_num_samples"),
2850                 "Expected no bgFetch attempts");
2851 
2852         // Delete
2853         for (j = 0; j < 10; j++) {
2854             std::stringstream key;
2855             key << "del-" << j;
2856             checkeq(ENGINE_KEY_ENOENT,
2857                     del(h, h1, key.str().c_str(), 0, 0),
2858                     "Failed remove with value.");
2859         }
2860 
2861         checkeq(num_read_attempts,
2862                 get_int_stat_or_default(h, h1, 0, "ep_bg_num_samples"),
2863                 "Expected no bgFetch attempts");
2864 
2865     }
2866 
2867     return SUCCESS;
2868 }
2869 
test_bloomfilter_delete_plus_set_scenario( ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2870 static enum test_result test_bloomfilter_delete_plus_set_scenario(
2871                                        ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2872     if (get_bool_stat(h, h1, "ep_bfilter_enabled") == false) {
2873         check(set_param(h, h1, protocol_binary_engine_param_flush,
2874                     "bfilter_enabled", "true"),
2875                 "Set bloomfilter_enabled should have worked");
2876     }
2877     check(get_bool_stat(h, h1, "ep_bfilter_enabled"),
2878             "Bloom filter wasn't enabled");
2879 
2880     // Ensure vbucket's bloom filter is enabled
2881     checkeq(std::string("ENABLED"),
2882             get_str_stat(h, h1, "vb_0:bloom_filter", "vbucket-details 0"),
2883             "Vbucket 0's bloom filter wasn't enabled upon setup!");
2884 
2885     checkeq(ENGINE_SUCCESS,
2886             store(h, h1, NULL, OPERATION_SET, "k1", "v1"),
2887             "Failed to fail to store an item.");
2888 
2889     wait_for_flusher_to_settle(h, h1);
2890     int num_writes = get_int_stat(h, h1, "rw_0:io_num_write", "kvstore");
2891     int num_persisted = get_int_stat(h, h1, "ep_total_persisted");
2892     cb_assert(num_writes == 1 && num_persisted == 1);
2893 
2894     checkeq(ENGINE_SUCCESS,
2895             del(h, h1, "k1", 0, 0), "Failed remove with value.");
2896     stop_persistence(h, h1);
2897     checkeq(ENGINE_SUCCESS,
2898             store(h, h1, NULL, OPERATION_SET, "k1", "v2", nullptr, 0, 0),
2899             "Failed to fail to store an item.");
2900     int key_count = get_int_stat(h, h1, "vb_0:bloom_filter_key_count",
2901                                  "vbucket-details 0");
2902 
2903     if (key_count == 0) {
2904         check(get_int_stat(h, h1, "rw_0:io_num_write", "kvstore") <= 2,
2905                 "Unexpected number of writes");
2906         start_persistence(h, h1);
2907         wait_for_flusher_to_settle(h, h1);
2908         checkeq(0, get_int_stat(h, h1, "vb_0:bloom_filter_key_count",
2909                                 "vbucket-details 0"),
2910                 "Unexpected number of keys in bloomfilter");
2911     } else {
2912         cb_assert(key_count == 1);
2913         checkeq(2, get_int_stat(h, h1, "rw_0:io_num_write", "kvstore"),
2914                 "Unexpected number of writes");
2915         start_persistence(h, h1);
2916         wait_for_flusher_to_settle(h, h1);
2917         checkeq(1, get_int_stat(h, h1, "vb_0:bloom_filter_key_count",
2918                                 "vbucket-details 0"),
2919                 "Unexpected number of keys in bloomfilter");
2920     }
2921 
2922     return SUCCESS;
2923 }
2924 
test_datatype(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2925 static enum test_result test_datatype(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2926     const void *cookie = testHarness.create_cookie();
2927     testHarness.set_datatype_support(cookie, true);
2928 
2929     item *itm = NULL;
2930     const std::string key("{\"foo\":\"bar\"}");
2931     const protocol_binary_datatype_t datatype = PROTOCOL_BINARY_DATATYPE_JSON;
2932     uint64_t cas = 0;
2933     std::string value("x");
2934     checkeq(ENGINE_SUCCESS,
2935             storeCasOut(h, h1, NULL, 0, key, value, datatype, itm, cas),
2936             "Expected set to succeed");
2937 
2938     auto ret = get(h, h1, cookie, key, 0);
2939     checkeq(cb::engine_errc::success, ret.first, "Unable to get stored item");
2940 
2941     item_info info;
2942     h1->get_item_info(h, ret.second.get(), &info);
2943     checkeq(static_cast<uint8_t>(PROTOCOL_BINARY_DATATYPE_JSON),
2944             info.datatype, "Invalid datatype");
2945 
2946     const char* key1 = "foo";
2947     const char* val1 = "{\"foo1\":\"bar1\"}";
2948     ItemMetaData itm_meta;
2949     itm_meta.revSeqno = 10;
2950     itm_meta.cas = info.cas;
2951     itm_meta.exptime = info.exptime;
2952     itm_meta.flags = info.flags;
2953     set_with_meta(h, h1, key1, strlen(key1), val1, strlen(val1), 0, &itm_meta,
2954                   last_cas, 0, info.datatype, cookie);
2955 
2956     ret = get(h, h1, cookie, key1, 0);
2957     checkeq(cb::engine_errc::success, ret.first, "Unable to get stored item");
2958 
2959     h1->get_item_info(h, ret.second.get(), &info);
2960     checkeq(static_cast<uint8_t>(PROTOCOL_BINARY_DATATYPE_JSON),
2961             info.datatype, "Invalid datatype, when setWithMeta");
2962 
2963     testHarness.destroy_cookie(cookie);
2964     return SUCCESS;
2965 }
2966 
test_datatype_with_unknown_command(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2967 static enum test_result test_datatype_with_unknown_command(ENGINE_HANDLE *h,
2968                                                            ENGINE_HANDLE_V1 *h1) {
2969     const void *cookie = testHarness.create_cookie();
2970     testHarness.set_datatype_support(cookie, true);
2971     const char* key = "foo";
2972     const char* val = "{\"foo\":\"bar\"}";
2973     uint8_t datatype = PROTOCOL_BINARY_DATATYPE_JSON;
2974 
2975     ItemMetaData itm_meta;
2976     itm_meta.revSeqno = 10;
2977     itm_meta.cas = 0x1;
2978     itm_meta.exptime = 0;
2979     itm_meta.flags = 0;
2980 
2981     //SET_WITH_META
2982     set_with_meta(h, h1, key, strlen(key), val, strlen(val), 0, &itm_meta,
2983                   0, 0, datatype, cookie);
2984 
2985     auto ret = get(h, h1, cookie, key, 0);
2986     checkeq(cb::engine_errc::success, ret.first, "Unable to get stored item");
2987 
2988     item_info info;
2989     h1->get_item_info(h, ret.second.get(), &info);
2990     checkeq(static_cast<uint8_t>(PROTOCOL_BINARY_DATATYPE_JSON),
2991             info.datatype, "Invalid datatype, when setWithMeta");
2992