1 /* -*- MODE: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3  *     Copyright 2010 Couchbase, Inc
4  *
5  *   Licensed under the Apache License, Version 2.0 (the "License");
6  *   you may not use this file except in compliance with the License.
7  *   You may obtain a copy of the License at
8  *
9  *       http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *   Unless required by applicable law or agreed to in writing, software
12  *   distributed under the License is distributed on an "AS IS" BASIS,
13  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *   See the License for the specific language governing permissions and
15  *   limitations under the License.
16  */
17 
18 // Usage: (to run just a single test case)
19 // make engine_tests EP_TEST_NUM=3
20 
21 #include "config.h"
22 
23 #include <stdio.h>
24 #include <stdlib.h>
25 #include <string.h>
26 
27 #include <chrono>
28 #include <condition_variable>
29 #include <cstdlib>
30 #include <iomanip>
31 #include <iostream>
32 #include <map>
33 #include <mutex>
34 #include <regex>
35 #include <set>
36 #include <sstream>
37 #include <string>
38 #include <thread>
39 #include <unordered_map>
40 #include <unordered_set>
41 #include <vector>
42 
43 #include "atomic.h"
44 #include "couch-kvstore/couch-kvstore-metadata.h"
45 #include "ep_test_apis.h"
46 
47 #include "ep_testsuite_common.h"
48 #include "locks.h"
49 #include <libcouchstore/couch_db.h>
50 #include <memcached/engine.h>
51 #include <memcached/engine_testapp.h>
52 #include <platform/cb_malloc.h>
53 #include <platform/dirutils.h>
54 #include <JSON_checker.h>
55 #include <memcached/types.h>
56 #include <string_utilities.h>
57 #include <xattr/blob.h>
58 #include <xattr/utils.h>
59 
60 #ifdef linux
61 /* /usr/include/netinet/in.h defines macros from ntohs() to _bswap_nn to
62  * optimize the conversion functions, but the prototypes generate warnings
63  * from gcc. The conversion methods isn't the bottleneck for my app, so
64  * just remove the warnings by undef'ing the optimization ..
65  */
66 #undef ntohs
67 #undef ntohl
68 #undef htons
69 #undef htonl
70 #endif
71 
72 // ptr_fun don't like the extern "C" thing for unlock cookie.. cast it
73 // away ;)
74 typedef void (*UNLOCK_COOKIE_T)(const void *cookie);
75 
76 class ThreadData {
77 public:
ThreadData(ENGINE_HANDLE *eh, ENGINE_HANDLE_V1 *ehv1, int e=0)78     ThreadData(ENGINE_HANDLE *eh, ENGINE_HANDLE_V1 *ehv1,
79                int e=0) : h(eh), h1(ehv1), extra(e) {}
80     ENGINE_HANDLE    *h;
81     ENGINE_HANDLE_V1 *h1;
82     int               extra;
83 };
84 
85 enum class BucketType { EP, Ephemeral };
86 
check_observe_seqno(bool failover, BucketType bucket_type, uint8_t format_type, uint16_t vb_id, uint64_t vb_uuid, uint64_t last_persisted_seqno, uint64_t current_seqno, uint64_t failover_vbuuid = 0, uint64_t failover_seqno = 0)87 static void check_observe_seqno(bool failover,
88                                 BucketType bucket_type,
89                                 uint8_t format_type,
90                                 uint16_t vb_id,
91                                 uint64_t vb_uuid,
92                                 uint64_t last_persisted_seqno,
93                                 uint64_t current_seqno,
94                                 uint64_t failover_vbuuid = 0,
95                                 uint64_t failover_seqno = 0) {
96     uint8_t recv_format_type;
97     uint16_t recv_vb_id;
98     uint64_t recv_vb_uuid;
99     uint64_t recv_last_persisted_seqno;
100     uint64_t recv_current_seqno;
101     uint64_t recv_failover_vbuuid;
102     uint64_t recv_failover_seqno;
103 
104     memcpy(&recv_format_type, last_body.data(), sizeof(uint8_t));
105     checkeq(format_type, recv_format_type, "Wrong format type in result");
106     memcpy(&recv_vb_id, last_body.data() + 1, sizeof(uint16_t));
107     checkeq(vb_id, ntohs(recv_vb_id), "Wrong vbucket id in result");
108     memcpy(&recv_vb_uuid, last_body.data() + 3, sizeof(uint64_t));
109     checkeq(vb_uuid, ntohll(recv_vb_uuid), "Wrong vbucket uuid in result");
110     memcpy(&recv_last_persisted_seqno, last_body.data() + 11, sizeof(uint64_t));
111 
112     switch (bucket_type) {
113     case BucketType::EP:
114         // Should get the "real" persisted seqno:
115         checkeq(last_persisted_seqno,
116                 ntohll(recv_last_persisted_seqno),
117                 "Wrong persisted seqno in result (EP)");
118         break;
119     case BucketType::Ephemeral:
120         // For ephemeral, this should always be zero, as there is no
121         // persistence.
122         checkeq(uint64_t(0),
123                 ntohll(recv_last_persisted_seqno),
124                 "Wrong persisted seqno in result (Ephemeral)");
125         break;
126     }
127 
128     memcpy(&recv_current_seqno, last_body.data() + 19, sizeof(uint64_t));
129     checkeq(current_seqno, ntohll(recv_current_seqno), "Wrong current seqno in result");
130 
131     if (failover) {
132         memcpy(&recv_failover_vbuuid, last_body.data() + 27, sizeof(uint64_t));
133         checkeq(failover_vbuuid, ntohll(recv_failover_vbuuid),
134                 "Wrong failover uuid in result");
135         memcpy(&recv_failover_seqno, last_body.data() + 35, sizeof(uint64_t));
136         checkeq(failover_seqno, ntohll(recv_failover_seqno),
137                 "Wrong failover seqno in result");
138     }
139 }
140 
test_replace_with_eviction(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)141 static enum test_result test_replace_with_eviction(ENGINE_HANDLE *h,
142                                                    ENGINE_HANDLE_V1 *h1) {
143     checkeq(ENGINE_SUCCESS,
144             store(h, h1, NULL, OPERATION_SET, "key", "somevalue"),
145             "Failed to set value.");
146     wait_for_flusher_to_settle(h, h1);
147     evict_key(h, h1, "key");
148     int numBgFetched = get_int_stat(h, h1, "ep_bg_fetched");
149 
150     checkeq(ENGINE_SUCCESS,
151             store(h, h1, NULL, OPERATION_REPLACE, "key", "somevalue1"),
152             "Failed to replace existing value.");
153 
154     checkeq(ENGINE_SUCCESS,
155             get_stats(h, {}, add_stats),
156             "Failed to get stats.");
157     std::string eviction_policy = vals.find("ep_item_eviction_policy")->second;
158     if (eviction_policy == "full_eviction") {
159         numBgFetched++;
160     }
161 
162     checkeq(numBgFetched,
163             get_int_stat(h, h1, "ep_bg_fetched"),
164             "Bg fetched value didn't match");
165 
166     check_key_value(h, h1, "key", "somevalue1", 10);
167     return SUCCESS;
168 }
169 
test_wrong_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, ENGINE_STORE_OPERATION op)170 static enum test_result test_wrong_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
171                                                ENGINE_STORE_OPERATION op) {
172     int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
173     uint64_t cas = 11;
174     if (op == OPERATION_ADD) {
175         // Add operation with cas != 0 doesn't make sense
176         cas = 0;
177     }
178     checkeq(ENGINE_NOT_MY_VBUCKET,
179             store(h, h1, NULL, op, "key", "somevalue", nullptr, cas, 1),
180             "Expected not_my_vbucket");
181     wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
182     return SUCCESS;
183 }
184 
test_pending_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, ENGINE_STORE_OPERATION op)185 static enum test_result test_pending_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
186                                                  ENGINE_STORE_OPERATION op) {
187     const void *cookie = testHarness.create_cookie();
188     testHarness.set_ewouldblock_handling(cookie, false);
189     check(set_vbucket_state(h, h1, 1, vbucket_state_pending),
190           "Failed to set vbucket state.");
191     check(verify_vbucket_state(h, h1, 1, vbucket_state_pending),
192           "Bucket state was not set to pending.");
193     uint64_t cas = 11;
194     if (op == OPERATION_ADD) {
195         // Add operation with cas != 0 doesn't make sense..
196         cas = 0;
197     }
198     checkeq(ENGINE_EWOULDBLOCK,
199             store(h, h1, cookie, op, "key", "somevalue", nullptr, cas, 1),
200             "Expected ewouldblock");
201     testHarness.destroy_cookie(cookie);
202     return SUCCESS;
203 }
204 
test_replica_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, ENGINE_STORE_OPERATION op)205 static enum test_result test_replica_vb_mutation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
206                                                  ENGINE_STORE_OPERATION op) {
207     check(set_vbucket_state(h, h1, 1, vbucket_state_replica),
208           "Failed to set vbucket state.");
209     check(verify_vbucket_state(h, h1, 1, vbucket_state_replica),
210           "Bucket state was not set to replica.");
211     int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
212 
213     uint64_t cas = 11;
214     if (op == OPERATION_ADD) {
215         // performing add with a CAS != 0 doesn't make sense...
216         cas = 0;
217     }
218     checkeq(ENGINE_NOT_MY_VBUCKET,
219             store(h, h1, NULL, op, "key", "somevalue", nullptr, cas, 1),
220             "Expected not my vbucket");
221     wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
222     return SUCCESS;
223 }
224 
225 //
226 // ----------------------------------------------------------------------
227 // The actual tests are below.
228 // ----------------------------------------------------------------------
229 //
230 
checkCurrItemsAfterShutdown(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, int numItems2Load, bool shutdownForce)231 static int checkCurrItemsAfterShutdown(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
232                                        int numItems2Load, bool shutdownForce) {
233     if (!isWarmupEnabled(h, h1)) {
234         return SKIPPED;
235     }
236 
237     std::vector<std::string> keys;
238     for (int index = 0; index < numItems2Load; ++index) {
239         std::stringstream s;
240         s << "keys_2_load-" << index;
241         std::string key(s.str());
242         keys.push_back(key);
243     }
244 
245     // Check preconditions.
246     checkeq(0, get_int_stat(h, h1, "ep_total_persisted"),
247             "Expected ep_total_persisted equals 0");
248     checkeq(0, get_int_stat(h, h1, "curr_items"),
249             "Expected curr_items equals 0");
250 
251     // stop flusher before loading new items
252     protocol_binary_request_header *pkt = createPacket(PROTOCOL_BINARY_CMD_STOP_PERSISTENCE);
253     checkeq(ENGINE_SUCCESS,
254             h1->unknown_command(h, NULL, pkt, add_response, testHarness.doc_namespace),
255             "CMD_STOP_PERSISTENCE failed!");
256     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS,
257             last_status.load(),
258             "Failed to stop persistence!");
259     cb_free(pkt);
260 
261     std::vector<std::string>::iterator itr;
262     for (itr = keys.begin(); itr != keys.end(); ++itr) {
263         checkeq(ENGINE_SUCCESS,
264                 store(h, h1, NULL, OPERATION_SET, itr->c_str(), "oracle"),
265                 "Failed to store a value");
266     }
267 
268     checkeq(0, get_int_stat(h, h1, "ep_total_persisted"),
269             "Incorrect ep_total_persisted, expected 0");
270 
271     // Can only check curr_items in value_only eviction; full-eviction
272     // relies on persistence to complete (via flusher) to update count.
273     const auto evictionPolicy = get_str_stat(h, h1, "ep_item_eviction_policy");
274     if (evictionPolicy == "value_only") {
275         checkeq(numItems2Load,
276                 get_int_stat(h, h1, "curr_items"),
277                 "Expected curr_items to reflect item count");
278     }
279 
280     // resume flusher before shutdown + warmup
281     pkt = createPacket(PROTOCOL_BINARY_CMD_START_PERSISTENCE);
282     checkeq(ENGINE_SUCCESS, h1->unknown_command(h, NULL, pkt, add_response, testHarness.doc_namespace),
283             "CMD_START_PERSISTENCE failed!");
284     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
285           "Failed to start persistence!");
286     cb_free(pkt);
287 
288     // shutdown engine force and restart
289     testHarness.reload_engine(&h, &h1,
290                               testHarness.engine_path,
291                               testHarness.get_current_testcase()->cfg,
292                               true, shutdownForce);
293     wait_for_warmup_complete(h, h1);
294     return get_int_stat(h, h1, "curr_items");
295 }
296 
test_flush_shutdown_force(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)297 static enum test_result test_flush_shutdown_force(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
298     if (!isWarmupEnabled(h, h1)) {
299         return SKIPPED;
300     }
301 
302     int numItems2load = 3000;
303     bool shutdownForce = true;
304     int currItems = checkCurrItemsAfterShutdown(h, h1, numItems2load, shutdownForce);
305     check (currItems <= numItems2load,
306            "Number of curr items should be <= 3000, unless previous "
307            "shutdown force had to wait for the flusher");
308     return SUCCESS;
309 }
310 
test_flush_shutdown_noforce(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)311 static enum test_result test_flush_shutdown_noforce(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
312     if (!isWarmupEnabled(h, h1)) {
313         return SKIPPED;
314     }
315 
316     int numItems2load = 3000;
317     bool shutdownForce = false;
318     int currItems = checkCurrItemsAfterShutdown(h, h1, numItems2load, shutdownForce);
319     check (currItems == numItems2load,
320            "Number of curr items should be equal to 3000, unless previous "
321            "shutdown did not wait for the flusher");
322     return SUCCESS;
323 }
324 
test_shutdown_snapshot_range(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)325 static enum test_result test_shutdown_snapshot_range(ENGINE_HANDLE *h,
326                                                      ENGINE_HANDLE_V1 *h1) {
327     if (!isWarmupEnabled(h, h1)) {
328         return SKIPPED;
329     }
330 
331     const int num_items = 100;
332     for (int j = 0; j < num_items; ++j) {
333         std::stringstream ss;
334         ss << "key" << j;
335         checkeq(ENGINE_SUCCESS,
336                 store(h, h1, NULL, OPERATION_SET, ss.str().c_str(), "data"),
337                 "Failed to store a value");
338     }
339 
340     wait_for_flusher_to_settle(h, h1);
341     int end = get_int_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno");
342 
343     /* change vb state to replica before restarting (as it happens in graceful
344        failover)*/
345     check(set_vbucket_state(h, h1, 0, vbucket_state_replica),
346           "Failed set vbucket 0 to replica state.");
347 
348     /* trigger persist vb state task */
349     check(set_param(h, h1, protocol_binary_engine_param_flush,
350                     "vb_state_persist_run", "0"),
351           "Failed to trigger vb state persist");
352 
353     /* restart the engine */
354     testHarness.reload_engine(&h, &h1,
355                               testHarness.engine_path,
356                               testHarness.get_current_testcase()->cfg,
357                               true, false);
358     wait_for_warmup_complete(h, h1);
359 
360     /* Check if snapshot range is persisted correctly */
361     checkeq(end, get_int_stat(h, h1, "vb_0:last_persisted_snap_start",
362                               "vbucket-seqno"),
363             "Wrong snapshot start persisted");
364     checkeq(end, get_int_stat(h, h1, "vb_0:last_persisted_snap_end",
365                                     "vbucket-seqno"),
366             "Wrong snapshot end persisted");
367 
368     return SUCCESS;
369 }
370 
test_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)371 static enum test_result test_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
372     if (!isWarmupEnabled(h, h1)) {
373         return SKIPPED;
374     }
375 
376     static const char val[] = "somevalue";
377     ENGINE_ERROR_CODE ret = store(h, h1, NULL, OPERATION_SET, "key", val);
378     checkeq(ENGINE_SUCCESS, ret, "Failed set.");
379 
380     testHarness.reload_engine(&h, &h1,
381                               testHarness.engine_path,
382                               testHarness.get_current_testcase()->cfg,
383                               true, false);
384     wait_for_warmup_complete(h, h1);
385     check_key_value(h, h1, "key", val, strlen(val));
386     return SUCCESS;
387 }
388 
test_specialKeys(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)389 static enum test_result test_specialKeys(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
390     ENGINE_ERROR_CODE ret;
391 
392     // Simplified Chinese "Couchbase"
393     static const char key0[] = "沙发数据库";
394     static const char val0[] = "some Chinese value";
395     check((ret = store(h, h1, NULL, OPERATION_SET, key0, val0)) ==
396                   ENGINE_SUCCESS,
397           "Failed set Chinese key");
398     check_key_value(h, h1, key0, val0, strlen(val0));
399 
400     // Traditional Chinese "Couchbase"
401     static const char key1[] = "沙發數據庫";
402     static const char val1[] = "some Traditional Chinese value";
403     check((ret = store(h, h1, NULL, OPERATION_SET, key1, val1)) ==
404                   ENGINE_SUCCESS,
405           "Failed set Traditional Chinese key");
406 
407     // Korean "couch potato"
408     static const char key2[] = "쇼파감자";
409     static const char val2[] = "some Korean value";
410     check((ret = store(h, h1, NULL, OPERATION_SET, key2, val2)) ==
411                   ENGINE_SUCCESS,
412           "Failed set Korean key");
413 
414     // Russian "couch potato"
415     static const char key3[] = "лодырь, лентяй";
416     static const char val3[] = "some Russian value";
417     check((ret = store(h, h1, NULL, OPERATION_SET, key3, val3)) ==
418                   ENGINE_SUCCESS,
419           "Failed set Russian key");
420 
421     // Japanese "couch potato"
422     static const char key4[] = "カウチポテト";
423     static const char val4[] = "some Japanese value";
424     check((ret = store(h, h1, NULL, OPERATION_SET, key4, val4)) ==
425                   ENGINE_SUCCESS,
426           "Failed set Japanese key");
427 
428     // Indian char key, and no idea what it is
429     static const char key5[] = "हरियानवी";
430     static const char val5[] = "some Indian value";
431     check((ret = store(h, h1, NULL, OPERATION_SET, key5, val5)) ==
432                   ENGINE_SUCCESS,
433           "Failed set Indian key");
434 
435     // Portuguese translation "couch potato"
436     static const char key6[] = "sedentário";
437     static const char val6[] = "some Portuguese value";
438     check((ret = store(h, h1, NULL, OPERATION_SET, key6, val6)) ==
439                   ENGINE_SUCCESS,
440           "Failed set Portuguese key");
441 
442     // Arabic translation "couch potato"
443     static const char key7[] = "الحافلةالبطاطة";
444     static const char val7[] = "some Arabic value";
445     check((ret = store(h, h1, NULL, OPERATION_SET, key7, val7)) ==
446                   ENGINE_SUCCESS,
447           "Failed set Arabic key");
448 
449     if (isWarmupEnabled(h, h1)) {
450         // Check that after warmup the keys are still present.
451         testHarness.reload_engine(&h, &h1,
452                                   testHarness.engine_path,
453                                   testHarness.get_current_testcase()->cfg,
454                                   true, false);
455         wait_for_warmup_complete(h, h1);
456         check_key_value(h, h1, key0, val0, strlen(val0));
457         check_key_value(h, h1, key1, val1, strlen(val1));
458         check_key_value(h, h1, key2, val2, strlen(val2));
459         check_key_value(h, h1, key3, val3, strlen(val3));
460         check_key_value(h, h1, key4, val4, strlen(val4));
461         check_key_value(h, h1, key5, val5, strlen(val5));
462         check_key_value(h, h1, key6, val6, strlen(val6));
463         check_key_value(h, h1, key7, val7, strlen(val7));
464     }
465     return SUCCESS;
466 }
467 
test_binKeys(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)468 static enum test_result test_binKeys(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
469     ENGINE_ERROR_CODE ret;
470 
471     // binary key with char values beyond 0x7F
472     static const char key0[] = "\xe0\xed\xf1\x6f\x7f\xf8\xfa";
473     static const char val0[] = "some value val8";
474     check((ret = store(h, h1, NULL, OPERATION_SET, key0, val0)) ==
475                   ENGINE_SUCCESS,
476           "Failed set binary key0");
477     check_key_value(h, h1, key0, val0, strlen(val0));
478 
479     // binary keys with char values beyond 0x7F
480     static const char key1[] = "\xf1\xfd\xfe\xff\xf0\xf8\xef";
481     static const char val1[] = "some value val9";
482     check((ret = store(h, h1, NULL, OPERATION_SET, key1, val1)) ==
483                   ENGINE_SUCCESS,
484           "Failed set binary key1");
485     check_key_value(h, h1, key1, val1, strlen(val1));
486 
487     // binary keys with special utf-8 BOM (Byte Order Mark) values 0xBB 0xBF
488     // 0xEF
489     static const char key2[] = "\xff\xfe\xbb\xbf\xef";
490     static const char val2[] = "some utf-8 bom value";
491     check((ret = store(h, h1, NULL, OPERATION_SET, key2, val2)) ==
492                   ENGINE_SUCCESS,
493           "Failed set binary utf-8 bom key");
494     check_key_value(h, h1, key2, val2, strlen(val2));
495 
496     // binary keys with special utf-16BE BOM values "U+FEFF"
497     static const char key3[] = "U+\xfe\xff\xefU+\xff\xfe";
498     static const char val3[] = "some utf-16 bom value";
499     check((ret = store(h, h1, NULL, OPERATION_SET, key3, val3)) ==
500                   ENGINE_SUCCESS,
501           "Failed set binary utf-16 bom key");
502     check_key_value(h, h1, key3, val3, strlen(val3));
503 
504     if (isWarmupEnabled(h, h1)) {
505         testHarness.reload_engine(&h, &h1,
506                                   testHarness.engine_path,
507                                   testHarness.get_current_testcase()->cfg,
508                                   true, false);
509         wait_for_warmup_complete(h, h1);
510         check_key_value(h, h1, key0, val0, strlen(val0));
511         check_key_value(h, h1, key1, val1, strlen(val1));
512         check_key_value(h, h1, key2, val2, strlen(val2));
513         check_key_value(h, h1, key3, val3, strlen(val3));
514     }
515     return SUCCESS;
516 }
517 
test_restart_bin_val(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)518 static enum test_result test_restart_bin_val(ENGINE_HANDLE *h,
519                                              ENGINE_HANDLE_V1 *h1) {
520     if (!isWarmupEnabled(h, h1)) {
521         return SKIPPED;
522     }
523 
524     char binaryData[] = "abcdefg\0gfedcba";
525     cb_assert(sizeof(binaryData) != strlen(binaryData));
526 
527     checkeq(cb::engine_errc::success,
528             storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
529                          binaryData, sizeof(binaryData), 82758, 0, 0).first,
530             "Failed set.");
531 
532     testHarness.reload_engine(&h, &h1,
533                               testHarness.engine_path,
534                               testHarness.get_current_testcase()->cfg,
535                               true, false);
536     wait_for_warmup_complete(h, h1);
537 
538     check_key_value(h, h1, "key", binaryData, sizeof(binaryData));
539     return SUCCESS;
540 }
541 
test_wrong_vb_get(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)542 static enum test_result test_wrong_vb_get(ENGINE_HANDLE *h,
543                                           ENGINE_HANDLE_V1 *h1) {
544     int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
545     checkeq(ENGINE_NOT_MY_VBUCKET, verify_key(h, h1, "key", 1),
546             "Expected wrong bucket.");
547     wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
548     return SUCCESS;
549 }
550 
test_vb_get_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)551 static enum test_result test_vb_get_pending(ENGINE_HANDLE *h,
552                                             ENGINE_HANDLE_V1 *h1) {
553     check(set_vbucket_state(h, h1, 1, vbucket_state_pending),
554           "Failed to set vbucket state.");
555     const void *cookie = testHarness.create_cookie();
556     testHarness.set_ewouldblock_handling(cookie, false);
557 
558     checkeq(cb::engine_errc::would_block,
559             get(h, h1, cookie, "key", 1).first,
560             "Expected woodblock.");
561 
562     testHarness.destroy_cookie(cookie);
563     return SUCCESS;
564 }
565 
test_vb_get_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)566 static enum test_result test_vb_get_replica(ENGINE_HANDLE *h,
567                                             ENGINE_HANDLE_V1 *h1) {
568     check(set_vbucket_state(h, h1, 1, vbucket_state_replica),
569           "Failed to set vbucket state.");
570     int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
571     checkeq(ENGINE_NOT_MY_VBUCKET,
572             verify_key(h, h1, "key", 1),
573             "Expected not my bucket.");
574     wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
575     return SUCCESS;
576 }
577 
test_wrong_vb_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)578 static enum test_result test_wrong_vb_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
579     return test_wrong_vb_mutation(h, h1, OPERATION_SET);
580 }
581 
test_wrong_vb_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)582 static enum test_result test_wrong_vb_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
583     return test_wrong_vb_mutation(h, h1, OPERATION_CAS);
584 }
585 
test_wrong_vb_add(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)586 static enum test_result test_wrong_vb_add(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
587     return test_wrong_vb_mutation(h, h1, OPERATION_ADD);
588 }
589 
test_wrong_vb_replace(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)590 static enum test_result test_wrong_vb_replace(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
591     return test_wrong_vb_mutation(h, h1, OPERATION_REPLACE);
592 }
593 
test_wrong_vb_del(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)594 static enum test_result test_wrong_vb_del(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
595     int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
596     checkeq(ENGINE_NOT_MY_VBUCKET, del(h, h1, "key", 0, 1),
597             "Expected wrong bucket.");
598     wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
599     return SUCCESS;
600 }
601 
602 /* Returns a string in the format "%Y-%m-%d %H:%M:%S" of the specified
603  * time point.
604  */
make_time_string(std::chrono::system_clock::time_point time_point)605 std::string make_time_string(std::chrono::system_clock::time_point time_point) {
606     time_t tt = std::chrono::system_clock::to_time_t(time_point);
607 #ifdef _MSC_VER
608     // Windows' gmtime() is already thread-safe.
609     struct tm* split = gmtime(&tt);
610 #else
611     struct tm local_storage;
612     struct tm* split = gmtime_r(&tt, &local_storage);
613 #endif
614     char timeStr[20];
615     strftime(timeStr, 20, "%Y-%m-%d %H:%M:%S", split);
616     return timeStr;
617 }
618 
test_expiry_pager_settings(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)619 static enum test_result test_expiry_pager_settings(ENGINE_HANDLE *h,
620                                                    ENGINE_HANDLE_V1 *h1) {
621 
622     cb_assert(!get_bool_stat(h, h1, "ep_exp_pager_enabled"));
623     checkeq(3600, get_int_stat(h, h1, "ep_exp_pager_stime"),
624             "Expiry pager sleep time not expected");
625     set_param(h, h1, protocol_binary_engine_param_flush,
626               "exp_pager_stime", "1");
627     checkeq(1, get_int_stat(h, h1, "ep_exp_pager_stime"),
628             "Expiry pager sleep time not updated");
629     cb_assert(!get_bool_stat(h, h1, "ep_exp_pager_enabled"));
630     sleep(1);
631     checkeq(0, get_int_stat(h, h1, "ep_num_expiry_pager_runs"),
632             "Expiry pager run count is not zero");
633 
634     set_param(h, h1, protocol_binary_engine_param_flush,
635               "exp_pager_enabled", "true");
636     checkeq(1, get_int_stat(h, h1, "ep_exp_pager_stime"),
637             "Expiry pager sleep time not updated");
638     wait_for_stat_to_be_gte(h, h1, "ep_num_expiry_pager_runs", 1);
639 
640     // Reload engine
641     testHarness.reload_engine(&h, &h1,
642                               testHarness.engine_path,
643                               testHarness.get_current_testcase()->cfg,
644                               true, false);
645     wait_for_warmup_complete(h, h1);
646     cb_assert(!get_bool_stat(h, h1, "ep_exp_pager_enabled"));
647 
648     // Enable expiry pager again
649     set_param(h, h1, protocol_binary_engine_param_flush,
650               "exp_pager_enabled", "true");
651 
652     checkeq(get_int_stat(h, h1, "ep_exp_pager_initial_run_time"), -1,
653             "Task time should be disable upon warmup");
654 
655     std::string err_msg;
656     // Update exp_pager_initial_run_time and ensure the update is successful
657     set_param(h, h1, protocol_binary_engine_param_flush,
658               "exp_pager_initial_run_time", "3");
659     std::string expected_time = "03:00";
660     std::string str;
661     // [MB-21806] - Need to repeat the fetch as the set_param for
662     // "exp_pager_initial_run_time" schedules a task that sets the stats later
663     repeat_till_true([&]() {
664         str = get_str_stat(h, h1, "ep_expiry_pager_task_time");
665         return 0 == str.substr(11, 5).compare(expected_time);
666     });
667     err_msg.assign("Updated time incorrect, expect: " +
668                    expected_time + ", actual: " + str.substr(11, 5));
669     checkeq(0, str.substr(11, 5).compare(expected_time), err_msg.c_str());
670 
671     // Update exp_pager_stime by 30 minutes and ensure that the update is successful
672     const std::chrono::minutes update_by{30};
673     std::string targetTaskTime1{make_time_string(std::chrono::system_clock::now() +
674                                                  update_by)};
675 
676     set_param(h, h1, protocol_binary_engine_param_flush, "exp_pager_stime",
677               std::to_string(update_by.count() * 60).c_str());
678     str = get_str_stat(h, h1, "ep_expiry_pager_task_time");
679 
680     std::string targetTaskTime2{make_time_string(std::chrono::system_clock::now() +
681                                                  update_by)};
682 
683     // ep_expiry_pager_task_time should fall within the range of
684     // targetTaskTime1 and targetTaskTime2
685     err_msg.assign("Unexpected task time range, expect: " +
686                    targetTaskTime1 + " <= " + str + " <= " + targetTaskTime2);
687     check(targetTaskTime1 <= str, err_msg.c_str());
688     check(str <= targetTaskTime2, err_msg.c_str());
689 
690     return SUCCESS;
691 }
692 
test_expiry_with_xattr(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1)693 static enum test_result test_expiry_with_xattr(ENGINE_HANDLE* h,
694                                                ENGINE_HANDLE_V1* h1) {
695     const char* key = "test_expiry";
696     cb::xattr::Blob blob;
697 
698     //Add a few XAttrs
699     blob.set("user", "{\"author\":\"bubba\"}");
700     blob.set("_sync", "{\"cas\":\"0xdeadbeefcafefeed\"}");
701     blob.set("meta", "{\"content-type\":\"text\"}");
702 
703     auto xattr_value = blob.finalize();
704 
705     //Now, append user data to the xattrs and store the data
706     std::string value_data("test_expiry_value");
707     std::vector<char> data;
708     std::copy(xattr_value.buf, xattr_value.buf + xattr_value.len,
709               std::back_inserter(data));
710     std::copy(value_data.c_str(), value_data.c_str() + value_data.length(),
711               std::back_inserter(data));
712 
713     const void* cookie = testHarness.create_cookie();
714 
715     checkeq(cb::engine_errc::success,
716             storeCasVb11(h, h1, cookie, OPERATION_SET, key,
717                          reinterpret_cast<char*>(data.data()),
718                          data.size(), 9258, 0, 0, 10,
719                          PROTOCOL_BINARY_DATATYPE_XATTR).first,
720             "Failed to store xattr document");
721 
722     if (isPersistentBucket(h, h1)) {
723         wait_for_flusher_to_settle(h, h1);
724     }
725 
726     testHarness.time_travel(11);
727 
728     cb::EngineErrorMetadataPair errorMetaPair;
729 
730     check(get_meta(h, h1, "test_expiry", errorMetaPair, cookie),
731           "Get meta command failed");
732     auto prev_revseqno = errorMetaPair.second.seqno;
733 
734     checkeq(PROTOCOL_BINARY_DATATYPE_XATTR,
735             errorMetaPair.second.datatype,
736             "Datatype is not XATTR");
737 
738     auto ret = get(h, h1, cookie, key, 0, DocStateFilter::AliveOrDeleted);
739     checkeq(cb::engine_errc::success,
740             ret.first,
741             "Unable to get a deleted item");
742 
743     check(get_meta(h, h1, "test_expiry", errorMetaPair, cookie),
744           "Get meta command failed");
745 
746     checkeq(errorMetaPair.second.seqno,
747             prev_revseqno + 1,
748             "rev seqno must have incremented by 1");
749 
750     /* Retrieve the item info and create a new blob out of the data */
751     item_info info;
752     checkeq(true,
753             h1->get_item_info(h, ret.second.get(), &info),
754             "Unable to retrieve item info");
755 
756     cb::char_buffer value_buf{static_cast<char*>(info.value[0].iov_base),
757                               info.value[0].iov_len};
758 
759     cb::xattr::Blob new_blob(value_buf, false);
760 
761     /* Only system extended attributes need to be present at this point.
762      * Thus, check the blob length with the system size.
763      */
764     const auto systemsize = new_blob.finalize().len;
765 
766     checkeq(systemsize, new_blob.get_system_size(),
767             "The size of the blob doesn't match the size of system attributes");
768 
769     const std::string& cas_str{"{\"cas\":\"0xdeadbeefcafefeed\"}"};
770     const std::string& sync_str = to_string(blob.get("_sync"));
771 
772     checkeq(cas_str, sync_str , "system xattr is invalid");
773 
774     testHarness.destroy_cookie(cookie);
775 
776     return SUCCESS;
777 }
778 
test_expiry(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)779 static enum test_result test_expiry(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
780     const char *key = "test_expiry";
781     const char *data = "some test data here.";
782 
783     const void* cookie = testHarness.create_cookie();
784     auto ret = allocate(h,
785                         h1,
786                         cookie,
787                         key,
788                         strlen(data),
789                         0,
790                         2,
791                         PROTOCOL_BINARY_RAW_BYTES,
792                         0);
793     checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
794 
795     item_info info;
796     if (!h1->get_item_info(h, ret.second.get(), &info)) {
797         abort();
798     }
799     memcpy(info.value[0].iov_base, data, strlen(data));
800 
801     uint64_t cas = 0;
802     auto rv = h1->store(h,
803                         cookie,
804                         ret.second.get(),
805                         cas,
806                         OPERATION_SET,
807                         DocumentState::Alive);
808     checkeq(ENGINE_SUCCESS, rv, "Set failed.");
809     check_key_value(h, h1, key, data, strlen(data));
810 
811     testHarness.time_travel(5);
812     checkeq(cb::engine_errc::no_such_key,
813             get(h, h1, cookie, key, 0).first,
814             "Item didn't expire");
815 
816     int expired_access = get_int_stat(h, h1, "ep_expired_access");
817     int expired_pager = get_int_stat(h, h1, "ep_expired_pager");
818     int active_expired = get_int_stat(h, h1, "vb_active_expired");
819     checkeq(0, expired_pager, "Expected zero expired item by pager");
820     checkeq(1, expired_access, "Expected an expired item on access");
821     checkeq(1, active_expired, "Expected an expired active item");
822     checkeq(ENGINE_SUCCESS,
823             store(h, h1, cookie, OPERATION_SET, key, data),
824             "Failed set.");
825 
826     // When run under full eviction, the total item stats are set from the
827     // flusher. So we need to wait for it to finish before checking the
828     // total number of items.
829     wait_for_flusher_to_settle(h, h1);
830 
831     std::stringstream ss;
832     ss << "curr_items stat should be still 1 after ";
833     ss << "overwriting the key that was expired, but not purged yet";
834     checkeq(1, get_int_stat(h, h1, "curr_items"), ss.str().c_str());
835 
836     testHarness.destroy_cookie(cookie);
837     return SUCCESS;
838 }
839 
test_expiry_loader(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)840 static enum test_result test_expiry_loader(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
841     if (!isWarmupEnabled(h, h1)) {
842         return SKIPPED;
843     }
844     const char *key = "test_expiry_loader";
845     const char *data = "some test data here.";
846 
847     const void* cookie = testHarness.create_cookie();
848     auto ret = allocate(h,
849                         h1,
850                         cookie,
851                         key,
852                         strlen(data),
853                         0,
854                         2,
855                         PROTOCOL_BINARY_RAW_BYTES,
856                         0);
857     checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
858 
859     item_info info;
860     if (!h1->get_item_info(h, ret.second.get(), &info)) {
861         abort();
862     }
863     memcpy(info.value[0].iov_base, data, strlen(data));
864 
865     uint64_t cas = 0;
866     auto rv = h1->store(h,
867                         cookie,
868                         ret.second.get(),
869                         cas,
870                         OPERATION_SET,
871                         DocumentState::Alive);
872     checkeq(ENGINE_SUCCESS, rv, "Set failed.");
873     check_key_value(h, h1, key, data, strlen(data));
874 
875     testHarness.time_travel(3);
876 
877     ret = get(h, h1, cookie, key, 0);
878     checkeq(cb::engine_errc::no_such_key, ret.first, "Item didn't expire");
879 
880     // Restart the engine to ensure the above expired item is not loaded
881     testHarness.reload_engine(&h, &h1,
882                               testHarness.engine_path,
883                               testHarness.get_current_testcase()->cfg,
884                               true, false);
885     wait_for_warmup_complete(h, h1);
886     cb_assert(0 == get_int_stat(h, h1, "ep_warmup_value_count", "warmup"));
887 
888     testHarness.destroy_cookie(cookie);
889 
890     return SUCCESS;
891 }
892 
test_expiration_on_compaction(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)893 static enum test_result test_expiration_on_compaction(ENGINE_HANDLE *h,
894                                                       ENGINE_HANDLE_V1 *h1) {
895     if (get_bool_stat(h, h1, "ep_exp_pager_enabled")) {
896         set_param(h, h1, protocol_binary_engine_param_flush,
897                   "exp_pager_enabled", "false");
898     }
899 
900     checkeq(1,
901             get_int_stat(h, h1, "vb_0:persistence:num_visits", "checkpoint"),
902             "Cursor moved before item load");
903 
904     for (int i = 0; i < 25; i++) {
905         std::stringstream ss;
906         ss << "key" << i;
907         checkeq(ENGINE_SUCCESS,
908                 store(h,
909                       h1,
910                       NULL,
911                       OPERATION_SET,
912                       ss.str().c_str(),
913                       "somevalue",
914                       nullptr,
915                       0,
916                       0,
917                       10,
918                       PROTOCOL_BINARY_RAW_BYTES),
919                 "Set failed.");
920     }
921 
922     // Throw an xattr document in (and later compressed)
923     cb::xattr::Blob builder;
924     builder.set("_ep", "{\"foo\":\"bar\"}");
925     builder.set("key", "{\"foo\":\"bar\"}");
926     builder.set("_sync", "{\"foo\":\"bar\"}");
927     builder.set("stuff", "{\"foo\":\"bar\"}");
928     builder.set("misc", "{\"foo\":\"bar\"}");
929     builder.set("things", "{\"foo\":\"bar\"}");
930     builder.set("that", "{\"foo\":\"bar\"}");
931 
932     auto blob = builder.finalize();
933     std::string data;
934     std::copy(blob.buf, blob.buf + blob.size(), std::back_inserter(data));
935     for (int i = 0; i < 12; i++) {
936         std::stringstream ss;
937         ss << "xattr_key" << i;
938 
939         checkeq(cb::engine_errc::success,
940                 storeCasVb11(h,
941                              h1,
942                              nullptr,
943                              OPERATION_SET,
944                              ss.str().c_str(),
945                              data.data(),
946                              data.size(),
947                              0,
948                              0,
949                              0,
950                              10,
951                              PROTOCOL_BINARY_DATATYPE_XATTR,
952                              DocumentState::Alive)
953                         .first,
954                 "Unable to store item");
955     }
956 
957     cb::compression::Buffer compressedDoc;
958     cb::compression::deflate(
959             cb::compression::Algorithm::Snappy, data, compressedDoc);
960 
961     for (int i = 0; i < 13; i++) {
962         std::stringstream ss;
963         ss << "compressed_xattr_key" << i;
964 
965         checkeq(cb::engine_errc::success,
966                 storeCasVb11(h,
967                              h1,
968                              nullptr,
969                              OPERATION_SET,
970                              ss.str().c_str(),
971                              compressedDoc.data(),
972                              compressedDoc.size(),
973                              0,
974                              0,
975                              0,
976                              10,
977                              PROTOCOL_BINARY_DATATYPE_XATTR |
978                                      PROTOCOL_BINARY_DATATYPE_SNAPPY,
979                              DocumentState::Alive)
980                         .first,
981                 "Unable to store item");
982     }
983 
984     wait_for_flusher_to_settle(h, h1);
985     checkeq(50, get_int_stat(h, h1, "curr_items"),
986             "Unexpected number of items on database");
987     check(1 < get_int_stat(h, h1, "vb_0:persistence:num_visits", "checkpoint"),
988           "Cursor not moved even after flusher runs");
989 
990     testHarness.time_travel(15);
991 
992     // Compaction on VBucket
993     compact_db(h, h1, 0, 0, 0, 0, 0);
994     wait_for_stat_to_be(h, h1, "ep_pending_compactions", 0);
995 
996     checkeq(50, get_int_stat(h, h1, "ep_expired_compactor"),
997             "Unexpected expirations by compactor");
998 
999     return SUCCESS;
1000 }
1001 
test_expiration_on_warmup(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1002 static enum test_result test_expiration_on_warmup(ENGINE_HANDLE *h,
1003                                                   ENGINE_HANDLE_V1 *h1) {
1004     if (!isWarmupEnabled(h, h1)) {
1005         return SKIPPED;
1006     }
1007 
1008     const auto* cookie = testHarness.create_cookie();
1009     set_param(h, h1, protocol_binary_engine_param_flush,
1010               "exp_pager_enabled", "false");
1011     int pager_runs = get_int_stat(h, h1, "ep_num_expiry_pager_runs");
1012 
1013     const char *key = "KEY";
1014     const char *data = "VALUE";
1015 
1016     auto ret = allocate(h,
1017                         h1,
1018                         cookie,
1019                         key,
1020                         strlen(data),
1021                         0,
1022                         10,
1023                         PROTOCOL_BINARY_RAW_BYTES,
1024                         0);
1025     checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
1026 
1027     item_info info;
1028     if (!h1->get_item_info(h, ret.second.get(), &info)) {
1029         abort();
1030     }
1031     memcpy(info.value[0].iov_base, data, strlen(data));
1032 
1033     uint64_t cas = 0;
1034     auto rv = h1->store(h,
1035                         cookie,
1036                         ret.second.get(),
1037                         cas,
1038                         OPERATION_SET,
1039                         DocumentState::Alive);
1040     checkeq(ENGINE_SUCCESS, rv, "Set failed.");
1041     check_key_value(h, h1, key, data, strlen(data));
1042     ret.second.reset();
1043     wait_for_flusher_to_settle(h, h1);
1044 
1045     checkeq(1, get_int_stat(h, h1, "curr_items"), "Failed store item");
1046     testHarness.time_travel(15);
1047 
1048     checkeq(pager_runs, get_int_stat(h, h1, "ep_num_expiry_pager_runs"),
1049             "Expiry pager shouldn't have run during this time");
1050 
1051     // Restart the engine to ensure the above item is expired
1052     testHarness.reload_engine(&h, &h1,
1053                               testHarness.engine_path,
1054                               testHarness.get_current_testcase()->cfg,
1055                               true, false);
1056     wait_for_warmup_complete(h, h1);
1057     check(get_bool_stat(h, h1, "ep_exp_pager_enabled"),
1058           "Expiry pager should be enabled on warmup");
1059 
1060     // Wait for the expiry pager to run and expire our item.
1061     wait_for_stat_to_be_gte(h, h1, "ep_expired_pager", 1, nullptr, /*secs*/10);
1062 
1063     // Note: previously we checked that curr_items was zero here (immediately
1064     // after waiting for ep_expired_pager == 1), however we cannot assume that
1065     // - items are actually expired asynchronously.
1066     // See EPStore::deleteExpiredItem - for non-temporary, expired items we
1067     // call processSoftDelete (soft-marking the item as deleted in the
1068     // hashtable), and then call queueDirty to queue a deletion, and then
1069     // increment the expired stat. Only when that delete is actually persisted
1070     // and the deleted callback is invoked -
1071     // PeristenceCallback::callback(int&) - is curr_items finally decremented.
1072     // Therefore we need to wait for the flusher to settle (i.e. delete
1073     // callback to be called) for the curr_items stat to be accurate.
1074     wait_for_flusher_to_settle(h, h1);
1075 
1076     checkeq(0, get_int_stat(h, h1, "curr_items"),
1077             "The item should have been expired.");
1078 
1079     testHarness.destroy_cookie(cookie);
1080     return SUCCESS;
1081 
1082 }
1083 
test_bug3454(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1084 static enum test_result test_bug3454(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1085     if (!isWarmupEnabled(h, h1)) {
1086         return SKIPPED;
1087     }
1088 
1089     const char *key = "test_expiry_duplicate_warmup";
1090     const char *data = "some test data here.";
1091 
1092     const void* cookie = testHarness.create_cookie();
1093     auto ret = allocate(h,
1094                         h1,
1095                         cookie,
1096                         key,
1097                         strlen(data),
1098                         0,
1099                         5,
1100                         PROTOCOL_BINARY_RAW_BYTES,
1101                         0);
1102     checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
1103 
1104     item_info info;
1105     if (!h1->get_item_info(h, ret.second.get(), &info)) {
1106         abort();
1107     }
1108     memcpy(info.value[0].iov_base, data, strlen(data));
1109 
1110     uint64_t cas = 0;
1111     auto rv = h1->store(h,
1112                         cookie,
1113                         ret.second.get(),
1114                         cas,
1115                         OPERATION_SET,
1116                         DocumentState::Alive);
1117     checkeq(ENGINE_SUCCESS, rv, "Set failed.");
1118     check_key_value(h, h1, key, data, strlen(data));
1119     wait_for_flusher_to_settle(h, h1);
1120 
1121     // Advance the ep_engine time by 10 sec for the above item to be expired.
1122     testHarness.time_travel(10);
1123     ret = get(h, h1, cookie, key, 0);
1124     checkeq(cb::engine_errc::no_such_key, ret.first, "Item didn't expire");
1125 
1126     ret = allocate(h,
1127                    h1,
1128                    cookie,
1129                    key,
1130                    strlen(data),
1131                    0,
1132                    0,
1133                    PROTOCOL_BINARY_RAW_BYTES,
1134                    0);
1135     checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
1136 
1137     if (!h1->get_item_info(h, ret.second.get(), &info)) {
1138         abort();
1139     }
1140     memcpy(info.value[0].iov_base, data, strlen(data));
1141 
1142     cas = 0;
1143     // Add a new item with the same key.
1144     rv = h1->store(h,
1145                    cookie,
1146                    ret.second.get(),
1147                    cas,
1148                    OPERATION_ADD,
1149                    DocumentState::Alive);
1150     checkeq(ENGINE_SUCCESS, rv, "Add failed.");
1151     check_key_value(h, h1, key, data, strlen(data));
1152     ret.second.reset();
1153     wait_for_flusher_to_settle(h, h1);
1154 
1155     checkeq(cb::engine_errc::success,
1156             get(h, h1, cookie, key, 0).first,
1157             "Item shouldn't expire");
1158 
1159     // Restart the engine to ensure the above unexpired new item is loaded
1160     testHarness.reload_engine(&h, &h1,
1161                               testHarness.engine_path,
1162                               testHarness.get_current_testcase()->cfg,
1163                               true, false);
1164     wait_for_warmup_complete(h, h1);
1165     cb_assert(1 == get_int_stat(h, h1, "ep_warmup_value_count", "warmup"));
1166     cb_assert(0 == get_int_stat(h, h1, "ep_warmup_dups", "warmup"));
1167 
1168     testHarness.destroy_cookie(cookie);
1169     return SUCCESS;
1170 }
1171 
test_bug3522(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1172 static enum test_result test_bug3522(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1173     if (!isWarmupEnabled(h, h1)) {
1174         return SKIPPED;
1175     }
1176 
1177     const char *key = "test_expiry_no_items_warmup";
1178     const char *data = "some test data here.";
1179 
1180     const void* cookie = testHarness.create_cookie();
1181     auto ret = allocate(h,
1182                         h1,
1183                         cookie,
1184                         key,
1185                         strlen(data),
1186                         0,
1187                         0,
1188                         PROTOCOL_BINARY_RAW_BYTES,
1189                         0);
1190     checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
1191 
1192     item_info info;
1193     if (!h1->get_item_info(h, ret.second.get(), &info)) {
1194         abort();
1195     }
1196     memcpy(info.value[0].iov_base, data, strlen(data));
1197 
1198     uint64_t cas = 0;
1199     auto rv = h1->store(h,
1200                         cookie,
1201                         ret.second.get(),
1202                         cas,
1203                         OPERATION_SET,
1204                         DocumentState::Alive);
1205     checkeq(ENGINE_SUCCESS, rv, "Set failed.");
1206     check_key_value(h, h1, key, data, strlen(data));
1207     wait_for_flusher_to_settle(h, h1);
1208 
1209     // Add a new item with the same key and 2 sec of expiration.
1210     const char *new_data = "new data here.";
1211     ret = allocate(h,
1212                    h1,
1213                    cookie,
1214                    key,
1215                    strlen(new_data),
1216                    0,
1217                    2,
1218                    PROTOCOL_BINARY_RAW_BYTES,
1219                    0);
1220     checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
1221 
1222     if (!h1->get_item_info(h, ret.second.get(), &info)) {
1223         abort();
1224     }
1225     memcpy(info.value[0].iov_base, new_data, strlen(new_data));
1226 
1227     int pager_runs = get_int_stat(h, h1, "ep_num_expiry_pager_runs");
1228     cas = 0;
1229     rv = h1->store(h,
1230                    cookie,
1231                    ret.second.get(),
1232                    cas,
1233                    OPERATION_SET,
1234                    DocumentState::Alive);
1235     checkeq(ENGINE_SUCCESS, rv, "Set failed.");
1236     check_key_value(h, h1, key, new_data, strlen(new_data));
1237     ret.second.reset();
1238     testHarness.time_travel(3);
1239     wait_for_stat_change(h, h1, "ep_num_expiry_pager_runs", pager_runs);
1240     wait_for_flusher_to_settle(h, h1);
1241 
1242     // Restart the engine.
1243     testHarness.reload_engine(&h, &h1,
1244                               testHarness.engine_path,
1245                               testHarness.get_current_testcase()->cfg,
1246                               true, false);
1247     wait_for_warmup_complete(h, h1);
1248     // TODO: modify this for a better test case
1249     cb_assert(0 == get_int_stat(h, h1, "ep_warmup_dups", "warmup"));
1250 
1251     testHarness.destroy_cookie(cookie);
1252     return SUCCESS;
1253 }
1254 
test_get_replica_active_state(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1255 static enum test_result test_get_replica_active_state(ENGINE_HANDLE *h,
1256                                                       ENGINE_HANDLE_V1 *h1) {
1257     protocol_binary_request_header *pkt;
1258     pkt = prepare_get_replica(h, h1, vbucket_state_active);
1259     checkeq(ENGINE_NOT_MY_VBUCKET,
1260             h1->unknown_command(
1261                     h, NULL, pkt, add_response, testHarness.doc_namespace),
1262             "Get Replica Failed");
1263 
1264     cb_free(pkt);
1265     return SUCCESS;
1266 }
1267 
test_get_replica_pending_state(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1268 static enum test_result test_get_replica_pending_state(ENGINE_HANDLE *h,
1269                                                        ENGINE_HANDLE_V1 *h1) {
1270     protocol_binary_request_header *pkt;
1271 
1272     const void *cookie = testHarness.create_cookie();
1273     testHarness.set_ewouldblock_handling(cookie, false);
1274     pkt = prepare_get_replica(h, h1, vbucket_state_pending);
1275     checkeq(ENGINE_EWOULDBLOCK,
1276             h1->unknown_command(h, cookie, pkt, add_response, testHarness.doc_namespace),
1277             "Should have returned error for pending state");
1278     testHarness.destroy_cookie(cookie);
1279     cb_free(pkt);
1280     return SUCCESS;
1281 }
1282 
test_get_replica_dead_state(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1283 static enum test_result test_get_replica_dead_state(ENGINE_HANDLE *h,
1284                                                     ENGINE_HANDLE_V1 *h1) {
1285     protocol_binary_request_header *pkt;
1286     pkt = prepare_get_replica(h, h1, vbucket_state_dead);
1287     checkeq(ENGINE_NOT_MY_VBUCKET,
1288             h1->unknown_command(
1289                     h, NULL, pkt, add_response, testHarness.doc_namespace),
1290             "Get Replica Failed");
1291     cb_free(pkt);
1292     return SUCCESS;
1293 }
1294 
test_get_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1295 static enum test_result test_get_replica(ENGINE_HANDLE *h,
1296                                          ENGINE_HANDLE_V1 *h1) {
1297     protocol_binary_request_header *pkt;
1298     pkt = prepare_get_replica(h, h1, vbucket_state_replica);
1299     checkeq(ENGINE_SUCCESS,
1300             h1->unknown_command(h, NULL, pkt, add_response, testHarness.doc_namespace),
1301             "Get Replica Failed");
1302     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1303             "Expected PROTOCOL_BINARY_RESPONSE_SUCCESS response.");
1304     checkeq(std::string("replicadata"), last_body,
1305             "Should have returned identical value");
1306 
1307     cb_free(pkt);
1308     return SUCCESS;
1309 }
1310 
test_get_replica_non_resident(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1311 static enum test_result test_get_replica_non_resident(ENGINE_HANDLE *h,
1312                                                       ENGINE_HANDLE_V1 *h1) {
1313     checkeq(ENGINE_SUCCESS,
1314             store(h, h1, NULL, OPERATION_SET, "key", "value"),
1315             "Store Failed");
1316     wait_for_flusher_to_settle(h, h1);
1317     wait_for_stat_to_be(h, h1, "ep_total_persisted", 1);
1318 
1319     evict_key(h, h1, "key", 0, "Ejected.");
1320     check(set_vbucket_state(h, h1, 0, vbucket_state_replica),
1321           "Failed to set vbucket to replica");
1322 
1323     get_replica(h, h1, "key", 0);
1324     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1325             "Expected success");
1326 
1327     return SUCCESS;
1328 }
1329 
test_get_replica_invalid_key(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1330 static enum test_result test_get_replica_invalid_key(ENGINE_HANDLE *h,
1331                                                      ENGINE_HANDLE_V1 *h1) {
1332     protocol_binary_request_header *pkt;
1333     bool makeinvalidkey = true;
1334     pkt = prepare_get_replica(h, h1, vbucket_state_replica, makeinvalidkey);
1335     checkeq(ENGINE_NOT_MY_VBUCKET,
1336             h1->unknown_command(
1337                     h, NULL, pkt, add_response, testHarness.doc_namespace),
1338             "Get Replica Failed");
1339     cb_free(pkt);
1340     return SUCCESS;
1341 }
1342 
test_vb_del_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1343 static enum test_result test_vb_del_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1344     const void *cookie = testHarness.create_cookie();
1345     testHarness.set_ewouldblock_handling(cookie, false);
1346     check(set_vbucket_state(h, h1, 1, vbucket_state_pending),
1347           "Failed to set vbucket state.");
1348     checkeq(ENGINE_EWOULDBLOCK, del(h, h1, "key", 0, 1, cookie),
1349             "Expected woodblock.");
1350     testHarness.destroy_cookie(cookie);
1351     return SUCCESS;
1352 }
1353 
test_vb_del_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1354 static enum test_result test_vb_del_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1355     check(set_vbucket_state(h, h1, 1, vbucket_state_replica),
1356           "Failed to set vbucket state.");
1357     int numNotMyVBucket = get_int_stat(h, h1, "ep_num_not_my_vbuckets");
1358     checkeq(ENGINE_NOT_MY_VBUCKET, del(h, h1, "key", 0, 1),
1359             "Expected not my vbucket.");
1360     wait_for_stat_change(h, h1, "ep_num_not_my_vbuckets", numNotMyVBucket);
1361     return SUCCESS;
1362 }
1363 
test_vbucket_get_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1364 static enum test_result test_vbucket_get_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1365     return verify_vbucket_missing(h, h1, 1) ? SUCCESS : FAIL;
1366 }
1367 
test_vbucket_get(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1368 static enum test_result test_vbucket_get(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1369     return verify_vbucket_state(h, h1, 0, vbucket_state_active) ? SUCCESS : FAIL;
1370 }
1371 
test_vbucket_create(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1372 static enum test_result test_vbucket_create(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1373     if (!verify_vbucket_missing(h, h1, 1)) {
1374         fprintf(stderr, "vbucket wasn't missing.\n");
1375         return FAIL;
1376     }
1377 
1378     if (!set_vbucket_state(h, h1, 1, vbucket_state_active)) {
1379         fprintf(stderr, "set state failed.\n");
1380         return FAIL;
1381     }
1382 
1383     return verify_vbucket_state(h, h1, 1, vbucket_state_active) ? SUCCESS : FAIL;
1384 }
1385 
test_takeover_stats_race_with_vb_create_DCP( ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1386 static enum test_result test_takeover_stats_race_with_vb_create_DCP(
1387                                     ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1388     check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1389           "Failed to set vbucket state information");
1390 
1391     checkeq(0,
1392             get_int_stat(h, h1, "on_disk_deletes", "dcp-vbtakeover 1"),
1393             "Invalid number of on-disk deletes");
1394 
1395     return SUCCESS;
1396 }
1397 
test_takeover_stats_num_persisted_deletes( ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1398 static enum test_result test_takeover_stats_num_persisted_deletes(
1399                                     ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1400     /* set an item */
1401     std::string key("key");
1402     checkeq(ENGINE_SUCCESS,
1403             store(h, h1, NULL, OPERATION_SET, key.c_str(), "data"),
1404             "Failed to store an item");
1405 
1406     /* delete the item */
1407     checkeq(ENGINE_SUCCESS, del(h, h1, key.c_str(), 0, 0),
1408             "Failed to delete the item");
1409 
1410     /* wait for persistence */
1411     wait_for_flusher_to_settle(h, h1);
1412 
1413     /* check if persisted deletes stats is got correctly */
1414     checkeq(1,
1415             get_int_stat(h, h1, "on_disk_deletes", "dcp-vbtakeover 0"),
1416             "Invalid number of on-disk deletes");
1417 
1418     return SUCCESS;
1419 }
1420 
test_vbucket_compact(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1421 static enum test_result test_vbucket_compact(ENGINE_HANDLE *h,
1422                                              ENGINE_HANDLE_V1 *h1) {
1423     const char* exp_key = "Carss";
1424     const char* exp_value = "pollute";
1425     const char* non_exp_key = "trees";
1426     const char* non_exp_value = "cleanse";
1427 
1428     // Set two keys - one to be expired and other to remain...
1429     // Set expiring key
1430     checkeq(ENGINE_SUCCESS,
1431             store(h, h1, NULL, OPERATION_SET, exp_key, exp_value),
1432             "Failed to set expiring key");
1433     check_key_value(h, h1, exp_key, exp_value, strlen(exp_value));
1434 
1435     // Set a non-expiring key...
1436     checkeq(ENGINE_SUCCESS,
1437             store(h, h1, NULL, OPERATION_SET, non_exp_key, non_exp_value),
1438             "Failed to set non-expiring key");
1439     check_key_value(h, h1, non_exp_key, non_exp_value, strlen(non_exp_value));
1440 
1441     // Touch expiring key with an expire time
1442     const int exp_time = 11;
1443     checkeq(ENGINE_SUCCESS,
1444             touch(h, h1, exp_key, 0, exp_time),
1445             "Touch expiring key failed");
1446 
1447     // Move beyond expire time
1448     testHarness.time_travel(exp_time + 1);
1449 
1450     // Wait for the item to be expired
1451     wait_for_stat_to_be(h, h1, "vb_active_expired", 1);
1452     const int exp_purge_seqno =
1453             get_int_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno");
1454 
1455     // non_exp_key and its value should be intact...
1456     checkeq(ENGINE_SUCCESS,
1457             verify_key(h, h1, non_exp_key),
1458             "key trees should be found.");
1459     // exp_key should have disappeared...
1460     ENGINE_ERROR_CODE val = verify_key(h, h1, exp_key);
1461     checkeq(ENGINE_KEY_ENOENT, val, "Key Carss has not expired.");
1462 
1463     // Store a dummy item since we do not purge the item with highest seqno
1464     checkeq(ENGINE_SUCCESS,
1465             store(h, h1, NULL, OPERATION_SET, "dummykey", "dummyvalue"),
1466             "Error setting dummy key");
1467     wait_for_flusher_to_settle(h, h1);
1468 
1469     checkeq(0, get_int_stat(h, h1, "vb_0:purge_seqno", "vbucket-seqno"),
1470             "purge_seqno not found to be zero before compaction");
1471 
1472     // Compaction on VBucket
1473     compact_db(
1474             h,
1475             h1,
1476             0 /* vbucket_id */,
1477             0 /* db_file_id */,
1478             2 /* purge_before_ts */,
1479             exp_purge_seqno - 1 /* purge_before_seq */,
1480             1 /* drop deletes (forces purge irrespective purge_before_seq) */);
1481     wait_for_stat_to_be(h, h1, "ep_pending_compactions", 0);
1482     checkeq(exp_purge_seqno,
1483             get_int_stat(h, h1, "vb_0:purge_seqno", "vbucket-seqno"),
1484             "purge_seqno didn't match expected value");
1485 
1486     return SUCCESS;
1487 }
1488 
test_compaction_config(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1489 static enum test_result test_compaction_config(ENGINE_HANDLE *h,
1490                                                ENGINE_HANDLE_V1 *h1) {
1491 
1492     checkeq(10000,
1493             get_int_stat(h, h1, "ep_compaction_write_queue_cap"),
1494             "Expected compaction queue cap to be 10000");
1495     set_param(h, h1, protocol_binary_engine_param_flush,
1496               "compaction_write_queue_cap", "100000");
1497     checkeq(100000, get_int_stat(h, h1, "ep_compaction_write_queue_cap"),
1498             "Expected compaction queue cap to be 100000");
1499     return SUCCESS;
1500 }
1501 
1502 struct comp_thread_ctx {
1503     ENGINE_HANDLE *h;
1504     ENGINE_HANDLE_V1 *h1;
1505     uint16_t vbid;
1506     uint16_t db_file_id;
1507 };
1508 
1509 extern "C" {
compaction_thread(void *arg)1510     static void compaction_thread(void *arg) {
1511         struct comp_thread_ctx *ctx = static_cast<comp_thread_ctx *>(arg);
1512         compact_db(ctx->h, ctx->h1, ctx->vbid, ctx->db_file_id, 0, 0, 0);
1513     }
1514 }
1515 
test_multiple_vb_compactions(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1516 static enum test_result test_multiple_vb_compactions(ENGINE_HANDLE *h,
1517                                                      ENGINE_HANDLE_V1 *h1) {
1518     for (uint16_t i = 0; i < 4; ++i) {
1519         if (!set_vbucket_state(h, h1, i, vbucket_state_active)) {
1520             fprintf(stderr, "set state failed for vbucket %d.\n", i);
1521             return FAIL;
1522         }
1523         check(verify_vbucket_state(h, h1, i, vbucket_state_active),
1524               "VBucket state not active");
1525     }
1526 
1527     std::vector<std::string> keys;
1528     for (int j = 0; j < 20000; ++j) {
1529         std::stringstream ss;
1530         ss << "key" << j;
1531         std::string key(ss.str());
1532         keys.push_back(key);
1533     }
1534 
1535     int count = 0;
1536     std::vector<std::string>::iterator it;
1537     for (it = keys.begin(); it != keys.end(); ++it) {
1538         uint16_t vbid = count % 4;
1539         checkeq(ENGINE_SUCCESS,
1540                 store(h,
1541                       h1,
1542                       NULL,
1543                       OPERATION_SET,
1544                       it->c_str(),
1545                       it->c_str(),
1546                       nullptr,
1547                       0,
1548                       vbid),
1549                 "Failed to store a value");
1550         ++count;
1551     }
1552 
1553     // Compact multiple vbuckets.
1554     const int n_threads = 4;
1555     cb_thread_t threads[n_threads];
1556     struct comp_thread_ctx ctx[n_threads];
1557 
1558     const int num_shards = get_int_stat(h, h1, "ep_workload:num_shards",
1559                                         "workload");
1560 
1561     for (int i = 0; i < n_threads; i++) {
1562         ctx[i].h = h;
1563         ctx[i].h1 = h1;
1564         ctx[i].vbid = static_cast<uint16_t>(i);
1565         ctx[i].db_file_id = ctx[i].vbid % num_shards;
1566         int r = cb_create_thread(&threads[i], compaction_thread, &ctx[i], 0);
1567         cb_assert(r == 0);
1568     }
1569 
1570     for (int i = 0; i < n_threads; i++) {
1571         int r = cb_join_thread(threads[i]);
1572         cb_assert(r == 0);
1573     }
1574 
1575     wait_for_stat_to_be(h, h1, "ep_pending_compactions", 0);
1576 
1577     return SUCCESS;
1578 }
1579 
1580 static enum test_result
test_multi_vb_compactions_with_workload(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1581 test_multi_vb_compactions_with_workload(ENGINE_HANDLE *h,
1582                                         ENGINE_HANDLE_V1 *h1) {
1583     for (uint16_t i = 0; i < 4; ++i) {
1584         if (!set_vbucket_state(h, h1, i, vbucket_state_active)) {
1585             fprintf(stderr, "set state failed for vbucket %d.\n", i);
1586             return FAIL;
1587         }
1588         check(verify_vbucket_state(h, h1, i, vbucket_state_active),
1589               "VBucket state not active");
1590     }
1591 
1592     std::vector<std::string> keys;
1593     for (int j = 0; j < 10000; ++j) {
1594         std::stringstream ss;
1595         ss << "key" << j;
1596         std::string key(ss.str());
1597         keys.push_back(key);
1598     }
1599 
1600     int count = 0;
1601     std::vector<std::string>::iterator it;
1602     for (it = keys.begin(); it != keys.end(); ++it) {
1603         uint16_t vbid = count % 4;
1604         checkeq(ENGINE_SUCCESS,
1605                 store(h,
1606                       h1,
1607                       NULL,
1608                       OPERATION_SET,
1609                       it->c_str(),
1610                       it->c_str(),
1611                       nullptr,
1612                       0,
1613                       vbid),
1614                 "Failed to store a value");
1615         ++count;
1616     }
1617     wait_for_flusher_to_settle(h, h1);
1618 
1619     for (int i = 0; i < 2; ++i) {
1620         count = 0;
1621         for (it = keys.begin(); it != keys.end(); ++it) {
1622             uint16_t vbid = count % 4;
1623             checkeq(cb::engine_errc::success,
1624                     get(h, h1, NULL, it->c_str(), vbid).first,
1625                     "Unable to get stored item");
1626             ++count;
1627         }
1628     }
1629     wait_for_stat_to_be(h, h1, "ep_workload_pattern", std::string{"read_heavy"});
1630 
1631     // Compact multiple vbuckets.
1632     const int n_threads = 4;
1633     cb_thread_t threads[n_threads];
1634     struct comp_thread_ctx ctx[n_threads];
1635 
1636     for (int i = 0; i < n_threads; i++) {
1637         ctx[i].h = h;
1638         ctx[i].h1 = h1;
1639         ctx[i].vbid = static_cast<uint16_t>(i);
1640         int r = cb_create_thread(&threads[i], compaction_thread, &ctx[i], 0);
1641         cb_assert(r == 0);
1642     }
1643 
1644     for (int i = 0; i < n_threads; i++) {
1645         int r = cb_join_thread(threads[i]);
1646         cb_assert(r == 0);
1647     }
1648 
1649     wait_for_stat_to_be(h, h1, "ep_pending_compactions", 0);
1650 
1651     return SUCCESS;
1652 }
1653 
vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char* value = NULL)1654 static enum test_result vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1655                                              const char* value = NULL) {
1656     check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1657           "Failed to set vbucket state.");
1658 
1659     checkeq(ENGINE_NOT_MY_VBUCKET,
1660             vbucketDelete(h, h1, 2, value),
1661             "Expected NMVB");
1662 
1663     check(set_vbucket_state(h, h1, 1, vbucket_state_dead),
1664           "Failed set set vbucket 1 state.");
1665 
1666     checkeq(ENGINE_SUCCESS, vbucketDelete(h, h1, 1, value), "Expected success");
1667     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1668             "Expected failure deleting non-existent bucket.");
1669 
1670     check(verify_vbucket_missing(h, h1, 1),
1671           "vbucket 0 was not missing after deleting it.");
1672 
1673     return SUCCESS;
1674 }
1675 
test_vbucket_destroy_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1676 static enum test_result test_vbucket_destroy_stats(ENGINE_HANDLE *h,
1677                                                    ENGINE_HANDLE_V1 *h1) {
1678 
1679     int cacheSize = get_int_stat(h, h1, "ep_total_cache_size");
1680     int overhead = get_int_stat(h, h1, "ep_overhead");
1681     int nonResident = get_int_stat(h, h1, "ep_num_non_resident");
1682 
1683     check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1684           "Failed to set vbucket state.");
1685 
1686     std::vector<std::string> keys;
1687     for (int j = 0; j < 2000; ++j) {
1688         std::stringstream ss;
1689         ss << "key" << j;
1690         std::string key(ss.str());
1691         keys.push_back(key);
1692     }
1693 
1694     int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
1695     std::vector<std::string>::iterator it;
1696     for (it = keys.begin(); it != keys.end(); ++it) {
1697         checkeq(ENGINE_SUCCESS,
1698                 store(h,
1699                       h1,
1700                       NULL,
1701                       OPERATION_SET,
1702                       it->c_str(),
1703                       it->c_str(),
1704                       nullptr,
1705                       0,
1706                       1),
1707                 "Failed to store a value");
1708     }
1709     wait_for_flusher_to_settle(h, h1);
1710     testHarness.time_travel(65);
1711     wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
1712 
1713     check(set_vbucket_state(h, h1, 1, vbucket_state_dead),
1714           "Failed set set vbucket 1 state.");
1715 
1716     int vbucketDel = get_int_stat(h, h1, "ep_vbucket_del");
1717     checkeq(ENGINE_SUCCESS, vbucketDelete(h, h1, 1), "Expected success");
1718     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS,
1719             last_status.load(),
1720             "Expected failure deleting non-existent bucket.");
1721 
1722     check(verify_vbucket_missing(h, h1, 1),
1723           "vbucket 1 was not missing after deleting it.");
1724 
1725     wait_for_stat_change(h, h1, "ep_vbucket_del", vbucketDel);
1726 
1727     wait_for_stat_to_be(h, h1, "ep_total_cache_size", cacheSize);
1728     wait_for_stat_to_be(h, h1, "ep_overhead", overhead);
1729     wait_for_stat_to_be(h, h1, "ep_num_non_resident", nonResident);
1730 
1731     return SUCCESS;
1732 }
1733 
vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char* value = NULL)1734 static enum test_result vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1735                                                 const char* value = NULL) {
1736     if (!isWarmupEnabled(h, h1)) {
1737         return SKIPPED;
1738     }
1739 
1740     check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1741           "Failed to set vbucket state.");
1742 
1743     // Store a value so the restart will try to resurrect it.
1744     checkeq(ENGINE_SUCCESS,
1745             store(h,
1746                   h1,
1747                   NULL,
1748                   OPERATION_SET,
1749                   "key",
1750                   "somevalue",
1751                   nullptr,
1752                   0,
1753                   1),
1754             "Failed to set a value");
1755     check_key_value(h, h1, "key", "somevalue", 9, 1);
1756 
1757     // Reload to get a flush forced.
1758     testHarness.reload_engine(&h, &h1,
1759                               testHarness.engine_path,
1760                               testHarness.get_current_testcase()->cfg,
1761                               true, false);
1762     wait_for_warmup_complete(h, h1);
1763 
1764     check(verify_vbucket_state(h, h1, 1, vbucket_state_active),
1765           "Bucket state was what it was initially, after restart.");
1766     check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1767           "Failed to set vbucket state.");
1768     check_key_value(h, h1, "key", "somevalue", 9, 1);
1769 
1770     check(set_vbucket_state(h, h1, 1, vbucket_state_dead),
1771           "Failed set set vbucket 1 state.");
1772 
1773     checkeq(ENGINE_SUCCESS, vbucketDelete(h, h1, 1, value), "Expected success");
1774     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1775             "Expected failure deleting non-existent bucket.");
1776 
1777     check(verify_vbucket_missing(h, h1, 1),
1778           "vbucket 1 was not missing after deleting it.");
1779 
1780     testHarness.reload_engine(&h, &h1,
1781                               testHarness.engine_path,
1782                               testHarness.get_current_testcase()->cfg,
1783                               true, false);
1784     wait_for_warmup_complete(h, h1);
1785 
1786     if (verify_vbucket_state(h, h1, 1, vbucket_state_pending, true)) {
1787         std::cerr << "Bucket came up in pending state after delete." << std::endl;
1788         abort();
1789     }
1790 
1791     check(verify_vbucket_missing(h, h1, 1),
1792           "vbucket 1 was not missing after restart.");
1793 
1794     return SUCCESS;
1795 }
1796 
test_async_vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1797 static enum test_result test_async_vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1798     return vbucket_destroy(h, h1);
1799 }
1800 
test_sync_vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1801 static enum test_result test_sync_vbucket_destroy(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1802     return vbucket_destroy(h, h1, "async=0");
1803 }
1804 
test_async_vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1805 static enum test_result test_async_vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1806     return vbucket_destroy_restart(h, h1);
1807 }
1808 
test_sync_vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1809 static enum test_result test_sync_vbucket_destroy_restart(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1810     return vbucket_destroy_restart(h, h1, "async=0");
1811 }
1812 
test_vb_set_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1813 static enum test_result test_vb_set_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1814     return test_pending_vb_mutation(h, h1, OPERATION_SET);
1815 }
1816 
test_vb_add_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1817 static enum test_result test_vb_add_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1818     return test_pending_vb_mutation(h, h1, OPERATION_ADD);
1819 }
1820 
test_vb_cas_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1821 static enum test_result test_vb_cas_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1822     return test_pending_vb_mutation(h, h1, OPERATION_CAS);
1823 }
1824 
test_vb_set_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1825 static enum test_result test_vb_set_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1826     return test_replica_vb_mutation(h, h1, OPERATION_SET);
1827 }
1828 
test_vb_replace_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1829 static enum test_result test_vb_replace_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1830     return test_replica_vb_mutation(h, h1, OPERATION_REPLACE);
1831 }
1832 
test_vb_replace_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1833 static enum test_result test_vb_replace_pending(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1834     return test_pending_vb_mutation(h, h1, OPERATION_REPLACE);
1835 }
1836 
test_vb_add_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1837 static enum test_result test_vb_add_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1838     return test_replica_vb_mutation(h, h1, OPERATION_ADD);
1839 }
1840 
test_vb_cas_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1841 static enum test_result test_vb_cas_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1842     return test_replica_vb_mutation(h, h1, OPERATION_CAS);
1843 }
1844 
test_stats_seqno(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1845 static enum test_result test_stats_seqno(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1846     check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1847           "Failed to set vbucket state.");
1848 
1849     int num_keys = 100;
1850     for (int ii = 0; ii < num_keys; ++ii) {
1851         std::stringstream ss;
1852         ss << "key" << ii;
1853         checkeq(ENGINE_SUCCESS,
1854                 store(h, h1, NULL, OPERATION_SET, ss.str().c_str(),
1855                       "value", NULL, 0, 0),
1856                 "Failed to store an item.");
1857     }
1858     wait_for_flusher_to_settle(h, h1);
1859 
1860     checkeq(100, get_int_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno"),
1861             "Invalid seqno");
1862 
1863     if (isPersistentBucket(h, h1)) {
1864         checkeq(100,
1865                 get_int_stat(h, h1, "vb_0:last_persisted_seqno", "vbucket-seqno"),
1866                 "Unexpected last_persisted_seqno");
1867     }
1868     checkeq(0, get_int_stat(h, h1, "vb_1:high_seqno", "vbucket-seqno"),
1869             "Invalid seqno");
1870     checkeq(0, get_int_stat(h, h1, "vb_1:high_seqno", "vbucket-seqno 1"),
1871             "Invalid seqno");
1872     if (isPersistentBucket(h, h1)) {
1873         checkeq(0,
1874                 get_int_stat(h, h1, "vb_1:last_persisted_seqno", "vbucket-seqno 1"),
1875                 "Invalid last_persisted_seqno");
1876     }
1877 
1878     uint64_t vb_uuid = get_ull_stat(h, h1, "vb_1:0:id", "failovers");
1879 
1880     auto seqno_stats = get_all_stats(h, h1, "vbucket-seqno 1");
1881     checkeq(vb_uuid, uint64_t(std::stoull(seqno_stats.at("vb_1:uuid"))),
1882             "Invalid uuid");
1883 
1884     checkeq(size_t(7), seqno_stats.size(), "Expected seven stats");
1885 
1886     // Check invalid vbucket
1887     checkeq(ENGINE_NOT_MY_VBUCKET,
1888             get_stats(h, "vbucket-seqno 2"_ccb, add_stats),
1889             "Expected not my vbucket");
1890 
1891     // Check bad vbucket parameter (not numeric)
1892     checkeq(ENGINE_EINVAL,
1893             get_stats(h, "vbucket-seqno tt2"_ccb, add_stats),
1894             "Expected invalid");
1895 
1896     // Check extra spaces at the end
1897     checkeq(ENGINE_EINVAL,
1898             get_stats(h, "vbucket-seqno    "_ccb, add_stats),
1899             "Expected invalid");
1900 
1901     return SUCCESS;
1902 }
1903 
test_stats_diskinfo(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1904 static enum test_result test_stats_diskinfo(ENGINE_HANDLE *h,
1905                                             ENGINE_HANDLE_V1 *h1) {
1906     check(set_vbucket_state(h, h1, 1, vbucket_state_active),
1907           "Failed to set vbucket state.");
1908 
1909     int num_keys = 100;
1910     for (int ii = 0; ii < num_keys; ++ii) {
1911         std::stringstream ss;
1912         ss << "key" << ii;
1913         checkeq(ENGINE_SUCCESS,
1914                 store(h, h1, NULL, OPERATION_SET, ss.str().c_str(),
1915                       "value", NULL, 0, 1),
1916                 "Failed to store an item.");
1917     }
1918     wait_for_flusher_to_settle(h, h1);
1919 
1920     size_t file_size = get_int_stat(h, h1, "ep_db_file_size", "diskinfo");
1921     size_t data_size = get_int_stat(h, h1, "ep_db_data_size", "diskinfo");
1922     check(file_size > 0, "DB file size should be greater than 0");
1923     check(data_size > 0, "DB data size should be greater than 0");
1924     check(file_size >= data_size, "DB file size should be >= DB data size");
1925     check(get_int_stat(h, h1, "vb_1:data_size", "diskinfo detail") > 0,
1926           "VB 1 data size should be greater than 0");
1927 
1928     checkeq(ENGINE_EINVAL,
1929             get_stats(h, "diskinfo "_ccb, add_stats),
1930             "Expected invalid");
1931 
1932     checkeq(ENGINE_EINVAL,
1933             get_stats(h, "diskinfo detai"_ccb, add_stats),
1934             "Expected invalid");
1935 
1936     checkeq(ENGINE_EINVAL,
1937             get_stats(h, "diskinfo detaillll"_ccb, add_stats),
1938             "Expected invalid");
1939 
1940     return SUCCESS;
1941 }
1942 
test_uuid_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1943 static enum test_result test_uuid_stats(ENGINE_HANDLE *h,
1944                                         ENGINE_HANDLE_V1 *h1)
1945 {
1946     vals.clear();
1947     checkeq(ENGINE_SUCCESS,
1948             get_stats(h, "uuid"_ccb, add_stats),
1949             "Failed to get stats.");
1950     check(vals["uuid"] == "foobar", "Incorrect uuid");
1951     return SUCCESS;
1952 }
1953 
test_item_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1954 static enum test_result test_item_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1955     checkeq(ENGINE_SUCCESS,
1956             store(h, h1, NULL, OPERATION_SET, "key", "somevalue"),
1957             "Failed set.");
1958     wait_for_flusher_to_settle(h, h1);
1959     checkeq(ENGINE_SUCCESS,
1960             store(h, h1, NULL, OPERATION_SET, "key", "somevalueX"),
1961             "Failed set.");
1962     wait_for_flusher_to_settle(h, h1);
1963     checkeq(ENGINE_SUCCESS,
1964             store(h, h1, NULL, OPERATION_SET, "key1", "somevalueY"),
1965             "Failed set.");
1966     wait_for_flusher_to_settle(h, h1);
1967 
1968     check_key_value(h, h1, "key", "somevalueX", 10);
1969     check_key_value(h, h1, "key1", "somevalueY", 10);
1970 
1971     checkeq(ENGINE_SUCCESS, del(h, h1, "key1", 0, 0),
1972             "Failed remove with value.");
1973     wait_for_flusher_to_settle(h, h1);
1974 
1975     checkeq(ENGINE_SUCCESS,
1976             store(h, h1, NULL, OPERATION_SET, "key1", "someothervalue"),
1977             "Failed set.");
1978     wait_for_flusher_to_settle(h, h1);
1979 
1980     check_key_value(h, h1, "key1", "someothervalue", 14);
1981 
1982     checkeq(3,
1983             get_int_stat(h, h1, "vb_active_ops_create"),
1984             "Expected 3 creations");
1985     checkeq(1,
1986             get_int_stat(h, h1, "vb_active_ops_update"),
1987             "Expected 1 updation");
1988     checkeq(1,
1989             get_int_stat(h, h1, "vb_active_ops_delete"),
1990             "Expected 1 deletion");
1991 
1992     return SUCCESS;
1993 }
1994 
test_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1995 static enum test_result test_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1996     vals.clear();
1997     checkeq(ENGINE_SUCCESS,
1998             get_stats(h, {}, add_stats),
1999             "Failed to get stats.");
2000     check(vals.size() > 10, "Kind of expected more stats than that.");
2001 
2002     return SUCCESS;
2003 }
2004 
test_mem_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2005 static enum test_result test_mem_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2006     char value[2048];
2007     memset(value, 'b', sizeof(value));
2008     strcpy(value + sizeof(value) - 4, "\r\n");
2009     int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
2010     wait_for_persisted_value(h, h1, "key", value);
2011     testHarness.time_travel(65);
2012     if (isPersistentBucket(h, h1)) {
2013         wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
2014     }
2015 
2016     if (isActiveCompressionEnabled(h, h1)) {
2017         wait_for_item_compressor_to_settle(h, h1);
2018     }
2019 
2020     int mem_used = get_int_stat(h, h1, "mem_used");
2021     int cache_size = get_int_stat(h, h1, "ep_total_cache_size");
2022     int overhead = get_int_stat(h, h1, "ep_overhead");
2023     int value_size = get_int_stat(h, h1, "ep_value_size");
2024     check((mem_used - overhead) > cache_size,
2025           "ep_kv_size should be greater than the hashtable cache size due to "
2026           "the checkpoint overhead");
2027 
2028     if (isPersistentBucket(h, h1)) {
2029         evict_key(h, h1, "key", 0, "Ejected.");
2030 
2031         check(get_int_stat(h, h1, "ep_total_cache_size") <= cache_size,
2032               "Evict a value shouldn't increase the total cache size");
2033         check(get_int_stat(h, h1, "mem_used") < mem_used,
2034               "Expected mem_used to decrease when an item is evicted");
2035 
2036         check_key_value(h, h1, "key", value, strlen(value), 0); // Load an item from disk again.
2037 
2038         if (isActiveCompressionEnabled(h, h1)) {
2039             wait_for_item_compressor_to_settle(h, h1);
2040         }
2041 
2042         check(get_int_stat(h, h1, "mem_used") >= mem_used,
2043               "Expected mem_used to remain the same after an item is loaded from disk");
2044         check(get_int_stat(h, h1, "ep_value_size") == value_size,
2045               "Expected ep_value_size to remain the same after item is "
2046               "loaded from disk");
2047     }
2048 
2049     return SUCCESS;
2050 }
2051 
test_io_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2052 static enum test_result test_io_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2053     int exp_write_bytes;
2054     std::string backend = get_str_stat(h, h1, "ep_backend");
2055     if (backend == "couchdb") {
2056         exp_write_bytes = 22; /* TBD: Do not hard code the value */
2057     } else if (backend == "rocksdb") {
2058         // TODO RDB:
2059         return SKIPPED_UNDER_ROCKSDB;
2060     }
2061     else {
2062         return SKIPPED;
2063     }
2064 
2065     reset_stats(h);
2066 
2067     checkeq(0,
2068             get_int_stat(h, h1, "rw_0:io_bg_fetch_docs_read", "kvstore"),
2069             "Expected reset stats to set io_bg_fetch_docs_read to zero");
2070     checkeq(0, get_int_stat(h, h1, "rw_0:io_num_write", "kvstore"),
2071             "Expected reset stats to set io_num_write to zero");
2072     checkeq(0,
2073             get_int_stat(h, h1, "rw_0:io_bg_fetch_doc_bytes", "kvstore"),
2074             "Expected reset stats to set io_bg_fetch_doc_bytes to zero");
2075     checkeq(0, get_int_stat(h, h1, "rw_0:io_write_bytes", "kvstore"),
2076             "Expected reset stats to set io_write_bytes to zero");
2077 
2078     const std::string key("a");
2079     const std::string value("b\r\n");
2080     wait_for_persisted_value(h, h1, key.c_str(), value.c_str());
2081     checkeq(0,
2082             get_int_stat(h, h1, "rw_0:io_bg_fetch_docs_read", "kvstore"),
2083             "Expected storing one value to not change the read counter");
2084     checkeq(0,
2085             get_int_stat(h, h1, "rw_0:io_bg_fetch_doc_bytes", "kvstore"),
2086             "Expected storing one value to not change the bgfetch doc bytes");
2087     checkeq(1, get_int_stat(h, h1, "rw_0:io_num_write", "kvstore"),
2088             "Expected storing the key to update the write counter");
2089     checkeq(exp_write_bytes,
2090             get_int_stat(h, h1, "rw_0:io_write_bytes", "kvstore"),
2091             "Expected storing the key to update the write bytes");
2092 
2093     evict_key(h, h1, key.c_str(), 0, "Ejected.");
2094 
2095     check_key_value(h, h1, "a", value.c_str(), value.size(), 0);
2096 
2097     std::stringstream numReadStatStr;
2098     std::stringstream readBytesStatStr;
2099 
2100     if (backend == "couchdb") {
2101         numReadStatStr << "ro_" << 0 << ":io_bg_fetch_docs_read";
2102         readBytesStatStr << "ro_" << 0 << ":io_bg_fetch_doc_bytes";
2103     } else {
2104         cb_assert(false);
2105     }
2106 
2107     checkeq(1,
2108             get_int_stat(h, h1, numReadStatStr.str().c_str(), "kvstore"),
2109             "Expected reading the value back in to update the read counter");
2110 
2111     const uint64_t exp_read_bytes =
2112             key.size() + value.size() +
2113             MetaData::getMetaDataSize(MetaData::Version::V1);
2114     checkeq(exp_read_bytes,
2115             get_stat<uint64_t>(
2116                     h, h1, readBytesStatStr.str().c_str(), "kvstore"),
2117             "Expected reading the value back in to update the read bytes");
2118 
2119     // For read amplification, exact value depends on couchstore file layout,
2120     // but generally see a value of 2 here.
2121     checkge(get_float_stat(h, h1, "ep_bg_fetch_avg_read_amplification"),
2122             2.0f,
2123             "Expected sensible bgFetch read amplification value");
2124 
2125     checkeq(1, get_int_stat(h, h1, "rw_0:io_num_write", "kvstore"),
2126             "Expected reading the value back in to not update the write counter");
2127     checkeq(exp_write_bytes,
2128             get_int_stat(h, h1, "rw_0:io_write_bytes", "kvstore"),
2129             "Expected reading the value back in to not update the write bytes");
2130 
2131     return SUCCESS;
2132 }
2133 
test_vb_file_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2134 static enum test_result test_vb_file_stats(ENGINE_HANDLE *h,
2135                                         ENGINE_HANDLE_V1 *h1) {
2136     wait_for_flusher_to_settle(h, h1);
2137     wait_for_stat_change(h, h1, "ep_db_data_size", 0);
2138 
2139     int old_data_size = get_int_stat(h, h1, "ep_db_data_size");
2140     int old_file_size = get_int_stat(h, h1, "ep_db_file_size");
2141     check(old_file_size != 0, "Expected a non-zero value for ep_db_file_size");
2142 
2143     // Write a value and test ...
2144     wait_for_persisted_value(h, h1, "a", "b\r\n");
2145     check(get_int_stat(h, h1, "ep_db_data_size") > old_data_size,
2146           "Expected the DB data size to increase");
2147     check(get_int_stat(h, h1, "ep_db_file_size") > old_file_size,
2148           "Expected the DB file size to increase");
2149 
2150     check(get_int_stat(h, h1, "vb_0:db_data_size", "vbucket-details 0") > 0,
2151           "Expected the vbucket DB data size to non-zero");
2152     check(get_int_stat(h, h1, "vb_0:db_file_size", "vbucket-details 0") > 0,
2153           "Expected the vbucket DB file size to non-zero");
2154     return SUCCESS;
2155 }
2156 
test_vb_file_stats_after_warmup(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2157 static enum test_result test_vb_file_stats_after_warmup(ENGINE_HANDLE *h,
2158                                                         ENGINE_HANDLE_V1 *h1) {
2159     if (!isWarmupEnabled(h, h1)) {
2160         return SKIPPED;
2161     }
2162 
2163     for (int i = 0; i < 100; ++i) {
2164         std::stringstream key;
2165         key << "key-" << i;
2166         checkeq(ENGINE_SUCCESS,
2167                 store(h,
2168                       h1,
2169                       NULL,
2170                       OPERATION_SET,
2171                       key.str().c_str(),
2172                       "somevalue"),
2173                 "Error setting.");
2174     }
2175     wait_for_flusher_to_settle(h, h1);
2176 
2177     int fileSize = get_int_stat(h, h1, "vb_0:db_file_size", "vbucket-details 0");
2178     int spaceUsed = get_int_stat(h, h1, "vb_0:db_data_size", "vbucket-details 0");
2179 
2180     // Restart the engine.
2181     testHarness.reload_engine(&h, &h1,
2182                               testHarness.engine_path,
2183                               testHarness.get_current_testcase()->cfg,
2184                               true, false);
2185     wait_for_warmup_complete(h, h1);
2186 
2187     int newFileSize = get_int_stat(h, h1, "vb_0:db_file_size", "vbucket-details 0");
2188     int newSpaceUsed = get_int_stat(h, h1, "vb_0:db_data_size", "vbucket-details 0");
2189 
2190     check((float)newFileSize >= 0.9 * fileSize, "Unexpected fileSize for vbucket");
2191     check((float)newSpaceUsed >= 0.9 * spaceUsed, "Unexpected spaceUsed for vbucket");
2192 
2193     return SUCCESS;
2194 }
2195 
test_bg_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2196 static enum test_result test_bg_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2197     reset_stats(h);
2198     wait_for_persisted_value(h, h1, "a", "b\r\n");
2199     evict_key(h, h1, "a", 0, "Ejected.");
2200     testHarness.time_travel(43);
2201     check_key_value(h, h1, "a", "b\r\n", 3, 0);
2202 
2203     auto stats = get_all_stats(h, h1);
2204     checkeq(1, std::stoi(stats.at("ep_bg_num_samples")),
2205                "Expected one sample");
2206 
2207     const char* bg_keys[] = { "ep_bg_min_wait",
2208                               "ep_bg_max_wait",
2209                               "ep_bg_wait_avg",
2210                               "ep_bg_min_load",
2211                               "ep_bg_max_load",
2212                               "ep_bg_load_avg"};
2213     for (const auto* key : bg_keys) {
2214         check(stats.find(key) != stats.end(),
2215               (std::string("Found no ") + key).c_str());
2216     }
2217 
2218     evict_key(h, h1, "a", 0, "Ejected.");
2219     check_key_value(h, h1, "a", "b\r\n", 3, 0);
2220     check(get_int_stat(h, h1, "ep_bg_num_samples") == 2,
2221           "Expected one sample");
2222 
2223     reset_stats(h);
2224     checkeq(0, get_int_stat(h, h1, "ep_bg_fetched"),
2225             "ep_bg_fetched is not reset to 0");
2226     return SUCCESS;
2227 }
2228 
test_bg_meta_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2229 static enum test_result test_bg_meta_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2230     reset_stats(h);
2231 
2232     wait_for_persisted_value(h, h1, "k1", "v1");
2233     wait_for_persisted_value(h, h1, "k2", "v2");
2234 
2235     evict_key(h, h1, "k1", 0, "Ejected.");
2236     checkeq(ENGINE_SUCCESS,
2237             del(h, h1, "k2", 0, 0), "Failed remove with value.");
2238     wait_for_flusher_to_settle(h, h1);
2239 
2240     checkeq(0, get_int_stat(h, h1, "ep_bg_fetched"), "Expected bg_fetched to be 0");
2241     checkeq(0, get_int_stat(h, h1, "ep_bg_meta_fetched"), "Expected bg_meta_fetched to be 0");
2242 
2243     check(get_meta(h, h1, "k2"), "Get meta failed");
2244     checkeq(0, get_int_stat(h, h1, "ep_bg_fetched"), "Expected bg_fetched to be 0");
2245     checkeq(1, get_int_stat(h, h1, "ep_bg_meta_fetched"), "Expected bg_meta_fetched to be 1");
2246 
2247     checkeq(cb::engine_errc::success,
2248             get(h, h1, NULL, "k1", 0).first,
2249             "Missing key");
2250     checkeq(1, get_int_stat(h, h1, "ep_bg_fetched"), "Expected bg_fetched to be 1");
2251     checkeq(1, get_int_stat(h, h1, "ep_bg_meta_fetched"), "Expected bg_meta_fetched to be 1");
2252 
2253     // store new key with some random metadata
2254     const size_t keylen = strlen("k3");
2255     ItemMetaData itemMeta;
2256     itemMeta.revSeqno = 10;
2257     itemMeta.cas = 0xdeadbeef;
2258     itemMeta.exptime = 0;
2259     itemMeta.flags = 0xdeadbeef;
2260 
2261     add_with_meta(h, h1, "k3", keylen, NULL, 0, 0, &itemMeta);
2262     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Set meta failed");
2263 
2264     check(get_meta(h, h1, "k2"), "Get meta failed");
2265     checkeq(1, get_int_stat(h, h1, "ep_bg_fetched"), "Expected bg_fetched to be 1");
2266     checkeq(1, get_int_stat(h, h1, "ep_bg_meta_fetched"),
2267             "Expected bg_meta_fetched to remain at 1");
2268 
2269     return SUCCESS;
2270 }
2271 
test_key_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2272 static enum test_result test_key_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2273     check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed set vbucket 1 state.");
2274 
2275     // set (k1,v1) in vbucket 0
2276     checkeq(ENGINE_SUCCESS,
2277             store(h, h1, NULL, OPERATION_SET, "k1", "v1"),
2278             "Failed to store an item.");
2279 
2280     // set (k2,v2) in vbucket 1
2281     checkeq(ENGINE_SUCCESS,
2282             store(h, h1, NULL, OPERATION_SET, "k2", "v2", nullptr, 0, 1),
2283             "Failed to store an item.");
2284 
2285     const void *cookie = testHarness.create_cookie();
2286 
2287     // stat for key "k1" and vbucket "0"
2288     const char *statkey1 = "key k1 0";
2289     checkeq(ENGINE_SUCCESS,
2290             h1->get_stats(h, cookie, {statkey1, strlen(statkey1)}, add_stats),
2291             "Failed to get stats.");
2292     check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2293     check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2294     check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2295     check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2296     check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2297 
2298     // stat for key "k2" and vbucket "1"
2299     const char *statkey2 = "key k2 1";
2300     checkeq(ENGINE_SUCCESS,
2301             h1->get_stats(h, cookie, {statkey2, strlen(statkey2)}, add_stats),
2302             "Failed to get stats.");
2303     check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2304     check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2305     check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2306     check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2307     check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2308 
2309     testHarness.destroy_cookie(cookie);
2310     return SUCCESS;
2311 }
2312 
test_vkey_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2313 static enum test_result test_vkey_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2314     check(set_vbucket_state(h, h1, 1, vbucket_state_active), "Failed set vbucket 1 state.");
2315     check(set_vbucket_state(h, h1, 2, vbucket_state_active), "Failed set vbucket 2 state.");
2316     check(set_vbucket_state(h, h1, 3, vbucket_state_active), "Failed set vbucket 3 state.");
2317     check(set_vbucket_state(h, h1, 4, vbucket_state_active), "Failed set vbucket 4 state.");
2318 
2319     wait_for_persisted_value(h, h1, "k1", "v1");
2320     wait_for_persisted_value(h, h1, "k2", "v2", 1);
2321     wait_for_persisted_value(h, h1, "k3", "v3", 2);
2322     wait_for_persisted_value(h, h1, "k4", "v4", 3);
2323     wait_for_persisted_value(h, h1, "k5", "v5", 4);
2324 
2325     check(set_vbucket_state(h, h1, 2, vbucket_state_replica), "Failed to set VB2 state.");
2326     check(set_vbucket_state(h, h1, 3, vbucket_state_pending), "Failed to set VB3 state.");
2327     check(set_vbucket_state(h, h1, 4, vbucket_state_dead), "Failed to set VB4 state.");
2328 
2329     const void *cookie = testHarness.create_cookie();
2330 
2331     // stat for key "k1" and vbucket "0"
2332     const char *statkey1 = "vkey k1 0";
2333     checkeq(ENGINE_SUCCESS,
2334             h1->get_stats(h, cookie, {statkey1, strlen(statkey1)}, add_stats),
2335             "Failed to get stats.");
2336     check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2337     check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2338     check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2339     check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2340     check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2341     check(vals.find("key_valid") != vals.end(), "Found no key_valid");
2342 
2343     // stat for key "k2" and vbucket "1"
2344     const char *statkey2 = "vkey k2 1";
2345     checkeq(ENGINE_SUCCESS,
2346             h1->get_stats(h, cookie, {statkey2, strlen(statkey2)}, add_stats),
2347             "Failed to get stats.");
2348     check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2349     check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2350     check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2351     check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2352     check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2353     check(vals.find("key_valid") != vals.end(), "Found no key_valid");
2354 
2355     // stat for key "k3" and vbucket "2"
2356     const char *statkey3 = "vkey k3 2";
2357     checkeq(ENGINE_SUCCESS,
2358             h1->get_stats(h, cookie, {statkey3, strlen(statkey3)}, add_stats),
2359             "Failed to get stats.");
2360     check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2361     check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2362     check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2363     check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2364     check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2365     check(vals.find("key_valid") != vals.end(), "Found no key_valid");
2366 
2367     // stat for key "k4" and vbucket "3"
2368     const char *statkey4 = "vkey k4 3";
2369     checkeq(ENGINE_SUCCESS,
2370             h1->get_stats(h, cookie, {statkey4, strlen(statkey4)}, add_stats),
2371             "Failed to get stats.");
2372     check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2373     check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2374     check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2375     check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2376     check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2377     check(vals.find("key_valid") != vals.end(), "Found no key_valid");
2378 
2379     // stat for key "k5" and vbucket "4"
2380     const char *statkey5 = "vkey k5 4";
2381     checkeq(ENGINE_SUCCESS,
2382             h1->get_stats(h, cookie, {statkey5, strlen(statkey5)}, add_stats),
2383             "Failed to get stats.");
2384     check(vals.find("key_is_dirty") != vals.end(), "Found no key_is_dirty");
2385     check(vals.find("key_exptime") != vals.end(), "Found no key_exptime");
2386     check(vals.find("key_flags") != vals.end(), "Found no key_flags");
2387     check(vals.find("key_cas") != vals.end(), "Found no key_cas");
2388     check(vals.find("key_vb_state") != vals.end(), "Found no key_vb_state");
2389     check(vals.find("key_valid") != vals.end(), "Found no key_valid");
2390 
2391     testHarness.destroy_cookie(cookie);
2392     return SUCCESS;
2393 }
2394 
test_warmup_conf(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2395 static enum test_result test_warmup_conf(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2396     if (!isWarmupEnabled(h, h1)) {
2397         return SKIPPED;
2398     }
2399 
2400     checkeq(100, get_int_stat(h, h1, "ep_warmup_min_items_threshold"),
2401             "Incorrect initial warmup min items threshold.");
2402     checkeq(100, get_int_stat(h, h1, "ep_warmup_min_memory_threshold"),
2403             "Incorrect initial warmup min memory threshold.");
2404 
2405     check(!set_param(h, h1, protocol_binary_engine_param_flush,
2406                      "warmup_min_items_threshold", "a"),
2407           "Set warmup_min_items_threshold should have failed");
2408     check(!set_param(h, h1, protocol_binary_engine_param_flush,
2409                      "warmup_min_items_threshold", "a"),
2410           "Set warmup_min_memory_threshold should have failed");
2411 
2412     check(set_param(h, h1, protocol_binary_engine_param_flush,
2413                     "warmup_min_items_threshold", "80"),
2414           "Set warmup_min_items_threshold should have worked");
2415     check(set_param(h, h1, protocol_binary_engine_param_flush,
2416                     "warmup_min_memory_threshold", "80"),
2417           "Set warmup_min_memory_threshold should have worked");
2418 
2419     checkeq(80, get_int_stat(h, h1, "ep_warmup_min_items_threshold"),
2420             "Incorrect smaller warmup min items threshold.");
2421     checkeq(80, get_int_stat(h, h1, "ep_warmup_min_memory_threshold"),
2422             "Incorrect smaller warmup min memory threshold.");
2423 
2424     for (int i = 0; i < 100; ++i) {
2425         std::stringstream key;
2426         key << "key-" << i;
2427         checkeq(ENGINE_SUCCESS,
2428                 store(h,
2429                       h1,
2430                       NULL,
2431                       OPERATION_SET,
2432                       key.str().c_str(),
2433                       "somevalue"),
2434                 "Error setting.");
2435     }
2436 
2437     // Restart the server.
2438     std::string config(testHarness.get_current_testcase()->cfg);
2439     config = config + "warmup_min_memory_threshold=0";
2440     testHarness.reload_engine(&h, &h1,
2441                               testHarness.engine_path,
2442                               config.c_str(),
2443                               true, false);
2444     wait_for_warmup_complete(h, h1);
2445 
2446     const std::string eviction_policy = get_str_stat(h, h1, "ep_item_eviction_policy");
2447     if (eviction_policy == "value_only") {
2448         checkeq(100, get_int_stat(h, h1, "ep_warmup_key_count", "warmup"),
2449                 "Expected 100 keys loaded after warmup");
2450     } else { // Full eviction mode
2451         checkeq(0, get_int_stat(h, h1, "ep_warmup_key_count", "warmup"),
2452                 "Expected 0 keys loaded after warmup");
2453     }
2454 
2455     checkeq(0, get_int_stat(h, h1, "ep_warmup_value_count", "warmup"),
2456             "Expected 0 values loaded after warmup");
2457 
2458     return SUCCESS;
2459 }
2460 
2461 // Test that all the configuration parameters associated with the ItemPager,
2462 // can be set.
test_itempager_conf(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1)2463 static enum test_result test_itempager_conf(ENGINE_HANDLE* h,
2464                                             ENGINE_HANDLE_V1* h1) {
2465     check(set_param(h,
2466                     h1,
2467                     protocol_binary_engine_param_flush,
2468                     "pager_active_vb_pcnt",
2469                     "50"),
2470           "Setting pager_active_vb_pcnt should have worked");
2471     checkeq(50,
2472             get_int_stat(h, h1, "ep_pager_active_vb_pcnt"),
2473             "pager_active_vb_pcnt did not get set to the correct value");
2474 
2475     check(set_param(h,
2476                     h1,
2477                     protocol_binary_engine_param_flush,
2478                     "pager_sleep_time_ms",
2479                     "1000"),
2480           "Setting pager_sleep_time_ms should have worked");
2481     checkeq(1000,
2482             get_int_stat(h, h1, "ep_pager_sleep_time_ms"),
2483             "pager_sleep_time_ms did not get set to the correct value");
2484 
2485     check(set_param(h,
2486                     h1,
2487                     protocol_binary_engine_param_flush,
2488                     "ht_eviction_policy",
2489                     "2-bit_lru"),
2490           "Setting ht_eviction_policy should have worked");
2491     checkeq(std::string("2-bit_lru"),
2492             get_str_stat(h, h1, "ep_ht_eviction_policy"),
2493             "ht_eviction_policy did not get set to the correct value");
2494 
2495     check(set_param(h,
2496                     h1,
2497                     protocol_binary_engine_param_flush,
2498                     "item_eviction_age_percentage",
2499                     "100"),
2500           "Set item_eviction_age_percentage should have worked");
2501     checkeq(100,
2502             get_int_stat(h, h1, "ep_item_eviction_age_percentage"),
2503             "item_eviction_age_percentage did not get set to the correct "
2504             "value");
2505 
2506     check(set_param(h,
2507                     h1,
2508                     protocol_binary_engine_param_flush,
2509                     "item_eviction_freq_counter_age_threshold",
2510                     "10"),
2511           "Set item_eviction_freq_counter_age_threshold should have worked");
2512     checkeq(10,
2513             get_int_stat(h, h1, "ep_item_eviction_freq_counter_age_threshold"),
2514             "item_eviction_freq_counter_age_threshold did not get set to the "
2515             "correct value");
2516 
2517     check(set_param(h,
2518                     h1,
2519                     protocol_binary_engine_param_flush,
2520                     "item_freq_decayer_chunk_duration",
2521                     "1000"),
2522           "Set item_freq_decayer_chunk_duration should have worked");
2523     checkeq(1000,
2524             get_int_stat(h, h1, "ep_item_freq_decayer_chunk_duration"),
2525             "item_freq_decayer_chunk_duration did not get set to the correct "
2526             "value");
2527 
2528     check(set_param(h,
2529                     h1,
2530                     protocol_binary_engine_param_flush,
2531                     "item_freq_decayer_percent",
2532                     "100"),
2533           "Set item_freq_decayer_percent should have worked");
2534     checkeq(100,
2535             get_int_stat(h, h1, "ep_item_freq_decayer_percent"),
2536             "item_freq_decayer_percent did not get set to the correct value");
2537 
2538     return SUCCESS;
2539 }
2540 
test_bloomfilter_conf(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2541 static enum test_result test_bloomfilter_conf(ENGINE_HANDLE *h,
2542                                               ENGINE_HANDLE_V1 *h1) {
2543 
2544     if (get_bool_stat(h, h1, "ep_bfilter_enabled") == false) {
2545         check(set_param(h, h1, protocol_binary_engine_param_flush,
2546                         "bfilter_enabled", "true"),
2547               "Set bloomfilter_enabled should have worked");
2548     }
2549     check(get_bool_stat(h, h1, "ep_bfilter_enabled"),
2550           "Bloom filter wasn't enabled");
2551 
2552     check(get_float_stat(h, h1, "ep_bfilter_residency_threshold") == (float)0.1,
2553           "Incorrect initial bfilter_residency_threshold.");
2554 
2555     check(set_param(h, h1, protocol_binary_engine_param_flush,
2556           "bfilter_enabled", "false"),
2557           "Set bloomfilter_enabled should have worked.");
2558     check(set_param(h, h1, protocol_binary_engine_param_flush,
2559           "bfilter_residency_threshold", "0.15"),
2560           "Set bfilter_residency_threshold should have worked.");
2561 
2562     check(get_bool_stat(h, h1, "ep_bfilter_enabled") == false,
2563           "Bloom filter should have been disabled.");
2564     check(get_float_stat(h, h1, "ep_bfilter_residency_threshold") == (float)0.15,
2565           "Incorrect bfilter_residency_threshold.");
2566 
2567     return SUCCESS;
2568 }
2569 
test_bloomfilters(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2570 static enum test_result test_bloomfilters(ENGINE_HANDLE *h,
2571                                           ENGINE_HANDLE_V1 *h1) {
2572 
2573     if (get_bool_stat(h, h1, "ep_bfilter_enabled") == false) {
2574         check(set_param(h, h1, protocol_binary_engine_param_flush,
2575                     "bfilter_enabled", "true"),
2576                 "Set bloomfilter_enabled should have worked");
2577     }
2578     check(get_bool_stat(h, h1, "ep_bfilter_enabled"),
2579             "Bloom filter wasn't enabled");
2580 
2581     // Key is only present if bgOperations is non-zero.
2582     int num_read_attempts = get_int_stat_or_default(h, h1, 0,
2583                                                     "ep_bg_num_samples");
2584 
2585     // Ensure vbucket's bloom filter is enabled
2586     checkeq(std::string("ENABLED"),
2587             get_str_stat(h, h1, "vb_0:bloom_filter", "vbucket-details 0"),
2588             "Vbucket 0's bloom filter wasn't enabled upon setup!");
2589 
2590     int i;
2591 
2592     // Insert 10 items.
2593     for (i = 0; i < 10; ++i) {
2594         std::stringstream key;
2595         key << "key-" << i;
2596         checkeq(ENGINE_SUCCESS,
2597                 store(h,
2598                       h1,
2599                       NULL,
2600                       OPERATION_SET,
2601                       key.str().c_str(),
2602                       "somevalue"),
2603                 "Error setting.");
2604     }
2605     wait_for_flusher_to_settle(h, h1);
2606 
2607     // Evict all 10 items.
2608     for (i = 0; i < 10; ++i) {
2609         std::stringstream key;
2610         key << "key-" << i;
2611         evict_key(h, h1, key.str().c_str(), 0, "Ejected.");
2612     }
2613     wait_for_flusher_to_settle(h, h1);
2614 
2615     // Ensure 10 items are non-resident.
2616     cb_assert(10 == get_int_stat(h, h1, "ep_num_non_resident"));
2617 
2618     // Issue delete on first 5 items.
2619     for (i = 0; i < 5; ++i) {
2620         std::stringstream key;
2621         key << "key-" << i;
2622         checkeq(ENGINE_SUCCESS,
2623                 del(h, h1, key.str().c_str(), 0, 0),
2624                 "Failed remove with value.");
2625     }
2626     wait_for_flusher_to_settle(h, h1);
2627 
2628     // Ensure that there are 5 non-resident items
2629     cb_assert(5 == get_int_stat(h, h1, "ep_num_non_resident"));
2630     cb_assert(5 == get_int_stat(h, h1, "curr_items"));
2631 
2632     checkeq(ENGINE_SUCCESS,
2633             get_stats(h, {}, add_stats),
2634             "Failed to get stats.");
2635     std::string eviction_policy = vals.find("ep_item_eviction_policy")->second;
2636 
2637     useconds_t sleepTime = 128;
2638 
2639     if (eviction_policy == "value_only") {  // VALUE-ONLY EVICTION MODE
2640 
2641         checkeq(5,
2642                 get_int_stat(h, h1, "vb_0:bloom_filter_key_count",
2643                              "vbucket-details 0"),
2644                 "Unexpected no. of keys in bloom filter");
2645 
2646         checkeq(num_read_attempts,
2647                 get_int_stat_or_default(h, h1, 0, "ep_bg_num_samples"),
2648                 "Expected bgFetch attempts to remain unchanged");
2649 
2650         for (i = 0; i < 5; ++i) {
2651             std::stringstream key;
2652             key << "key-" << i;
2653             check(get_meta(h, h1, key.str().c_str()), "Get meta failed");
2654         }
2655 
2656         // GetMeta would cause bgFetches as bloomfilter contains
2657         // the deleted items.
2658         checkeq(num_read_attempts + 5,
2659                 get_int_stat(h, h1, "ep_bg_num_samples"),
2660                 "Expected bgFetch attempts to increase by five");
2661 
2662         // Run compaction, with drop_deletes
2663         compact_db(h, h1, 0, 0, 15, 15, 1);
2664         while (get_int_stat(h, h1, "ep_pending_compactions") != 0) {
2665             decayingSleep(&sleepTime);
2666         }
2667 
2668         for (i = 0; i < 5; ++i) {
2669             std::stringstream key;
2670             key << "key-" << i;
2671             check(get_meta(h, h1, key.str().c_str()), "Get meta failed");
2672         }
2673         checkeq(num_read_attempts + 5,
2674                 get_int_stat(h, h1, "ep_bg_num_samples"),
2675                 "Expected bgFetch attempts to stay as before");
2676 
2677     } else {                                // FULL EVICTION MODE
2678 
2679         checkeq(10,
2680                 get_int_stat(h, h1, "vb_0:bloom_filter_key_count",
2681                              "vbucket-details 0"),
2682                 "Unexpected no. of keys in bloom filter");
2683 
2684 
2685         // Because of issuing deletes on non-resident items
2686         checkeq(num_read_attempts + 5,
2687                 get_int_stat(h, h1, "ep_bg_num_samples"),
2688                 "Expected bgFetch attempts to increase by five, after deletes");
2689 
2690         // Run compaction, with drop_deletes, to exclude deleted items
2691         // from bloomfilter.
2692         compact_db(h, h1, 0, 0, 15, 15, 1);
2693         while (get_int_stat(h, h1, "ep_pending_compactions") != 0) {
2694             decayingSleep(&sleepTime);
2695         }
2696 
2697         for (i = 0; i < 5; i++) {
2698             std::stringstream key;
2699             key << "key-" << i;
2700             checkeq(cb::engine_errc::no_such_key,
2701                     get(h, h1, NULL, key.str(), 0).first,
2702                     "Unable to get stored item");
2703         }
2704         // + 6 because last delete is not purged by the compactor
2705         checkeq(num_read_attempts + 6,
2706                 get_int_stat(h, h1, "ep_bg_num_samples"),
2707                 "Expected bgFetch attempts to stay as before");
2708     }
2709 
2710     return SUCCESS;
2711 }
2712 
test_bloomfilters_with_store_apis(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2713 static enum test_result test_bloomfilters_with_store_apis(ENGINE_HANDLE *h,
2714                                                           ENGINE_HANDLE_V1 *h1) {
2715     if (get_bool_stat(h, h1, "ep_bfilter_enabled") == false) {
2716         check(set_param(h, h1, protocol_binary_engine_param_flush,
2717                     "bfilter_enabled", "true"),
2718                 "Set bloomfilter_enabled should have worked");
2719     }
2720     check(get_bool_stat(h, h1, "ep_bfilter_enabled"),
2721             "Bloom filter wasn't enabled");
2722 
2723     int num_read_attempts = get_int_stat_or_default(h, h1, 0,
2724                                                     "ep_bg_num_samples");
2725 
2726     // Ensure vbucket's bloom filter is enabled
2727     checkeq(std::string("ENABLED"),
2728             get_str_stat(h, h1, "vb_0:bloom_filter", "vbucket-details 0"),
2729             "Vbucket 0's bloom filter wasn't enabled upon setup!");
2730 
2731     for (int i = 0; i < 1000; i++) {
2732         std::stringstream key;
2733         key << "key-" << i;
2734         check(!get_meta(h, h1, key.str().c_str()),
2735                 "Get meta should fail.");
2736     }
2737 
2738     checkeq(num_read_attempts,
2739             get_int_stat_or_default(h, h1, 0, "ep_bg_num_samples"),
2740             "Expected no bgFetch attempts");
2741 
2742     checkeq(ENGINE_SUCCESS,
2743             get_stats(h, {}, add_stats),
2744             "Failed to get stats.");
2745     std::string eviction_policy = vals.find("ep_item_eviction_policy")->second;
2746 
2747     if (eviction_policy == "full_eviction") {  // FULL EVICTION MODE
2748         // Set with Meta
2749         int j;
2750         for (j = 0; j < 10; j++) {
2751             uint64_t cas_for_set = last_cas;
2752             // init some random metadata
2753             ItemMetaData itm_meta;
2754             itm_meta.revSeqno = 10;
2755             itm_meta.cas = 0xdeadbeef;
2756             itm_meta.exptime = time(NULL) + 300;
2757             itm_meta.flags = 0xdeadbeef;
2758 
2759             std::stringstream key;
2760             key << "swm-" << j;
2761             set_with_meta(h, h1, key.str().c_str(), key.str().length(),
2762                           "somevalue", 9, 0, &itm_meta, cas_for_set);
2763         }
2764 
2765         checkeq(num_read_attempts,
2766                 get_int_stat_or_default(h, h1, 0, "ep_bg_num_samples"),
2767                 "Expected no bgFetch attempts");
2768 
2769         // Add
2770         for (j = 0; j < 10; j++) {
2771             std::stringstream key;
2772             key << "add-" << j;
2773 
2774             checkeq(ENGINE_SUCCESS,
2775                     store(h,
2776                           h1,
2777                           NULL,
2778                           OPERATION_ADD,
2779                           key.str().c_str(),
2780                           "newvalue"),
2781                     "Failed to add value again.");
2782         }
2783 
2784         checkeq(num_read_attempts,
2785                 get_int_stat_or_default(h, h1, 0, "ep_bg_num_samples"),
2786                 "Expected no bgFetch attempts");
2787 
2788         // Delete
2789         for (j = 0; j < 10; j++) {
2790             std::stringstream key;
2791             key << "del-" << j;
2792             checkeq(ENGINE_KEY_ENOENT,
2793                     del(h, h1, key.str().c_str(), 0, 0),
2794                     "Failed remove with value.");
2795         }
2796 
2797         checkeq(num_read_attempts,
2798                 get_int_stat_or_default(h, h1, 0, "ep_bg_num_samples"),
2799                 "Expected no bgFetch attempts");
2800 
2801     }
2802 
2803     return SUCCESS;
2804 }
2805 
test_bloomfilter_delete_plus_set_scenario( ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2806 static enum test_result test_bloomfilter_delete_plus_set_scenario(
2807                                        ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2808     if (get_bool_stat(h, h1, "ep_bfilter_enabled") == false) {
2809         check(set_param(h, h1, protocol_binary_engine_param_flush,
2810                     "bfilter_enabled", "true"),
2811                 "Set bloomfilter_enabled should have worked");
2812     }
2813     check(get_bool_stat(h, h1, "ep_bfilter_enabled"),
2814             "Bloom filter wasn't enabled");
2815 
2816     // Ensure vbucket's bloom filter is enabled
2817     checkeq(std::string("ENABLED"),
2818             get_str_stat(h, h1, "vb_0:bloom_filter", "vbucket-details 0"),
2819             "Vbucket 0's bloom filter wasn't enabled upon setup!");
2820 
2821     checkeq(ENGINE_SUCCESS,
2822             store(h, h1, NULL, OPERATION_SET, "k1", "v1"),
2823             "Failed to fail to store an item.");
2824 
2825     wait_for_flusher_to_settle(h, h1);
2826     int num_writes = get_int_stat(h, h1, "rw_0:io_num_write", "kvstore");
2827     int num_persisted = get_int_stat(h, h1, "ep_total_persisted");
2828     cb_assert(num_writes == 1 && num_persisted == 1);
2829 
2830     checkeq(ENGINE_SUCCESS,
2831             del(h, h1, "k1", 0, 0), "Failed remove with value.");
2832     stop_persistence(h, h1);
2833     checkeq(ENGINE_SUCCESS,
2834             store(h, h1, NULL, OPERATION_SET, "k1", "v2", nullptr, 0, 0),
2835             "Failed to fail to store an item.");
2836     int key_count = get_int_stat(h, h1, "vb_0:bloom_filter_key_count",
2837                                  "vbucket-details 0");
2838 
2839     if (key_count == 0) {
2840         check(get_int_stat(h, h1, "rw_0:io_num_write", "kvstore") <= 2,
2841                 "Unexpected number of writes");
2842         start_persistence(h, h1);
2843         wait_for_flusher_to_settle(h, h1);
2844         checkeq(0, get_int_stat(h, h1, "vb_0:bloom_filter_key_count",
2845                                 "vbucket-details 0"),
2846                 "Unexpected number of keys in bloomfilter");
2847     } else {
2848         cb_assert(key_count == 1);
2849         checkeq(2, get_int_stat(h, h1, "rw_0:io_num_write", "kvstore"),
2850                 "Unexpected number of writes");
2851         start_persistence(h, h1);
2852         wait_for_flusher_to_settle(h, h1);
2853         checkeq(1, get_int_stat(h, h1, "vb_0:bloom_filter_key_count",
2854                                 "vbucket-details 0"),
2855                 "Unexpected number of keys in bloomfilter");
2856     }
2857 
2858     return SUCCESS;
2859 }
2860 
test_datatype(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2861 static enum test_result test_datatype(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2862     const void *cookie = testHarness.create_cookie();
2863     testHarness.set_datatype_support(cookie, true);
2864 
2865     item *itm = NULL;
2866     const std::string key("{\"foo\":\"bar\"}");
2867     const protocol_binary_datatype_t datatype = PROTOCOL_BINARY_DATATYPE_JSON;
2868     uint64_t cas = 0;
2869     std::string value("x");
2870     checkeq(ENGINE_SUCCESS,
2871             storeCasOut(h, h1, NULL, 0, key, value, datatype, itm, cas),
2872             "Expected set to succeed");
2873 
2874     auto ret = get(h, h1, cookie, key, 0);
2875     checkeq(cb::engine_errc::success, ret.first, "Unable to get stored item");
2876 
2877     item_info info;
2878     h1->get_item_info(h, ret.second.get(), &info);
2879     checkeq(static_cast<uint8_t>(PROTOCOL_BINARY_DATATYPE_JSON),
2880             info.datatype, "Invalid datatype");
2881 
2882     const char* key1 = "foo";
2883     const char* val1 = "{\"foo1\":\"bar1\"}";
2884     ItemMetaData itm_meta;
2885     itm_meta.revSeqno = 10;
2886     itm_meta.cas = info.cas;
2887     itm_meta.exptime = info.exptime;
2888     itm_meta.flags = info.flags;
2889     set_with_meta(h, h1, key1, strlen(key1), val1, strlen(val1), 0, &itm_meta,
2890                   last_cas, 0, info.datatype, cookie);
2891 
2892     ret = get(h, h1, cookie, key1, 0);
2893     checkeq(cb::engine_errc::success, ret.first, "Unable to get stored item");
2894 
2895     h1->get_item_info(h, ret.second.get(), &info);
2896     checkeq(static_cast<uint8_t>(PROTOCOL_BINARY_DATATYPE_JSON),
2897             info.datatype, "Invalid datatype, when setWithMeta");
2898 
2899     testHarness.destroy_cookie(cookie);
2900     return SUCCESS;
2901 }
2902 
test_datatype_with_unknown_command(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2903 static enum test_result test_datatype_with_unknown_command(ENGINE_HANDLE *h,
2904                                                            ENGINE_HANDLE_V1 *h1) {
2905     const void *cookie = testHarness.create_cookie();
2906     testHarness.set_datatype_support(cookie, true);
2907     const char* key = "foo";
2908     const char* val = "{\"foo\":\"bar\"}";
2909     uint8_t datatype = PROTOCOL_BINARY_DATATYPE_JSON;
2910 
2911     ItemMetaData itm_meta;
2912     itm_meta.revSeqno = 10;
2913     itm_meta.cas = 0x1;
2914     itm_meta.exptime = 0;
2915     itm_meta.flags = 0;
2916 
2917     //SET_WITH_META
2918     set_with_meta(h, h1, key, strlen(key), val, strlen(val), 0, &itm_meta,
2919                   0, 0, datatype, cookie);
2920 
2921     auto ret = get(h, h1, cookie, key, 0);
2922     checkeq(cb::engine_errc::success, ret.first, "Unable to get stored item");
2923 
2924     item_info info;
2925     h1->get_item_info(h, ret.second.get(), &info);
2926     checkeq(static_cast<uint8_t>(PROTOCOL_BINARY_DATATYPE_JSON),
2927             info.datatype, "Invalid datatype, when setWithMeta");
2928 
2929     //SET_RETURN_META
2930     checkeq(ENGINE_SUCCESS,
2931             set_ret_meta(h,
2932                          h1,
2933                          "foo1",
2934                          4,
2935                          val,
2936                          strlen(val),
2937                          0,
2938                          0,
2939                          0,
2940                          0,
2941                          datatype,
2942                          cookie),
2943             "Expected success");
2944     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
2945             "Expected set returing meta to succeed");
2946     checkeq(static_cast<uint8_t>(PROTOCOL_BINARY_DATATYPE_JSON),
2947             last_datatype.load(), "Invalid datatype, when set_return_meta");
2948 
2949     testHarness.destroy_cookie(cookie);
2950     return SUCCESS;
2951 }
2952 
test_session_cas_validation(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2953 static enum test_result test_session_cas_validation(ENGINE_HANDLE *h,
2954                                                     ENGINE_HANDLE_V1 *h1) {
2955     //Testing PROTOCOL_BINARY_CMD_SET_VBUCKET..
2956     char ext[4];
2957     protocol_binary_request_header *pkt;
2958     vbucket_state_t state = vbucket_state_active;
2959     uint32_t val = static_cast<uint32_t>(state);
2960     val = htonl(val);
2961     memcpy(ext, (char*)&val, sizeof(val));
2962 
2963     uint64_t cas = 0x0101010101010101;
2964     pkt = createPacket(PROTOCOL_BINARY_CMD_SET_VBUCKET, 0, cas, ext, 4);
2965     checkeq(ENGINE_SUCCESS,
2966             h1->unknown_command(h, NULL, pkt, add_response, testHarness.doc_namespace),
2967             "SET_VBUCKET command failed");
2968     cb_free(pkt);
2969     cb_assert(last_status == PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS);
2970 
2971     cas = 0x0102030405060708;
2972     pkt = createPacket(PROTOCOL_BINARY_CMD_SET_VBUCKET, 0, cas, ext, 4);
2973     checkeq(ENGINE_SUCCESS,
2974             h1->unknown_command(h, NULL, pkt, add_response, testHarness.doc_namespace),
2975             "SET_VBUCKET command failed");
2976     cb_free(pkt);
2977     cb_assert(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS);
2978 
2979     return SUCCESS;
2980 }
2981 
test_access_scanner_settings(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)2982 static enum test_result test_access_scanner_settings(ENGINE_HANDLE *h,