1 /* -*- MODE: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3  *     Copyright 2016 Couchbase, Inc
4  *
5  *   Licensed under the Apache License, Version 2.0 (the "License");
6  *   you may not use this file except in compliance with the License.
7  *   You may obtain a copy of the License at
8  *
9  *       http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *   Unless required by applicable law or agreed to in writing, software
12  *   distributed under the License is distributed on an "AS IS" BASIS,
13  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *   See the License for the specific language governing permissions and
15  *   limitations under the License.
16  */
17 
18 /*
19  * Testsuite for XDCR-related functionality in ep-engine.
20  */
21 #include "ep_test_apis.h"
22 #include "ep_testsuite_common.h"
23 #include "hlc.h"
24 #include <platform/cb_malloc.h>
25 #include <string_utilities.h>
26 #include <xattr/blob.h>
27 #include <xattr/utils.h>
28 
29 // Helper functions ///////////////////////////////////////////////////////////
30 
verifyMetaData(const ItemMetaData& imd, const item_info& metadata)31 static void verifyMetaData(const ItemMetaData& imd, const item_info& metadata) {
32     checkeq(uint64_t(imd.revSeqno), metadata.seqno, "Seqno didn't match");
33     checkeq(imd.cas, metadata.cas, "Cas didn't match");
34     checkeq(imd.exptime,
35             static_cast<time_t>(metadata.exptime),
36             "Expiration time didn't match");
37     checkeq(imd.flags, metadata.flags, "Flags didn't match");
38 }
39 
40 /**
41  * Create an XATTR document using the supplied string as the body
42  * @returns vector containing the body bytes
43  */
createXattrValue(const std::string& body)44 static std::vector<char> createXattrValue(const std::string& body) {
45     cb::xattr::Blob blob;
46 
47     //Add a few XAttrs
48     blob.set("user", "{\"author\":\"bubba\"}");
49     blob.set("_sync", "{\"cas\":\"0xdeadbeefcafefeed\"}");
50     blob.set("meta", "{\"content-type\":\"text\"}");
51 
52     auto xattr_value = blob.finalize();
53 
54     // append body to the xattrs and store in data
55     std::vector<char> data;
56     std::copy(xattr_value.buf, xattr_value.buf + xattr_value.len,
57               std::back_inserter(data));
58     std::copy(body.c_str(), body.c_str() + body.size(),
59               std::back_inserter(data));
60 
61     return data;
62 }
63 
64 
65 // Testcases //////////////////////////////////////////////////////////////////
66 
test_get_meta(EngineIface* h)67 static enum test_result test_get_meta(EngineIface* h) {
68     char const *key = "test_get_meta";
69     item *i = NULL;
70     checkeq(ENGINE_SUCCESS,
71             store(h, NULL, OPERATION_SET, key, "somevalue", &i),
72             "Failed set.");
73     Item *it = reinterpret_cast<Item*>(i);
74     // check the stat
75     auto temp = get_int_stat(h, "ep_num_ops_get_meta");
76     checkeq(0, temp, "Expect zero getMeta ops");
77 
78     cb::EngineErrorMetadataPair errorMetaPair;
79     check(get_meta(h, key, errorMetaPair), "Expected to get meta");
80 
81     ItemMetaData metadata(it->getCas(), it->getRevSeqno(),
82                           it->getFlags(), it->getExptime());
83     verifyMetaData(metadata, errorMetaPair.second);
84 
85     // check the stat again
86     temp = get_int_stat(h, "ep_num_ops_get_meta");
87     checkeq(1, temp, "Expect one getMeta op");
88 
89     h->release(i);
90     return SUCCESS;
91 }
92 
test_get_meta_with_extras(EngineIface* h)93 static enum test_result test_get_meta_with_extras(EngineIface* h) {
94     const char *key1 = "test_getm_one";
95     item *i = NULL;
96     checkeq(ENGINE_SUCCESS,
97             store(h, NULL, OPERATION_SET, key1, "somevalue", &i),
98             "Failed set.");
99 
100     wait_for_flusher_to_settle(h);
101 
102     Item *it1 = reinterpret_cast<Item*>(i);
103     // check the stat
104     auto temp = get_int_stat(h, "ep_num_ops_get_meta");
105     checkeq(0, temp, "Expect zero getMeta ops");
106 
107     cb::EngineErrorMetadataPair errorMetaPair;
108 
109     check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
110     ItemMetaData metadata1(it1->getCas(), it1->getRevSeqno(),
111                            it1->getFlags(), it1->getExptime());
112     verifyMetaData(metadata1, errorMetaPair.second);
113     // check the stat again
114     temp = get_int_stat(h, "ep_num_ops_get_meta");
115     checkeq(1, temp, "Expect one getMeta op");
116     h->release(i);
117 
118     if (isWarmupEnabled(h)) {
119         // restart
120         testHarness->reload_engine(&h,
121                                    testHarness->engine_path,
122                                    testHarness->get_current_testcase()->cfg,
123                                    true,
124                                    true);
125 
126         wait_for_warmup_complete(h);
127 
128         check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
129         verifyMetaData(metadata1, errorMetaPair.second);
130     }
131 
132     return SUCCESS;
133 }
134 
test_get_meta_deleted(EngineIface* h)135 static enum test_result test_get_meta_deleted(EngineIface* h) {
136     char const *key = "k1";
137     item *i = NULL;
138 
139     checkeq(ENGINE_SUCCESS,
140             store(h, NULL, OPERATION_SET, key, "somevalue"),
141             "Failed set.");
142     checkeq(ENGINE_SUCCESS,
143             store(h, NULL, OPERATION_SET, key, "somevalue", &i),
144             "Failed set.");
145 
146     Item *it = reinterpret_cast<Item*>(i);
147     wait_for_flusher_to_settle(h);
148 
149     checkeq(ENGINE_SUCCESS,
150             del(h, key, it->getCas(), Vbid(0)),
151             "Delete failed");
152     wait_for_flusher_to_settle(h);
153 
154     // check the stat
155     int temp = get_int_stat(h, "ep_num_ops_get_meta");
156    checkeq(0, temp, "Expect zero getMeta ops");
157 
158     cb::EngineErrorMetadataPair errorMetaPair;
159     check(get_meta(h, key, errorMetaPair), "Expected to get meta");
160     checkeq(DocumentState::Deleted,
161             errorMetaPair.second.document_state,
162             "Expected deleted flag to be set");
163     checkeq(errorMetaPair.second.seqno, (it->getRevSeqno() + 1),
164           "Expected seqno to match");
165     checkne(errorMetaPair.second.cas, it->getCas(),
166           "Expected cas to be different");
167     checkeq(errorMetaPair.second.flags, it->getFlags(),
168           "Expected flags to match");
169 
170     // check the stat again
171     temp = get_int_stat(h, "ep_num_ops_get_meta");
172     checkeq(1, temp, "Expect one getMeta op");
173 
174     h->release(i);
175     return SUCCESS;
176 }
177 
test_get_meta_nonexistent(EngineIface* h)178 static enum test_result test_get_meta_nonexistent(EngineIface* h) {
179     char const *key = "k1";
180 
181     // check the stat
182     int temp = get_int_stat(h, "ep_num_ops_get_meta");
183    checkeq(0, temp, "Expect zero getMeta ops");
184 
185     cb::EngineErrorMetadataPair errorMetaPair;
186     check(!get_meta(h, key, errorMetaPair),
187           "Expected get meta to return false");
188     checkeq(cb::engine_errc::no_such_key,
189             errorMetaPair.first,
190             "Expected no_such_key");
191 
192     // check the stat again
193     temp = get_int_stat(h, "ep_num_ops_get_meta");
194     checkeq(1, temp, "Expect one getMeta ops");
195 
196     return SUCCESS;
197 }
198 
test_get_meta_with_get(EngineIface* h)199 static enum test_result test_get_meta_with_get(EngineIface* h) {
200     char const *key1 = "key1";
201     char const *key2 = "key2";
202 
203     // test get_meta followed by get for an existing key. should pass.
204     checkeq(ENGINE_SUCCESS,
205             store(h, NULL, OPERATION_SET, key1, "somevalue"),
206             "Failed set.");
207     wait_for_flusher_to_settle(h);
208     // check the stat
209     int temp = get_int_stat(h, "ep_num_ops_get_meta");
210    checkeq(0, temp, "Expect zero getMeta ops");
211 
212     cb::EngineErrorMetadataPair errorMetaPair;
213 
214     check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
215     auto ret = get(h, NULL, key1, Vbid(0));
216     checkeq(cb::engine_errc::success, ret.first,
217             "Expected get success");
218     ret.second.reset();
219     // check the stat again
220     temp = get_int_stat(h, "ep_num_ops_get_meta");
221    checkeq(1, temp, "Expect one getMeta op");
222 
223     // test get_meta followed by get for a deleted key. should fail.
224     checkeq(ENGINE_SUCCESS, del(h, key1, 0, Vbid(0)), "Delete failed");
225     wait_for_flusher_to_settle(h);
226     check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
227     checkeq(DocumentState::Deleted,
228             errorMetaPair.second.document_state,
229             "Expected deleted flag to be set");
230     checkeq(cb::engine_errc::no_such_key,
231             get(h, NULL, key1, Vbid(0)).first,
232             "Expected enoent");
233     // check the stat again
234     temp = get_int_stat(h, "ep_num_ops_get_meta");
235     checkeq(2, temp, "Expect more getMeta ops");
236 
237     // test get_meta followed by get for a nonexistent key. should fail.
238     check(!get_meta(h, key2, errorMetaPair),
239           "Expected get meta to return false");
240     checkeq(cb::engine_errc::no_such_key,
241             errorMetaPair.first,
242             "Expected no_such_key");
243     checkeq(cb::engine_errc::no_such_key,
244             get(h, NULL, key2, Vbid(0)).first,
245             "Expected enoent");
246     // check the stat again
247     temp = get_int_stat(h, "ep_num_ops_get_meta");
248     checkeq(3, temp, "Expected one extra getMeta ops");
249 
250     return SUCCESS;
251 }
252 
test_get_meta_with_set(EngineIface* h)253 static enum test_result test_get_meta_with_set(EngineIface* h) {
254     char const *key1 = "key1";
255     char const *key2 = "key2";
256 
257     ItemMetaData itm_meta;
258 
259     // test get_meta followed by set for an existing key. should pass.
260     checkeq(ENGINE_SUCCESS,
261             store(h, NULL, OPERATION_SET, key1, "somevalue"),
262             "Failed set.");
263     wait_for_flusher_to_settle(h);
264     wait_for_stat_to_be(h, "curr_items", 1);
265 
266     // check the stat
267     checkeq(0,
268             get_int_stat(h, "ep_num_ops_get_meta"),
269             "Expect zero getMeta ops");
270 
271     cb::EngineErrorMetadataPair errorMetaPair;
272 
273     check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
274     checkeq(ENGINE_SUCCESS,
275             store(h, NULL, OPERATION_SET, key1, "someothervalue"),
276             "Failed set.");
277     // check the stat
278     checkeq(1, get_int_stat(h, "ep_num_ops_get_meta"), "Expect one getMeta op");
279     checkeq(1, get_int_stat(h, "curr_items"), "Expected single curr_items");
280     checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
281 
282     // check curr, temp item counts
283     checkeq(1, get_int_stat(h, "curr_items"), "Expected single curr_items");
284     checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
285 
286     // test get_meta followed by set for a deleted key. should pass.
287     checkeq(ENGINE_SUCCESS, del(h, key1, 0, Vbid(0)), "Delete failed");
288     wait_for_flusher_to_settle(h);
289 
290     wait_for_stat_to_be(h, "curr_items", 0);
291     check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
292     checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
293     checkPersistentBucketTempItems(h, 1);
294 
295     checkeq(DocumentState::Deleted,
296             errorMetaPair.second.document_state,
297             "Expected deleted flag to be set");
298     checkeq(ENGINE_SUCCESS,
299             store(h, NULL, OPERATION_SET, key1, "someothervalue"),
300             "Failed set.");
301     wait_for_flusher_to_settle(h);
302 
303     checkeq(1, get_int_stat(h, "curr_items"), "Expected single curr_items");
304     checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
305 
306     // check the stat
307     checkeq(2,
308             get_int_stat(h, "ep_num_ops_get_meta"),
309             "Expect more getMeta ops");
310 
311     // test get_meta followed by set for a nonexistent key. should pass.
312     check(!get_meta(h, key2, errorMetaPair),
313           "Expected get meta to return false");
314     checkeq(cb::engine_errc::no_such_key,
315             errorMetaPair.first,
316             "Expected no_such_key");
317     checkeq(ENGINE_SUCCESS,
318             store(h, NULL, OPERATION_SET, key2, "someothervalue"),
319             "Failed set.");
320     // check the stat again
321     checkeq(3,
322             get_int_stat(h, "ep_num_ops_get_meta"),
323             "Expected one extra getMeta ops");
324 
325     return SUCCESS;
326 }
327 
test_get_meta_with_delete(EngineIface* h)328 static enum test_result test_get_meta_with_delete(EngineIface* h) {
329     char const *key1 = "key1";
330     char const *key2 = "key2";
331 
332     // test get_meta followed by delete for an existing key. should pass.
333     checkeq(ENGINE_SUCCESS,
334             store(h, NULL, OPERATION_SET, key1, "somevalue"),
335             "Failed set.");
336     wait_for_flusher_to_settle(h);
337     // check the stat
338     int temp = get_int_stat(h, "ep_num_ops_get_meta");
339    checkeq(0, temp, "Expect zero getMeta ops");
340 
341     cb::EngineErrorMetadataPair errorMetaPair;
342 
343     check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
344     checkeq(ENGINE_SUCCESS, del(h, key1, 0, Vbid(0)), "Delete failed");
345     // check the stat
346     temp = get_int_stat(h, "ep_num_ops_get_meta");
347    checkeq(1, temp, "Expect one getMeta op");
348 
349     // test get_meta followed by delete for a deleted key. should fail.
350     wait_for_flusher_to_settle(h);
351     check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
352     checkeq(DocumentState::Deleted,
353             errorMetaPair.second.document_state,
354             "Expected deleted flag to be set");
355     checkeq(ENGINE_KEY_ENOENT, del(h, key1, 0, Vbid(0)), "Expected enoent");
356     // check the stat
357     temp = get_int_stat(h, "ep_num_ops_get_meta");
358     checkeq(2, temp, "Expect more getMeta op");
359 
360     // test get_meta followed by delete for a nonexistent key. should fail.
361     check(!get_meta(h, key2, errorMetaPair),
362           "Expected get meta to return false");
363     checkeq(cb::engine_errc::no_such_key,
364             errorMetaPair.first,
365             "Expected no_such_key");
366     checkeq(ENGINE_KEY_ENOENT, del(h, key2, 0, Vbid(0)), "Expected enoent");
367     // check the stat again
368     temp = get_int_stat(h, "ep_num_ops_get_meta");
369     checkeq(3, temp, "Expected one extra getMeta ops");
370 
371     return SUCCESS;
372 }
373 
test_get_meta_with_xattr(EngineIface* h)374 static enum test_result test_get_meta_with_xattr(EngineIface* h) {
375     const char* key = "get_meta_key";
376     std::vector<char> data = createXattrValue({"test_expiry_value"});
377 
378     const void* cookie = testHarness->create_cookie();
379 
380     checkeq(cb::engine_errc::success,
381             storeCasVb11(h,
382                          cookie,
383                          OPERATION_SET,
384                          key,
385                          reinterpret_cast<char*>(data.data()),
386                          data.size(),
387                          9258,
388                          0,
389                          Vbid(0),
390                          0,
391                          PROTOCOL_BINARY_DATATYPE_XATTR)
392                     .first,
393             "Failed to store xattr document");
394 
395     if (isPersistentBucket(h)) {
396         wait_for_flusher_to_settle(h);
397     }
398 
399     cb::EngineErrorMetadataPair errorMetaPair;
400 
401     // Check that the datatype is XATTR (at engine level the datatype is always
402     // returned).
403     check(get_meta(h, key, errorMetaPair, cookie), "Get meta command failed");
404     checkeq(PROTOCOL_BINARY_DATATYPE_XATTR,
405             errorMetaPair.second.datatype,
406             "Datatype is not XATTR");
407 
408     if (isPersistentBucket(h)) {
409         //Evict the key
410         evict_key(h, key);
411 
412         // This should result in a bg fetch
413         check(get_meta(h, key, errorMetaPair, cookie),
414               "Get meta command failed");
415         checkeq(PROTOCOL_BINARY_DATATYPE_XATTR,
416                 errorMetaPair.second.datatype,
417                 "Datatype is not XATTR");
418     }
419 
420     testHarness->destroy_cookie(cookie);
421 
422     return SUCCESS;
423 }
424 
425 /**
426  * Test that we can still get datatype of the deleted item after compaction
427  */
test_get_meta_mb23905(EngineIface* h)428 static enum test_result test_get_meta_mb23905(EngineIface* h) {
429     const char* key = "get_meta_key";
430     std::vector<char> data = createXattrValue({"test_expiry_value"});
431 
432     const void* cookie = testHarness->create_cookie();
433 
434     checkeq(cb::engine_errc::success,
435             storeCasVb11(h,
436                          cookie,
437                          OPERATION_SET,
438                          key,
439                          reinterpret_cast<char*>(data.data()),
440                          data.size(),
441                          9258,
442                          0,
443                          Vbid(0),
444                          0,
445                          PROTOCOL_BINARY_DATATYPE_XATTR)
446                     .first,
447             "Failed to store xattr document");
448 
449     if (isPersistentBucket(h)) {
450         wait_for_flusher_to_settle(h);
451     }
452 
453     if (isPersistentBucket(h)) {
454         cb::xattr::Blob systemXattrBlob;
455         systemXattrBlob.set("_sync", "{\"cas\":\"0xdeadbeefcafefeed\"}");
456         auto deletedValue = systemXattrBlob.finalize();
457 
458         checkeq(ENGINE_SUCCESS,
459                 delete_with_value(h,
460                                   cookie,
461                                   0,
462                                   key,
463                                   {reinterpret_cast<char*>(deletedValue.data()),
464                                    deletedValue.size()},
465                                   cb::mcbp::Datatype::Xattr),
466                 "delete_with_value() failed");
467 
468         // Run compaction to start using the bloomfilter
469         useconds_t sleepTime = 128;
470         compact_db(h, Vbid(0), Vbid(0), 1, 1, 0);
471         while (get_int_stat(h, "ep_pending_compactions") != 0) {
472             decayingSleep(&sleepTime);
473         }
474 
475         cb::EngineErrorMetadataPair errorMetaPair;
476         check(get_meta(h, key, errorMetaPair, cookie),
477               "Get meta command failed");
478         checkeq(PROTOCOL_BINARY_DATATYPE_XATTR,
479                 errorMetaPair.second.datatype,
480                 "Datatype is not XATTR");
481         checkeq(DocumentState::Deleted,
482                 errorMetaPair.second.document_state,
483                 "Expected deleted flag to be set");
484     }
485 
486     testHarness->destroy_cookie(cookie);
487 
488     return SUCCESS;
489 }
490 
test_add_with_meta(EngineIface* h)491 static enum test_result test_add_with_meta(EngineIface* h) {
492     const char *key = "mykey";
493     const size_t keylen = strlen(key);
494     ItemMetaData itemMeta;
495     int temp = 0;
496 
497     // put some random metadata
498     itemMeta.revSeqno = 10;
499     itemMeta.cas = 0xdeadbeef;
500     itemMeta.exptime = 0;
501     itemMeta.flags = 0xdeadbeef;
502     // check the stat
503     temp = get_int_stat(h, "ep_num_ops_set_meta");
504     checkeq(0, temp, "Expect zero setMeta ops");
505 
506     // store an item with meta data
507     checkeq(ENGINE_SUCCESS,
508             add_with_meta(h, key, keylen, NULL, 0, Vbid(0), &itemMeta),
509             "Expected to add item");
510     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
511 
512     // store the item again, expect key exists
513     checkeq(ENGINE_KEY_EEXISTS,
514             add_with_meta(h, key, keylen, NULL, 0, Vbid(0), &itemMeta, true),
515             "Expected add to fail when the item exists already");
516     // check the stat
517     temp = get_int_stat(h, "ep_num_ops_set_meta");
518     checkeq(1, temp, "Failed op does not count");
519 
520     return SUCCESS;
521 }
522 
test_delete_with_meta(EngineIface* h)523 static enum test_result test_delete_with_meta(EngineIface* h) {
524     const char *key1 = "delete_with_meta_key1";
525     const char *key2 = "delete_with_meta_key2";
526     const char *key3 = "delete_with_meta_key3";
527     const size_t keylen = strlen(key1);
528     ItemMetaData itemMeta;
529     uint64_t vb_uuid;
530     uint32_t high_seqno;
531     // check the stat
532     auto temp = get_int_stat(h, "ep_num_ops_del_meta");
533     checkeq(0, temp, "Expect zero setMeta ops");
534 
535     // put some random meta data
536     itemMeta.revSeqno = 10;
537     itemMeta.cas = 0xdeadbeef;
538     itemMeta.exptime = 0;
539     itemMeta.flags = 0xdeadbeef;
540 
541     // store an item
542     checkeq(ENGINE_SUCCESS,
543             store(h, NULL, OPERATION_SET, key1, "somevalue"),
544             "Failed set.");
545 
546     checkeq(ENGINE_SUCCESS,
547             store(h, NULL, OPERATION_SET, key2, "somevalue2"),
548             "Failed set.");
549 
550     checkeq(ENGINE_SUCCESS,
551             store(h, NULL, OPERATION_SET, key3, "somevalue3"),
552             "Failed set.");
553 
554     vb_uuid = get_ull_stat(h, "vb_0:0:id", "failovers");
555     high_seqno = get_ull_stat(h, "vb_0:high_seqno", "vbucket-seqno");
556 
557     const void* cookie = testHarness->create_cookie();
558 
559     // delete an item with meta data
560     checkeq(ENGINE_SUCCESS,
561             del_with_meta(h,
562                           key1,
563                           keylen,
564                           Vbid(0),
565                           &itemMeta,
566                           0 /*cas*/,
567                           0 /*options*/,
568                           cookie),
569             "Expected delete OK");
570 
571     checkeq(last_uuid.load(), vb_uuid, "Expected valid vbucket uuid");
572     checkeq(last_seqno.load(),
573             static_cast<uint64_t>(high_seqno + 1),
574             "Expected valid sequence number");
575     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
576     // check the stat
577     temp = get_int_stat(h, "ep_num_ops_del_meta");
578     checkeq(1, temp, "Expect more setMeta ops");
579 
580     testHarness->set_mutation_extras_handling(cookie, false);
581 
582     // delete an item with meta data
583     checkeq(ENGINE_SUCCESS,
584             del_with_meta(h,
585                           key2,
586                           keylen,
587                           Vbid(0),
588                           &itemMeta,
589                           0 /*cas*/,
590                           0 /*options*/,
591                           cookie),
592             "Expected delete OK");
593 
594     checkeq(last_uuid.load(), vb_uuid, "Expected valid vbucket uuid");
595     checkeq(last_seqno.load(),
596             static_cast<uint64_t>(high_seqno + 1),
597             "Expected valid sequence number");
598     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
599 
600     // delete an item with meta data
601     checkeq(ENGINE_SUCCESS,
602             del_with_meta(h, key3, keylen, Vbid(0), &itemMeta),
603             "Expected delete OK");
604 
605     checkeq(last_uuid.load(), vb_uuid, "Expected valid vbucket uuid");
606     checkeq(last_seqno.load(),
607             static_cast<uint64_t>(high_seqno + 3),
608             "Expected valid sequence number");
609     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
610 
611     testHarness->destroy_cookie(cookie);
612     return SUCCESS;
613 }
614 
test_delete_with_meta_deleted(EngineIface* h)615 static enum test_result test_delete_with_meta_deleted(EngineIface* h) {
616     const char *key = "delete_with_meta_key";
617     const size_t keylen = strlen(key);
618 
619     // check the stat
620     checkeq(0,
621             get_int_stat(h, "ep_num_ops_del_meta"),
622             "Expect zero setMeta ops");
623 
624     // add a key
625     checkeq(ENGINE_SUCCESS,
626             store(h, NULL, OPERATION_SET, key, "somevalue"),
627             "Failed set.");
628     wait_for_flusher_to_settle(h);
629 
630     // delete the key
631     checkeq(ENGINE_SUCCESS, del(h, key, 0, Vbid(0)), "Delete failed");
632     wait_for_flusher_to_settle(h);
633     wait_for_stat_to_be(h, "curr_items", 0);
634 
635     cb::EngineErrorMetadataPair errorMetaPair;
636 
637     // get metadata of deleted key
638     check(get_meta(h, key, errorMetaPair), "Expected to get meta");
639     checkeq(DocumentState::Deleted,
640             errorMetaPair.second.document_state,
641             "Expected deleted flag to be set");
642     checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
643     checkPersistentBucketTempItems(h, 1);
644 
645     // this is the cas to be used with a subsequent delete with meta
646     uint64_t valid_cas = last_cas;
647     uint64_t invalid_cas = 2012;
648     // put some random metadata and delete the item with new meta data
649     ItemMetaData itm_meta(
650             0xdeadbeef, 10, 0xdeadbeef, 1735689600); // expires in 2025
651 
652     // do delete with meta with an incorrect cas value. should fail.
653     checkeq(ENGINE_KEY_EEXISTS,
654             del_with_meta(h, key, keylen, Vbid(0), &itm_meta, invalid_cas),
655             "Expected invalid cas error");
656     checkeq(0,
657             get_int_stat(h, "ep_num_ops_del_meta"),
658             "Faild ops does not count");
659     checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
660     checkPersistentBucketTempItems(h, 1);
661 
662     // do delete with meta with the correct cas value. should pass.
663     checkeq(ENGINE_SUCCESS,
664             del_with_meta(h, key, keylen, Vbid(0), &itm_meta, valid_cas),
665             "Expected delete oK");
666     wait_for_flusher_to_settle(h);
667 
668     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
669     checkeq(1, get_int_stat(h, "ep_num_ops_del_meta"), "Expect some ops");
670     wait_for_stat_to_be(h, "curr_items", 0);
671     checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
672 
673     // get metadata again to verify that delete with meta was successful
674     check(get_meta(h, key, errorMetaPair), "Expected to get meta");
675     checkeq(DocumentState::Deleted,
676             errorMetaPair.second.document_state,
677             "Expected deleted flag to be set");
678     checkeq(static_cast<uint64_t>(itm_meta.revSeqno), errorMetaPair.second.seqno,
679             "Expected seqno to match");
680     checkeq(itm_meta.cas, errorMetaPair.second.cas, "Expected cas to match");
681     checkeq(itm_meta.flags, errorMetaPair.second.flags,
682             "Expected flags to match");
683 
684     checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
685     checkPersistentBucketTempItems(h, 1);
686 
687     return SUCCESS;
688 }
689 
test_delete_with_meta_nonexistent(EngineIface* h)690 static enum test_result test_delete_with_meta_nonexistent(EngineIface* h) {
691     const char *key = "delete_with_meta_key";
692     const size_t keylen = strlen(key);
693 
694     // check the stat
695     checkeq(0,
696             get_int_stat(h, "ep_num_ops_del_meta"),
697             "Expect zero setMeta ops");
698 
699     cb::EngineErrorMetadataPair errorMetaPair;
700 
701     // get metadata of nonexistent key
702     check(!get_meta(h, key, errorMetaPair),
703           "Expected get meta to return false");
704     checkeq(cb::engine_errc::no_such_key,
705             errorMetaPair.first,
706             "Expected no_such_key");
707     checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
708 
709     // this is the cas to be used with a subsequent delete with meta
710     uint64_t valid_cas = last_cas;
711     uint64_t invalid_cas = 2012;
712 
713     // do delete with meta
714     // put some random metadata and delete the item with new meta data
715     ItemMetaData itm_meta(
716             0xdeadbeef, 10, 0xdeadbeef, 1735689600); // expires in 2025
717 
718     // do delete with meta with an incorrect cas value. should fail.
719     checkeq(ENGINE_KEY_EEXISTS,
720             del_with_meta(h, key, keylen, Vbid(0), &itm_meta, invalid_cas),
721             "Expected invalid cas error");
722     // check the stat
723     checkeq(0,
724             get_int_stat(h, "ep_num_ops_del_meta"),
725             "Failed op does not count");
726     checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
727     checkPersistentBucketTempItems(h, 1);
728 
729     // do delete with meta with the correct cas value. should pass.
730     checkeq(ENGINE_SUCCESS,
731             del_with_meta(h, key, keylen, Vbid(0), &itm_meta, valid_cas),
732             "Expected delete OK");
733     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
734     wait_for_flusher_to_settle(h);
735 
736     // check the stat
737     checkeq(1, get_int_stat(h, "ep_num_ops_del_meta"), "Expect one op");
738     wait_for_stat_to_be(h, "curr_items", 0);
739     checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
740 
741     // get metadata again to verify that delete with meta was successful
742     check(get_meta(h, key, errorMetaPair), "Expected to get meta");
743     checkeq(DocumentState::Deleted,
744             errorMetaPair.second.document_state,
745             "Expected deleted flag to be set");
746     checkeq(static_cast<uint64_t>(itm_meta.revSeqno),
747             errorMetaPair.second.seqno,
748             "Expected seqno to match");
749     checkeq(itm_meta.cas, errorMetaPair.second.cas, "Expected cas to match");
750     checkeq(itm_meta.flags, errorMetaPair.second.flags,
751             "Expected flags to match");
752 
753     checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
754     checkPersistentBucketTempItems(h, 1);
755 
756     return SUCCESS;
757 }
758 
test_delete_with_meta_nonexistent_no_temp( EngineIface* h)759 static enum test_result test_delete_with_meta_nonexistent_no_temp(
760         EngineIface* h) {
761     const char *key1 = "delete_with_meta_no_temp_key1";
762     const size_t keylen1 = strlen(key1);
763     ItemMetaData itm_meta1;
764 
765     // Run compaction to start using the bloomfilter
766     useconds_t sleepTime = 128;
767     compact_db(h, Vbid(0), Vbid(0), 1, 1, 0);
768     while (get_int_stat(h, "ep_pending_compactions") != 0) {
769         decayingSleep(&sleepTime);
770     }
771 
772     // put some random metadata and delete the item with new meta data
773     itm_meta1.revSeqno = 10;
774     itm_meta1.cas = 0xdeadbeef;
775     itm_meta1.exptime = 1735689600; // expires in 2025
776     itm_meta1.flags = 0xdeadbeef;
777 
778     // do delete with meta with the correct cas value.
779     // skipConflictResolution false
780     checkeq(ENGINE_SUCCESS,
781             del_with_meta(h, key1, keylen1, Vbid(0), &itm_meta1, 0, false),
782             "Expected delete OK");
783     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
784     wait_for_flusher_to_settle(h);
785 
786     checkeq(1, get_int_stat(h, "ep_num_ops_del_meta"), "Expect one op");
787     wait_for_stat_to_be(h, "curr_items", 0);
788     checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
789 
790     // do delete with meta with the correct cas value.
791     // skipConflictResolution true
792     const char *key2 = "delete_with_meta_no_temp_key2";
793     const size_t keylen2 = strlen(key2);
794     ItemMetaData itm_meta2;
795 
796     // put some random metadata and delete the item with new meta data
797     itm_meta2.revSeqno = 10;
798     itm_meta2.cas = 0xdeadbeef;
799     itm_meta2.exptime = 1735689600; // expires in 2025
800     itm_meta2.flags = 0xdeadbeef;
801 
802     checkeq(ENGINE_SUCCESS,
803             del_with_meta(h, key2, keylen2, Vbid(0), &itm_meta2, 0, true),
804             "Expected delete OK");
805     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
806     wait_for_flusher_to_settle(h);
807 
808     checkeq(2, get_int_stat(h, "ep_num_ops_del_meta"), "Expect one op");
809     wait_for_stat_to_be(h, "curr_items", 0);
810     checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
811 
812     return SUCCESS;
813 }
814 
test_delete_with_meta_race_with_set(EngineIface* h)815 static enum test_result test_delete_with_meta_race_with_set(EngineIface* h) {
816     char const *key1 = "key1";
817     const size_t keylen1 = strlen(key1);
818 
819     ItemMetaData itm_meta;
820     itm_meta.revSeqno = 10;
821     itm_meta.cas = 0xdeadbeef;
822     itm_meta.exptime = 1735689600; // expires in 2025
823     itm_meta.flags = 0xdeadbeef;
824     // check the stat
825     size_t temp = get_int_stat(h, "ep_num_ops_del_meta");
826     checkeq(size_t{0}, temp, "Expect zero ops");
827 
828     //
829     // test race with a concurrent set for an existing key. should fail.
830     //
831 
832     // create a new key and do get_meta
833     checkeq(ENGINE_SUCCESS,
834             store(h, NULL, OPERATION_SET, key1, "somevalue"),
835             "Failed set.");
836     wait_for_flusher_to_settle(h);
837 
838     cb::EngineErrorMetadataPair errorMetaPair;
839 
840     check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
841 
842     // do a concurrent set that changes the cas
843     checkeq(ENGINE_SUCCESS,
844             store(h, NULL, OPERATION_SET, key1, "someothervalue"),
845             "Failed set.");
846 
847     // attempt delete_with_meta. should fail since cas is no longer valid.
848     checkeq(ENGINE_KEY_EEXISTS,
849             del_with_meta(h,
850                           key1,
851                           keylen1,
852                           Vbid(0),
853                           &itm_meta,
854                           errorMetaPair.second.cas),
855             "Expected invalid cas error");
856     // check the stat
857     temp = get_int_stat(h, "ep_num_ops_del_meta");
858     checkeq(size_t{0}, temp, "Failed op does not count");
859 
860     //
861     // test race with a concurrent set for a deleted key. should fail.
862     //
863 
864     // do get_meta for the deleted key
865     checkeq(ENGINE_SUCCESS, del(h, key1, 0, Vbid(0)), "Delete failed");
866     wait_for_flusher_to_settle(h);
867 
868     check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
869     checkeq(DocumentState::Deleted,
870             errorMetaPair.second.document_state,
871             "Expected deleted flag to be set");
872 
873     // do a concurrent set that changes the cas
874     checkeq(ENGINE_SUCCESS,
875             store(h, NULL, OPERATION_SET, key1, "someothervalue"),
876             "Failed set.");
877 
878     checkeq(ENGINE_KEY_EEXISTS,
879             del_with_meta(h,
880                           key1,
881                           keylen1,
882                           Vbid(0),
883                           &itm_meta,
884                           errorMetaPair.second.cas),
885             "Expected invalid cas error");
886     // check the stat
887     temp = get_int_stat(h, "ep_num_ops_del_meta");
888     checkeq(size_t{0}, temp, "Failed op does not count");
889 
890     return SUCCESS;
891 }
892 
test_delete_with_meta_race_with_delete(EngineIface* h)893 static enum test_result test_delete_with_meta_race_with_delete(EngineIface* h) {
894     char const *key1 = "key1";
895     uint16_t keylen1 = (uint16_t)strlen(key1);
896     char const *key2 = "key2";
897     uint16_t keylen2 = (uint16_t)strlen(key2);
898 
899     // check the stat
900     size_t temp = get_int_stat(h, "ep_num_ops_del_meta");
901     checkeq(size_t{0}, temp, "Expect zero ops");
902 
903     //
904     // test race with a concurrent delete for an existing key. should fail.
905     //
906 
907     // create a new key and do get_meta
908     checkeq(ENGINE_SUCCESS,
909             store(h, NULL, OPERATION_SET, key1, "somevalue"),
910             "Failed set.");
911     wait_for_flusher_to_settle(h);
912 
913     cb::EngineErrorMetadataPair errorMetaPair;
914 
915     check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
916 
917     //Store the CAS. This will be used in a subsequent delete_with_meta call
918     uint64_t cas_from_store = errorMetaPair.second.cas;
919 
920     //Do a concurrent delete. This should modify the CAS
921     checkeq(ENGINE_SUCCESS, del(h, key1, 0, Vbid(0)), "Delete failed");
922 
923     //Get the latest meta data
924     check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
925 
926     //Populate the item meta data in such a way, so that we will pass
927     //conflict resolution
928     ItemMetaData itm_meta(errorMetaPair.second.cas,
929                           errorMetaPair.second.seqno + 1,
930                           errorMetaPair.second.flags,
931                           errorMetaPair.second.exptime);
932 
933     // attempt delete_with_meta. should fail since cas is no longer valid.
934     checkeq(ENGINE_KEY_EEXISTS,
935             del_with_meta(h, key1, keylen1, Vbid(0), &itm_meta, cas_from_store),
936             "Expected invalid cas error");
937     // check the stat
938     temp = get_int_stat(h, "ep_num_ops_del_meta");
939     checkeq(size_t{0}, temp, "Failed op does not count");
940 
941     //
942     // test race with a concurrent delete for a deleted key. should pass since
943     // the delete itself will fail.
944     //
945 
946     // do get_meta for the deleted key
947     wait_for_flusher_to_settle(h);
948     check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
949     checkeq(DocumentState::Deleted,
950             errorMetaPair.second.document_state,
951             "Expected deleted flag to be set");
952 
953     // do a concurrent delete
954     checkeq(ENGINE_KEY_ENOENT, del(h, key1, 0, Vbid(0)), "Delete failed");
955 
956     // attempt delete_with_meta. should pass.
957     checkeq(ENGINE_SUCCESS,
958             del_with_meta(h, key1, keylen1, Vbid(0), &itm_meta, last_cas),
959             "Expected delete OK");
960     checkeq(cb::mcbp::Status::Success, last_status.load(),
961           "Expected delete_with_meta success");
962     // check the stat
963     temp = get_int_stat(h, "ep_num_ops_del_meta");
964     checkeq(size_t{1}, temp, "Expect some ops");
965 
966     //
967     // test race with a concurrent delete for a nonexistent key. should pass
968     // since the delete itself will fail.
969     //
970 
971     // do get_meta for a nonexisting key
972     check(!get_meta(h, key2, errorMetaPair),
973           "Expected get meta to return false");
974     checkeq(cb::engine_errc::no_such_key,
975             errorMetaPair.first,
976             "Expected no_such_key");
977 
978     // do a concurrent delete
979     checkeq(ENGINE_KEY_ENOENT, del(h, key1, 0, Vbid(0)), "Delete failed");
980 
981     // attempt delete_with_meta. should pass.
982     checkeq(ENGINE_SUCCESS,
983             del_with_meta(h,
984                           key2,
985                           keylen2,
986                           Vbid(0),
987                           &itm_meta,
988                           errorMetaPair.second.cas),
989             "Expected delete OK");
990     checkeq(cb::mcbp::Status::Success, last_status.load(),
991           "Expected delete_with_meta success");
992     // check the stat
993     temp = get_int_stat(h, "ep_num_ops_del_meta");
994     checkeq(size_t{2}, temp, "Expect some ops");
995 
996     return SUCCESS;
997 }
998 
test_set_with_meta(EngineIface* h)999 static enum test_result test_set_with_meta(EngineIface* h) {
1000     const char* key = "set_with_meta_key";
1001     size_t keylen = strlen(key);
1002     const char* val = "somevalue";
1003     const char* newVal = R"({"json":"yes"})";
1004     size_t newValLen = strlen(newVal);
1005     uint64_t vb_uuid;
1006     uint32_t high_seqno;
1007 
1008     // check the stat
1009     checkeq(0, get_int_stat(h, "ep_num_ops_set_meta"), "Expect zero ops");
1010     checkeq(0,
1011             get_int_stat(h, "ep_num_ops_get_meta_on_set_meta"),
1012             "Expect zero ops");
1013     checkeq(0, get_int_stat(h, "curr_items"), "Expect zero items");
1014     checkeq(0, get_int_stat(h, "curr_temp_items"), "Expect zero temp items");
1015 
1016     // create a new key
1017     checkeq(ENGINE_SUCCESS,
1018             store(h, NULL, OPERATION_SET, key, val),
1019             "Failed set.");
1020     wait_for_flusher_to_settle(h);
1021 
1022     // get metadata for the key
1023     cb::EngineErrorMetadataPair errorMetaPair;
1024     check(get_meta(h, key, errorMetaPair), "Expected to get meta");
1025     checkeq(1, get_int_stat(h, "curr_items"), "Expect one item");
1026     checkeq(0, get_int_stat(h, "curr_temp_items"), "Expect zero temp item");
1027 
1028     // this is the cas to be used with a subsequent set with meta
1029     uint64_t cas_for_set = errorMetaPair.second.cas;
1030     // init some random metadata
1031     ItemMetaData itm_meta(0xdeadbeef, 10, 0xdeadbeef, time(NULL) + 300);
1032 
1033     char *bigValue = new char[32*1024*1024];
1034     // do set with meta with the value size bigger than the max size allowed.
1035     checkeq(ENGINE_E2BIG,
1036             set_with_meta(h,
1037                           key,
1038                           keylen,
1039                           bigValue,
1040                           32 * 1024 * 1024,
1041                           Vbid(0),
1042                           &itm_meta,
1043                           cas_for_set),
1044             "Expected the max value size exceeding error");
1045     delete []bigValue;
1046 
1047     // do set with meta with an incorrect cas value. should fail.
1048     checkeq(ENGINE_KEY_EEXISTS,
1049             set_with_meta(h,
1050                           key,
1051                           keylen,
1052                           newVal,
1053                           newValLen,
1054                           Vbid(0),
1055                           &itm_meta,
1056                           1229),
1057             "Expected invalid cas error");
1058     // check the stat
1059     checkeq(0,
1060             get_int_stat(h, "ep_num_ops_set_meta"),
1061             "Failed op does not count");
1062 
1063     vb_uuid = get_ull_stat(h, "vb_0:0:id", "failovers");
1064     high_seqno = get_ull_stat(h, "vb_0:high_seqno", "vbucket-seqno");
1065 
1066     const void* cookie = testHarness->create_cookie();
1067     // We are explicitly going to test the !datatype paths, so turn it off.
1068     testHarness->set_datatype_support(cookie, false);
1069 
1070     // do set with meta with the correct cas value. should pass.
1071     checkeq(ENGINE_SUCCESS,
1072             set_with_meta(h,
1073                           key,
1074                           keylen,
1075                           newVal,
1076                           newValLen,
1077                           Vbid(0),
1078                           &itm_meta,
1079                           cas_for_set,
1080                           0,
1081                           0,
1082                           cookie),
1083             "Expected item to be stored");
1084     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
1085     checkeq(last_uuid.load(), vb_uuid, "Expected valid vbucket uuid");
1086     checkeq(last_seqno.load(),
1087             static_cast<uint64_t>(high_seqno + 1),
1088             "Expected valid sequence number");
1089 
1090     // Check that set_with_meta has marked the JSON input as JSON
1091     item_info info;
1092     check(get_item_info(h, &info, key, Vbid(0)), "get_item_info failed");
1093     checkeq(int(PROTOCOL_BINARY_DATATYPE_JSON), int(info.datatype),
1094         "Expected datatype to now include JSON");
1095 
1096     // check the stat
1097     checkeq(1, get_int_stat(h, "ep_num_ops_set_meta"), "Expect some ops");
1098     checkeq(1, get_int_stat(h, "curr_items"), "Expect one item");
1099     checkeq(0, get_int_stat(h, "curr_temp_items"), "Expect zero temp item");
1100 
1101     // get metadata again to verify that set with meta was successful
1102     check(get_meta(h, key, errorMetaPair), "Expected to get meta");
1103     checkeq(uint64_t{10},
1104             errorMetaPair.second.seqno,
1105             "Expected seqno to match");
1106     checkeq(uint64_t{0xdeadbeef},
1107             errorMetaPair.second.cas,
1108             "Expected cas to match");
1109     checkeq(uint32_t{0xdeadbeef},
1110             errorMetaPair.second.flags,
1111             "Expected flags to match");
1112 
1113     //disable getting vb uuid and seqno as extras
1114     testHarness->set_mutation_extras_handling(cookie, false);
1115     itm_meta.revSeqno++;
1116     cas_for_set = errorMetaPair.second.cas;
1117     checkeq(ENGINE_SUCCESS,
1118             set_with_meta(h,
1119                           key,
1120                           keylen,
1121                           newVal,
1122                           newValLen,
1123                           Vbid(0),
1124                           &itm_meta,
1125                           cas_for_set,
1126                           false,
1127                           0,
1128                           cookie),
1129             "Expected item to be stored");
1130     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
1131     checkeq(last_uuid.load(), vb_uuid, "Expected same vbucket uuid");
1132     checkeq(last_seqno.load(),
1133             static_cast<uint64_t>(high_seqno + 1),
1134             "Expected same sequence number");
1135 
1136     itm_meta.revSeqno++;
1137     cas_for_set = last_meta.cas;
1138     checkeq(ENGINE_SUCCESS,
1139             set_with_meta(h,
1140                           key,
1141                           keylen,
1142                           newVal,
1143                           newValLen,
1144                           Vbid(0),
1145                           &itm_meta,
1146                           cas_for_set),
1147             "Expected item to be stored");
1148     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
1149     checkeq(last_uuid.load(), vb_uuid, "Expected valid vbucket uuid");
1150     checkeq(last_seqno.load(),
1151             static_cast<uint64_t>(high_seqno + 3),
1152             "Expected valid sequence number");
1153 
1154     // Make sure the item expiration was processed correctly
1155     testHarness->time_travel(301);
1156     auto ret = get(h, NULL, key, Vbid(0));
1157     checkeq(cb::engine_errc::no_such_key, ret.first, "Failed to get value.");
1158 
1159     testHarness->destroy_cookie(cookie);
1160     return SUCCESS;
1161 }
1162 
test_set_with_meta_by_force(EngineIface* h)1163 static enum test_result test_set_with_meta_by_force(EngineIface* h) {
1164     const char* key = "set_with_meta_key";
1165     size_t keylen = strlen(key);
1166     const char* val = "somevalue";
1167 
1168     // init some random metadata
1169     ItemMetaData itm_meta(0xdeadbeef, 10, 0xdeadbeef, time(NULL) + 300);
1170 
1171     // Pass true to force SetWithMeta.
1172     checkeq(ENGINE_SUCCESS,
1173             set_with_meta(h,
1174                           key,
1175                           keylen,
1176                           val,
1177                           strlen(val),
1178                           Vbid(0),
1179                           &itm_meta,
1180                           0,
1181                           true),
1182             "Expected item to be stored");
1183     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
1184     wait_for_flusher_to_settle(h);
1185 
1186     // get metadata again to verify that the warmup loads an item correctly.
1187     cb::EngineErrorMetadataPair errorMetaPair;
1188     check(get_meta(h, key, errorMetaPair), "Expected to get meta");
1189     checkeq(uint64_t{10},
1190             errorMetaPair.second.seqno,
1191             "Expected seqno to match");
1192     checkeq(uint64_t{0xdeadbeef},
1193             errorMetaPair.second.cas,
1194             "Expected cas to match");
1195     checkeq(uint32_t{0xdeadbeef},
1196             errorMetaPair.second.flags,
1197             "Expected flags to match");
1198 
1199     check_key_value(h, key, val, strlen(val));
1200 
1201     return SUCCESS;
1202 }
1203 
test_set_with_meta_deleted(EngineIface* h)1204 static enum test_result test_set_with_meta_deleted(EngineIface* h) {
1205     const char* key = "set_with_meta_key";
1206     size_t keylen = strlen(key);
1207     const char* val = "somevalue";
1208     const char* newVal = "someothervalue";
1209     uint16_t newValLen = (uint16_t)strlen(newVal);
1210 
1211     // check the stat
1212     checkeq(0, get_int_stat(h, "ep_num_ops_set_meta"), "Expect zero ops");
1213     checkeq(0,
1214             get_int_stat(h, "ep_num_ops_get_meta_on_set_meta"),
1215             "Expect zero ops");
1216 
1217     // create a new key
1218     checkeq(ENGINE_SUCCESS,
1219             store(h, NULL, OPERATION_SET, key, val),
1220             "Failed set.");
1221     wait_for_flusher_to_settle(h);
1222     checkeq(1, get_int_stat(h, "curr_items"), "Expected single curr_items");
1223     checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
1224 
1225     // delete the key
1226     checkeq(ENGINE_SUCCESS, del(h, key, 0, Vbid(0)), "Delete failed");
1227     wait_for_flusher_to_settle(h);
1228     wait_for_stat_to_be(h, "curr_items", 0);
1229 
1230     cb::EngineErrorMetadataPair errorMetaPair;
1231 
1232     // get metadata for the key
1233     check(get_meta(h, key, errorMetaPair), "Expected to get meta");
1234     checkeq(DocumentState::Deleted,
1235             errorMetaPair.second.document_state,
1236             "Expected deleted flag to be set");
1237     checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
1238     checkPersistentBucketTempItems(h, 1);
1239 
1240     // this is the cas to be used with a subsequent set with meta
1241     uint64_t cas_for_set = errorMetaPair.second.cas;
1242     // init some random metadata
1243     ItemMetaData itm_meta(
1244             0xdeadbeef, 10, 0xdeadbeef, 1735689600); // expires in 2025
1245 
1246     // do set_with_meta with an incorrect cas for a deleted item. should fail.
1247     checkeq(ENGINE_KEY_ENOENT,
1248             set_with_meta(h,
1249                           key,
1250                           keylen,
1251                           newVal,
1252                           newValLen,
1253                           Vbid(0),
1254                           &itm_meta,
1255                           1229),
1256             "Expected key_not_found error");
1257     // check the stat
1258     checkeq(0,
1259             get_int_stat(h, "ep_num_ops_set_meta"),
1260             "Failed op does not count");
1261     checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
1262     checkPersistentBucketTempItems(h, 1);
1263 
1264     // do set with meta with the correct cas value. should pass.
1265     checkeq(ENGINE_SUCCESS,
1266             set_with_meta(h,
1267                           key,
1268                           keylen,
1269                           newVal,
1270                           newValLen,
1271                           Vbid(0),
1272                           &itm_meta,
1273                           cas_for_set),
1274             "Expected item to be stored");
1275     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
1276     wait_for_flusher_to_settle(h);
1277 
1278     // check the stat
1279     checkeq(1, get_int_stat(h, "ep_num_ops_set_meta"), "Expect some ops");
1280     checkeq(0,
1281             get_int_stat(h, "ep_num_ops_get_meta_on_set_meta"),
1282             "Expect some ops");
1283     checkeq(1, get_int_stat(h, "curr_items"), "Expected single curr_items");
1284     checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
1285 
1286     // get metadata again to verify that set with meta was successful
1287     check(get_meta(h, key, errorMetaPair), "Expected to get meta");
1288     ItemMetaData metadata(0xdeadbeef, 10, 0xdeadbeef, 1735689600);
1289     verifyMetaData(metadata, errorMetaPair.second);
1290     checkeq(1, get_int_stat(h, "curr_items"), "Expected single curr_items");
1291     checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
1292 
1293     return SUCCESS;
1294 }
1295 
test_set_with_meta_nonexistent(EngineIface* h)1296 static enum test_result test_set_with_meta_nonexistent(EngineIface* h) {
1297     const char* key = "set_with_meta_key";
1298     size_t keylen = strlen(key);
1299     const char* val = "somevalue";
1300     size_t valLen = strlen(val);
1301 
1302     // check the stat
1303     checkeq(0, get_int_stat(h, "ep_num_ops_set_meta"), "Expect zero ops");
1304 
1305     cb::EngineErrorMetadataPair errorMetaPair;
1306 
1307     // get metadata for the key
1308     check(!get_meta(h, key, errorMetaPair),
1309           "Expected get meta to return false");
1310     checkeq(cb::engine_errc::no_such_key,
1311             errorMetaPair.first,
1312             "Expected no_such_key");
1313     checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
1314 
1315     // this is the cas to be used with a subsequent set with meta
1316     uint64_t cas_for_set = errorMetaPair.second.cas;
1317     // init some random metadata
1318     ItemMetaData itm_meta(
1319             0xdeadbeef, 10, 0xdeadbeef, 1735689600); // expires in 2025
1320 
1321     // do set_with_meta with an incorrect cas for a non-existent item. should fail.
1322     checkeq(ENGINE_KEY_ENOENT,
1323             set_with_meta(
1324                     h, key, keylen, val, valLen, Vbid(0), &itm_meta, 1229),
1325             "Expected key_not_found error");
1326     // check the stat
1327     checkeq(0,
1328             get_int_stat(h, "ep_num_ops_set_meta"),
1329             "Failed op does not count");
1330     checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
1331 
1332     // do set with meta with the correct cas value. should pass.
1333     checkeq(ENGINE_SUCCESS,
1334             set_with_meta(h,
1335                           key,
1336                           keylen,
1337                           val,
1338                           valLen,
1339                           Vbid(0),
1340                           &itm_meta,
1341                           cas_for_set),
1342             "Expected item to be stored");
1343     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
1344     wait_for_flusher_to_settle(h);
1345 
1346     // check the stat
1347     checkeq(1, get_int_stat(h, "ep_num_ops_set_meta"), "Expect some ops");
1348     checkeq(1, get_int_stat(h, "curr_items"), "Expected single curr_items");
1349     checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
1350 
1351     // get metadata again to verify that set with meta was successful
1352     check(get_meta(h, key, errorMetaPair), "Expected to get meta");
1353     ItemMetaData metadata(0xdeadbeef, 10, 0xdeadbeef, 1735689600);
1354     verifyMetaData(metadata, errorMetaPair.second);
1355     checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
1356     checkeq(1, get_int_stat(h, "curr_items"), "Expected single curr_items");
1357 
1358     return SUCCESS;
1359 }
1360 
test_set_with_meta_race_with_set(EngineIface* h)1361 static enum test_result test_set_with_meta_race_with_set(EngineIface* h) {
1362     char const *key1 = "key1";
1363     size_t keylen1 = strlen(key1);
1364     // check the stat
1365     size_t temp = get_int_stat(h, "ep_num_ops_set_meta");
1366     checkeq(size_t{0}, temp, "Expect zero ops");
1367 
1368     //
1369     // test race with a concurrent set for an existing key. should fail.
1370     //
1371 
1372     // create a new key and do get_meta
1373     checkeq(ENGINE_SUCCESS,
1374             store(h, NULL, OPERATION_SET, key1, "somevalue"),
1375             "Failed set.");
1376     wait_for_flusher_to_settle(h);
1377     cb::EngineErrorMetadataPair errorMetaPair;
1378     check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
1379 
1380     // do a concurrent set that changes the cas
1381     checkeq(ENGINE_SUCCESS,
1382             store(h, NULL, OPERATION_SET, key1, "someothervalue"),
1383             "Failed set.");
1384 
1385     // attempt set_with_meta. should fail since cas is no longer valid.
1386     ItemMetaData meta(errorMetaPair.second.cas,
1387                       errorMetaPair.second.seqno + 2,
1388                       errorMetaPair.second.flags,
1389                       errorMetaPair.second.exptime);
1390     checkeq(ENGINE_KEY_EEXISTS,
1391             set_with_meta(h,
1392                           key1,
1393                           keylen1,
1394                           NULL,
1395                           0,
1396                           Vbid(0),
1397                           &meta,
1398                           errorMetaPair.second.cas),
1399             "Expected invalid cas error");
1400     // check the stat
1401     temp = get_int_stat(h, "ep_num_ops_set_meta");
1402     checkeq(size_t{0}, temp, "Failed op does not count");
1403 
1404     //
1405     // test race with a concurrent set for a deleted key. should fail.
1406     //
1407 
1408     // do get_meta for the deleted key
1409     checkeq(ENGINE_SUCCESS, del(h, key1, 0, Vbid(0)), "Delete failed");
1410     wait_for_flusher_to_settle(h);
1411     check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
1412     checkeq(DocumentState::Deleted,
1413             errorMetaPair.second.document_state,
1414             "Expected deleted flag to be set");
1415 
1416     // do a concurrent set that changes the cas
1417     checkeq(ENGINE_SUCCESS,
1418             store(h, NULL, OPERATION_SET, key1, "someothervalue"),
1419             "Failed set.");
1420 
1421     // attempt set_with_meta. should fail since cas is no longer valid.
1422     meta = ItemMetaData(errorMetaPair.second.cas,
1423                         errorMetaPair.second.seqno + 2,
1424                         errorMetaPair.second.flags,
1425                         errorMetaPair.second.exptime);
1426     checkeq(ENGINE_KEY_EEXISTS,
1427             set_with_meta(h,
1428                           key1,
1429                           keylen1,
1430                           NULL,
1431                           0,
1432                           Vbid(0),
1433                           &meta,
1434                           errorMetaPair.second.cas),
1435             "Expected invalid cas error");
1436     // check the stat
1437     temp = get_int_stat(h, "ep_num_ops_set_meta");
1438     checkeq(size_t{0}, temp, "Failed op does not count");
1439 
1440     return SUCCESS;
1441 }
1442 
test_set_with_meta_race_with_delete(EngineIface* h)1443 static enum test_result test_set_with_meta_race_with_delete(EngineIface* h) {
1444     char const *key1 = "key1";
1445     size_t keylen1 = strlen(key1);
1446     char const *key2 = "key2";
1447     size_t keylen2 = strlen(key2);
1448     // check the stat
1449     size_t temp = get_int_stat(h, "ep_num_ops_set_meta");
1450     checkeq(size_t{0}, temp, "Expect zero op");
1451 
1452     //
1453     // test race with a concurrent delete for an existing key. should fail.
1454     //
1455 
1456     // create a new key and do get_meta
1457     checkeq(ENGINE_SUCCESS,
1458             store(h, NULL, OPERATION_SET, key1, "somevalue"),
1459             "Failed set.");
1460     wait_for_flusher_to_settle(h);
1461     cb::EngineErrorMetadataPair errorMetaPair;
1462     check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
1463 
1464     // do a concurrent delete that changes the cas
1465     checkeq(ENGINE_SUCCESS, del(h, key1, 0, Vbid(0)), "Delete failed");
1466 
1467     // attempt set_with_meta. should fail since cas is no longer valid.
1468     ItemMetaData meta(errorMetaPair.second.cas,
1469                       errorMetaPair.second.seqno,
1470                       errorMetaPair.second.flags,
1471                       errorMetaPair.second.exptime);
1472     checkeq(ENGINE_KEY_ENOENT,
1473             set_with_meta(h,
1474                           key1,
1475                           keylen1,
1476                           NULL,
1477                           0,
1478                           Vbid(0),
1479                           &meta,
1480                           errorMetaPair.second.cas,
1481                           true),
1482             (std::string{"Expected invalid cas error (KEY_EXISTS or"
1483                          " KEY_ENOENT), got: "} +
1484              ::to_string(last_status.load()))
1485                     .c_str());
1486 
1487     // check the stat
1488     temp = get_int_stat(h, "ep_num_ops_set_meta");
1489     checkeq(size_t{0}, temp, "Expect zero op");
1490 
1491     //
1492     // test race with a concurrent delete for a deleted key. should pass since
1493     // the delete will fail.
1494     //
1495 
1496     // do get_meta for the deleted key
1497     wait_for_flusher_to_settle(h);
1498     check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
1499     checkeq(DocumentState::Deleted,
1500             errorMetaPair.second.document_state,
1501             "Expected deleted flag to be set");
1502 
1503     // do a concurrent delete. should fail.
1504     checkeq(ENGINE_KEY_ENOENT, del(h, key1, 0, Vbid(0)), "Delete failed");
1505 
1506     // attempt set_with_meta. should pass since cas is still valid.
1507     meta = ItemMetaData(errorMetaPair.second.cas,
1508                         errorMetaPair.second.seqno,
1509                         errorMetaPair.second.flags,
1510                         errorMetaPair.second.exptime);
1511     checkeq(ENGINE_SUCCESS,
1512             set_with_meta(h,
1513                           key1,
1514                           keylen1,
1515                           NULL,
1516                           0,
1517                           Vbid(0),
1518                           &meta,
1519                           errorMetaPair.second.cas,
1520                           true),
1521             "Expected item to be stored");
1522     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
1523     // check the stat
1524     temp = get_int_stat(h, "ep_num_ops_set_meta");
1525     checkeq(size_t{1}, temp, "Expect some op");
1526 
1527     //
1528     // test race with a concurrent delete for a nonexistent key. should pass
1529     // since the delete will fail.
1530     //
1531 
1532     // do get_meta for a nonexisting key
1533     check(!get_meta(h, key2, errorMetaPair),
1534           "Expected get meta to return false");
1535     checkeq(cb::engine_errc::no_such_key,
1536             errorMetaPair.first,
1537             "Expected no_such_key");
1538 
1539     // do a concurrent delete. should fail.
1540     checkeq(ENGINE_KEY_ENOENT, del(h, key2, 0, Vbid(0)), "Delete failed");
1541 
1542     // Attempt set_with_meta. This should pass as we set a new key passing 0 as
1543     // command CAS.
1544     checkeq(ENGINE_SUCCESS,
1545             set_with_meta(h, key2, keylen2, NULL, 0, Vbid(0), &meta, 0, true),
1546             "Expected item to be stored");
1547     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
1548     // check the stat
1549     temp = get_int_stat(h, "ep_num_ops_set_meta");
1550     checkeq(size_t{2}, temp, "Expect some ops");
1551 
1552     return SUCCESS;
1553 }
1554 
test_set_with_meta_xattr(EngineIface* h)1555 static enum test_result test_set_with_meta_xattr(EngineIface* h) {
1556     const char* key = "set_with_meta_xattr_key";
1557 
1558     // Create XATTR doc with JSON body
1559     std::string value_data = R"({"json":"yes"})";
1560     std::vector<char> data = createXattrValue(value_data);
1561 
1562     const void* cookie = testHarness->create_cookie();
1563 
1564     // store a value (so we can get its metadata)
1565     checkeq(ENGINE_SUCCESS,
1566             store(h, nullptr, OPERATION_SET, key, value_data.c_str()),
1567             "Failed set.");
1568 
1569     cb::EngineErrorMetadataPair errorMetaPair;
1570 
1571     check(get_meta(h, key, errorMetaPair), "Expected to get meta");
1572 
1573     //init the meta data
1574     ItemMetaData itm_meta(errorMetaPair.second.cas,
1575                           errorMetaPair.second.seqno,
1576                           errorMetaPair.second.flags,
1577                           errorMetaPair.second.exptime);
1578 
1579     int force = 0;
1580     if (strstr(testHarness->get_current_testcase()->cfg,
1581                "conflict_resolution_type=lww") != nullptr) {
1582         force = FORCE_ACCEPT_WITH_META_OPS;
1583     }
1584 
1585     // Only enable XATTR
1586     testHarness->set_datatype_support(cookie, PROTOCOL_BINARY_DATATYPE_XATTR);
1587 
1588     // Set with the same meta data but now with the xattr/json value
1589     checkeq(ENGINE_SUCCESS,
1590             set_with_meta(h,
1591                           key,
1592                           strlen(key),
1593                           data.data(),
1594                           data.size(),
1595                           Vbid(0),
1596                           &itm_meta,
1597                           errorMetaPair.second.cas,
1598                           force,
1599                           PROTOCOL_BINARY_DATATYPE_XATTR,
1600                           cookie),
1601             "Expected item to be stored");
1602 
1603     checkeq(cb::mcbp::Status::Success, last_status.load(),
1604             "Expected the set_with_meta to be successful");
1605 
1606     // set_with_meta will mark JSON input as JSON
1607     item_info info;
1608     check(get_item_info(h, &info, key, Vbid(0)), "get_item_info failed");
1609     checkeq(int(PROTOCOL_BINARY_DATATYPE_JSON|PROTOCOL_BINARY_DATATYPE_XATTR),
1610         int(info.datatype),
1611         "Expected datatype to be JSON and XATTR");
1612 
1613     if (isPersistentBucket(h)) {
1614         wait_for_flusher_to_settle(h);
1615         //evict the key
1616         evict_key(h, key);
1617 
1618         //set with the same meta data but now as RAW BYTES.
1619         //This should result in a bg fetch
1620         checkeq(ENGINE_KEY_EEXISTS,
1621                 set_with_meta(h,
1622                               key,
1623                               strlen(key),
1624                               data.data(),
1625                               data.size(),
1626                               Vbid(0),
1627                               &itm_meta,
1628                               last_meta.cas,
1629                               force,
1630                               PROTOCOL_BINARY_RAW_BYTES,
1631                               cookie),
1632                 "Expected return code to be EEXISTS");
1633     }
1634 
1635     testHarness->destroy_cookie(cookie);
1636 
1637     return SUCCESS;
1638 }
1639 
test_delete_with_meta_xattr(EngineIface* h)1640 static enum test_result test_delete_with_meta_xattr(EngineIface* h) {
1641     const char* key1 = "delete_with_meta_xattr_key1";
1642 
1643     const void* cookie = testHarness->create_cookie();
1644 
1645     // Create XATTR doc with a JSON body
1646     // In practice a del_with_meta should come along with only XATTR, but
1647     // the command should work with a complete xattr/body blob
1648     std::string body = R"({"key1":"value","key2":"value"})";
1649     std::vector<char> data = createXattrValue(body);
1650     cb::xattr::Blob xattr({data.data(), data.size()}, false);
1651     xattr.prune_user_keys();
1652     data.resize(xattr.finalize().size());
1653 
1654     checkeq(ENGINE_SUCCESS,
1655             store(h, nullptr, OPERATION_SET, key1, body.data()),
1656             "Failed to store key1.");
1657 
1658     if (isPersistentBucket(h)) {
1659         wait_for_flusher_to_settle(h);
1660     }
1661 
1662     // Get the metadata so we can build a del_with_meta
1663     cb::EngineErrorMetadataPair errorMetaPair;
1664     check(get_meta(h, key1, errorMetaPair), "Failed get_meta(key1)");
1665 
1666     // Init the meta data for a successful delete
1667     ItemMetaData itm_meta(
1668             errorMetaPair.second.cas + 1, // +1 for CAS conflicts
1669             errorMetaPair.second.seqno + 1, // +1 for seqno conflicts
1670             errorMetaPair.second.flags,
1671             errorMetaPair.second.exptime);
1672 
1673     int force = 0;
1674     if (strstr(testHarness->get_current_testcase()->cfg,
1675                "conflict_resolution_type=lww") != nullptr) {
1676         force = FORCE_ACCEPT_WITH_META_OPS;
1677     }
1678 
1679     // Now enable XATTR
1680     testHarness->set_datatype_support(cookie, PROTOCOL_BINARY_DATATYPE_XATTR);
1681 
1682     // Now delete with a value (marked with XATTR)
1683     checkeq(ENGINE_SUCCESS,
1684             del_with_meta(h,
1685                           key1,
1686                           strlen(key1),
1687                           Vbid(0),
1688                           &itm_meta,
1689                           0, // cas
1690                           force,
1691                           cookie,
1692                           {}, // nmeta
1693                           PROTOCOL_BINARY_DATATYPE_XATTR,
1694                           data),
1695             "Expected delete OK");
1696 
1697     /* @todo this should fail, but doesn't. We've just deleted the item, but
1698     it remains in the hash-table (marked deleted), the subsequent set finds the
1699     item and is happy to process this SET_CAS operation (returns success).
1700     A delete with no body would of dropped it from the hash-table and the
1701     SET_CAS would return enoent, delete_with_meta /should/ I think have the same
1702     effect.
1703     checkeq(ENGINE_KEY_ENOENT,
1704             store(h,
1705                   h1,
1706                   nullptr,
1707                   OPERATION_SET,
1708                   key1,
1709                   body.data(),
1710                   nullptr,
1711                   last_cas.load()),
1712             "Failed to store key1.");
1713     */
1714 
1715     checkeq(cb::mcbp::Status::Success,
1716             last_status.load(),
1717             "Expected delete_with_meta(key1) to succeed");
1718 
1719     // Verify the new value is as expected
1720     item_info info;
1721     auto ret = get(h, nullptr, key1, Vbid(0), DocStateFilter::AliveOrDeleted);
1722     checkeq(cb::engine_errc::success, ret.first, "Failed to get(key1)");
1723 
1724     check(h->get_item_info(ret.second.get(), &info),
1725           "Failed get_item_info of key1");
1726 
1727     checkeq(data.size(), info.value[0].iov_len, "Value length mismatch");
1728     checkeq(0, memcmp(info.value[0].iov_base, data.data(), data.size()),
1729            "New body mismatch");
1730     checkeq(int(DocumentState::Deleted), int(info.document_state),
1731           "document_state is not DocumentState::Deleted");
1732 
1733     // The new value is XATTR
1734     checkeq(int(PROTOCOL_BINARY_DATATYPE_XATTR),
1735             int(info.datatype),
1736             "datatype isn't XATTR");
1737 
1738     // @todo implement test for the deletion of a value that has xattr using
1739     // a delete that has none (i.e. non-xattr/!spock client)
1740 
1741     testHarness->destroy_cookie(cookie);
1742 
1743     return SUCCESS;
1744 }
1745 
test_exp_persisted_set_del(EngineIface* h)1746 static enum test_result test_exp_persisted_set_del(EngineIface* h) {
1747     cb::EngineErrorMetadataPair errorMetaPair;
1748 
1749     check(!get_meta(h, "key3", errorMetaPair), "Expected get_meta() to fail");
1750 
1751     ItemMetaData itm_meta(1, 1, 0, 0);
1752     checkeq(ENGINE_SUCCESS,
1753             set_with_meta(h,
1754                           "key3",
1755                           4,
1756                           "val0",
1757                           4,
1758                           Vbid(0),
1759                           &itm_meta,
1760                           errorMetaPair.second.cas),
1761             "Expected item to be stored");
1762 
1763     itm_meta.revSeqno = 2;
1764     itm_meta.cas = 2;
1765     checkeq(ENGINE_SUCCESS,
1766             set_with_meta(
1767                     h, "key3", 4, "val1", 4, Vbid(0), &itm_meta, last_meta.cas),
1768             "Expected item to be stored");
1769 
1770     // MB-21725 Depending on how fast the flusher is, we may see 1 or 2.
1771     wait_for_stat_to_be_gte(h, "ep_total_persisted", 1);
1772 
1773     itm_meta.revSeqno = 3;
1774     itm_meta.cas = 3;
1775     itm_meta.exptime = 1735689600; // expires in 2025
1776     checkeq(ENGINE_SUCCESS,
1777             set_with_meta(
1778                     h, "key3", 4, "val1", 4, Vbid(0), &itm_meta, last_meta.cas),
1779             "Expected item to be stored");
1780 
1781     testHarness->time_travel(500000000);
1782     // Wait for the item to be expired, either by the pager,
1783     // or by access (as part of persistence callback from a
1784     // previous set - slow disk), or the compactor (unlikely).
1785     wait_for_expired_items_to_be(h, 1);
1786 
1787     wait_for_flusher_to_settle(h);
1788     wait_for_stat_to_be(h, "curr_items", 0);
1789 
1790     check(get_meta(h, "key3", errorMetaPair), "Expected to get meta");
1791     checkeq(uint64_t{4}, errorMetaPair.second.seqno, "Expected seqno to match");
1792     checkne(uint64_t{3},
1793             errorMetaPair.second.cas,
1794             "Expected cas to be different");
1795     checkeq(uint32_t{0}, errorMetaPair.second.flags, "Expected flags to match");
1796 
1797     return SUCCESS;
1798 }
1799 
test_temp_item_deletion(EngineIface* h)1800 static enum test_result test_temp_item_deletion(EngineIface* h) {
1801     // Do get_meta for an existing key
1802     char const *k1 = "k1";
1803 
1804     checkeq(ENGINE_SUCCESS,
1805             store(h, NULL, OPERATION_SET, k1, "somevalue"),
1806             "Failed set.");
1807     wait_for_flusher_to_settle(h);
1808 
1809     checkeq(ENGINE_SUCCESS, del(h, k1, 0, Vbid(0)), "Delete failed");
1810     wait_for_flusher_to_settle(h);
1811     wait_for_stat_to_be(h, "curr_items", 0);
1812 
1813     // Issue a get_meta for a deleted key. This will need to bring in a temp
1814     // item into the hashtable as a placeholder for the (deleted) metadata
1815     // which needs to be loaded from disk via BG fetch
1816     // We need to temporarily disable the reader threads as to prevent the
1817     // BGfetch from immediately running and removing our temp_item before
1818     // we've had chance to validate its existence.
1819     set_param(h,
1820               cb::mcbp::request::SetParamPayload::Type::Flush,
1821               "num_reader_threads",
1822               "0");
1823 
1824     // Disable nonio so that we have better control of the expirypager
1825     set_param(h,
1826               cb::mcbp::request::SetParamPayload::Type::Flush,
1827               "num_nonio_threads",
1828               "0");
1829 
1830     // Tell the harness not to handle EWOULDBLOCK for us - we want it to
1831     // be outstanding while we check the below stats.
1832     const void* cookie = testHarness->create_cookie();
1833     testHarness->set_ewouldblock_handling(cookie, false);
1834 
1835     cb::EngineErrorMetadataPair errorMetaPair;
1836 
1837     check(!get_meta(h, k1, errorMetaPair, cookie),
1838           "Expected get_meta to fail (EWOULDBLOCK)");
1839     checkeq(cb::engine_errc::would_block,
1840             errorMetaPair.first,
1841             "Expected EWOULDBLOCK");
1842 
1843     checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
1844     checkeq(1,
1845             get_int_stat(h, "curr_temp_items"),
1846             "Expected single temp_items");
1847 
1848     // Re-enable EWOULDBLOCK handling (and reader threads), and re-issue.
1849     testHarness->set_ewouldblock_handling(cookie, true);
1850     set_param(h,
1851               cb::mcbp::request::SetParamPayload::Type::Flush,
1852               "num_reader_threads",
1853               "1");
1854 
1855     check(get_meta(h, k1, errorMetaPair, cookie),
1856           "Expected get_meta to succeed");
1857     checkeq(DocumentState::Deleted,
1858             errorMetaPair.second.document_state,
1859             "Expected deleted flag to be set");
1860 
1861     // Even though 2 get_meta calls are made, we may have one or two bg fetches
1862     // done. That is if first bgfetch restores in HT, the deleted item from
1863     // disk, before the second get_meta call tries to find that item in HT,
1864     // we will have only 1 bgfetch
1865     wait_for_stat_to_be_gte(h, "ep_bg_meta_fetched", 1);
1866     int exp_get_meta_ops = get_int_stat(h, "ep_num_ops_get_meta");
1867 
1868     // Do get_meta for a non-existing key.
1869     char const *k2 = "k2";
1870     check(!get_meta(h, k2, errorMetaPair), "Expected get meta to return false");
1871     checkeq(cb::engine_errc::no_such_key,
1872             errorMetaPair.first,
1873             "Expected no_such_key");
1874 
1875     // This call for get_meta may or may not result in bg fetch because
1876     // bloomfilter may predict that key does not exist.
1877     // However we still must increment the ep_num_ops_get_meta count
1878     checkeq(exp_get_meta_ops + 1,
1879             get_int_stat(h, "ep_num_ops_get_meta"),
1880             "Num get meta ops not as expected");
1881 
1882     // Trigger the expiry pager and verify that two temp items are deleted
1883     set_param(h,
1884               cb::mcbp::request::SetParamPayload::Type::Flush,
1885               "num_nonio_threads",
1886               "1");
1887 
1888     // When bloom filters are on, it skips 1 of the expired items.
1889     int ep_expired_pager = get_bool_stat(h, "ep_bfilter_enabled") ? 1 : 2;
1890     wait_for_stat_to_be(h, "ep_expired_pager", ep_expired_pager);
1891 
1892     checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
1893     checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
1894 
1895     testHarness->destroy_cookie(cookie);
1896 
1897     return SUCCESS;
1898 }
1899 
test_add_meta_conflict_resolution(EngineIface* h)1900 static enum test_result test_add_meta_conflict_resolution(EngineIface* h) {
1901     // put some random metadata
1902     ItemMetaData itemMeta;
1903     itemMeta.revSeqno = 10;
1904     itemMeta.cas = 0xdeadbeef;
1905     itemMeta.exptime = 0;
1906     itemMeta.flags = 0xdeadbeef;
1907 
1908     checkeq(ENGINE_SUCCESS,
1909             add_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta),
1910             "Expected to add item");
1911     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
1912 
1913     int expected_bg_meta_fetched =
1914             get_bool_stat(h, "ep_bfilter_enabled") ? 0 : 1;
1915     checkeq(expected_bg_meta_fetched,
1916             get_int_stat(h, "ep_bg_meta_fetched"),
1917             "ep_bg_meta_fetched");
1918 
1919     checkeq(ENGINE_SUCCESS, del(h, "key", 0, Vbid(0)), "Delete failed");
1920     wait_for_flusher_to_settle(h);
1921     wait_for_stat_to_be(h, "curr_items", 0);
1922 
1923     // Check all meta data is the same
1924     itemMeta.revSeqno++;
1925     itemMeta.cas++;
1926     checkeq(ENGINE_KEY_EEXISTS,
1927             add_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta),
1928             "Expected exists");
1929     expected_bg_meta_fetched = 0;
1930     if (isPersistentBucket(h)) {
1931         expected_bg_meta_fetched =
1932                 get_bool_stat(h, "ep_bfilter_enabled") ? 1 : 2;
1933     }
1934     checkeq(expected_bg_meta_fetched,
1935             get_int_stat(h, "ep_bg_meta_fetched"),
1936             "ep_bg_meta_fetched");
1937 
1938     checkeq(1,
1939             get_int_stat(h, "ep_num_ops_set_meta_res_fail"),
1940             "Expected set meta conflict resolution failure");
1941 
1942     // Check has older flags fails
1943     itemMeta.flags = 0xdeadbeee;
1944     checkeq(ENGINE_KEY_EEXISTS,
1945             add_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta),
1946             "Expected exists");
1947     checkeq(2,
1948             get_int_stat(h, "ep_num_ops_set_meta_res_fail"),
1949             "Expected set meta conflict resolution failure");
1950 
1951     // Check testing with old seqno
1952     itemMeta.revSeqno--;
1953     checkeq(ENGINE_KEY_EEXISTS,
1954             add_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta),
1955             "Expected exists");
1956     checkeq(3,
1957             get_int_stat(h, "ep_num_ops_set_meta_res_fail"),
1958             "Expected set meta conflict resolution failure");
1959 
1960     itemMeta.revSeqno += 10;
1961     checkeq(ENGINE_SUCCESS,
1962             add_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta),
1963             "Expected to add item");
1964     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
1965     checkeq(3,
1966             get_int_stat(h, "ep_num_ops_set_meta_res_fail"),
1967             "Expected set meta conflict resolution failure");
1968 
1969     return SUCCESS;
1970 }
1971 
test_set_meta_conflict_resolution(EngineIface* h)1972 static enum test_result test_set_meta_conflict_resolution(EngineIface* h) {
1973     // put some random metadata
1974     ItemMetaData itemMeta;
1975     itemMeta.revSeqno = 10;
1976     itemMeta.cas = 0xdeadbeef;
1977     itemMeta.exptime = 0;
1978     itemMeta.flags = 0xdeadbeef;
1979 
1980     checkeq(0,
1981             get_int_stat(h, "ep_num_ops_set_meta"),
1982             "Expect zero setMeta ops");
1983 
1984     checkeq(ENGINE_SUCCESS,
1985             set_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta, 0),
1986             "Expected item to be stored");
1987     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
1988 
1989     int expected_bg_meta_fetched = 0;
1990     if (isPersistentBucket(h)) {
1991         expected_bg_meta_fetched =
1992                 get_bool_stat(h, "ep_bfilter_enabled") ? 0 : 1;
1993     }
1994     checkeq(expected_bg_meta_fetched,
1995             get_int_stat(h, "ep_bg_meta_fetched"),
1996             "ep_bg_meta_fetched");
1997 
1998     // Check all meta data is the same
1999     checkeq(ENGINE_KEY_EEXISTS,
2000             set_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta, 0),
2001             "Expected exists");
2002     checkeq(1,
2003             get_int_stat(h, "ep_num_ops_set_meta_res_fail"),
2004             "Expected set meta conflict resolution failure");
2005 
2006     // Check has older flags fails
2007     itemMeta.flags = 0xdeadbeee;
2008     checkeq(ENGINE_KEY_EEXISTS,
2009             set_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta, 0),
2010             "Expected exists");
2011     checkeq(2,
2012             get_int_stat(h, "ep_num_ops_set_meta_res_fail"),
2013             "Expected set meta conflict resolution failure");
2014 
2015     // Check has newer flags passes
2016     itemMeta.flags = 0xdeadbeff;
2017     checkeq(ENGINE_SUCCESS,
2018             set_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta, 0),
2019             "Expected item to be stored");
2020     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
2021 
2022     // Check that newer exptime wins
2023     itemMeta.exptime = time(NULL) + 10;
2024     checkeq(ENGINE_SUCCESS,
2025             set_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta, 0),
2026             "Expected item to be stored");
2027     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
2028 
2029     // Check that smaller exptime loses
2030     itemMeta.exptime = 0;
2031     checkeq(ENGINE_KEY_EEXISTS,
2032             set_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta, 0),
2033             "Expected exists");
2034     checkeq(3,
2035             get_int_stat(h, "ep_num_ops_set_meta_res_fail"),
2036             "Expected set meta conflict resolution failure");
2037 
2038     // Check testing with old seqno
2039     itemMeta.revSeqno--;
2040     checkeq(ENGINE_KEY_EEXISTS,
2041             set_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta, 0),
2042             "Expected exists");
2043     checkeq(4,
2044             get_int_stat(h, "ep_num_ops_set_meta_res_fail"),
2045             "Expected set meta conflict resolution failure");
2046 
2047     itemMeta.revSeqno += 10;
2048     checkeq(ENGINE_SUCCESS,
2049             set_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta, 0),
2050             "Expected item to be stored");
2051     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
2052     checkeq(4,
2053             get_int_stat(h, "ep_num_ops_set_meta_res_fail"),
2054             "Expected set meta conflict resolution failure");
2055 
2056     expected_bg_meta_fetched =
2057             (!isPersistentBucket(h) || get_bool_stat(h, "ep_bfilter_enabled"))
2058                     ? 0
2059                     : 1;
2060     checkeq(expected_bg_meta_fetched,
2061             get_int_stat(h, "ep_bg_meta_fetched"),
2062             "ep_bg_meta_fetched");
2063 
2064     return SUCCESS;
2065 }
2066 
test_set_meta_lww_conflict_resolution(EngineIface* h)2067 static enum test_result test_set_meta_lww_conflict_resolution(EngineIface* h) {
2068     // put some random metadata
2069     ItemMetaData itemMeta;
2070     itemMeta.revSeqno = 10;
2071     itemMeta.cas = 0xdeadbeef;
2072     itemMeta.exptime = 0;
2073     itemMeta.flags = 0xdeadbeef;
2074 
2075     checkeq(0,
2076             get_int_stat(h, "ep_num_ops_set_meta"),
2077             "Expect zero setMeta ops");
2078 
2079     checkeq(ENGINE_SUCCESS,
2080             set_with_meta(h,
2081                           "key",
2082                           3,
2083                           NULL,
2084                           0,
2085                           Vbid(0),
2086                           &itemMeta,
2087                           0,
2088                           FORCE_ACCEPT_WITH_META_OPS),
2089             "Expected item to be stored");
2090     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
2091 
2092     int expected_bg_meta_fetched =
2093             (!isPersistentBucket(h) || get_bool_stat(h, "ep_bfilter_enabled"))
2094                     ? 0
2095                     : 1;
2096     checkeq(expected_bg_meta_fetched,
2097             get_int_stat(h, "ep_bg_meta_fetched"),
2098             "ep_bg_meta_fetched");
2099 
2100     // Check all meta data is the same
2101     checkeq(ENGINE_KEY_EEXISTS,
2102             set_with_meta(h,
2103                           "key",
2104                           3,
2105                           NULL,
2106                           0,
2107                           Vbid(0),
2108                           &itemMeta,
2109                           0,
2110                           FORCE_ACCEPT_WITH_META_OPS),
2111             "Expected exists");
2112     checkeq(1,
2113             get_int_stat(h, "ep_num_ops_set_meta_res_fail"),
2114             "Expected set meta conflict resolution failure");
2115 
2116     // Check that an older cas fails
2117     itemMeta.cas = 0xdeadbeee;
2118     checkeq(ENGINE_KEY_EEXISTS,
2119             set_with_meta(h,
2120                           "key",
2121                           3,
2122                           NULL,
2123                           0,
2124                           Vbid(0),
2125                           &itemMeta,
2126                           0,
2127                           FORCE_ACCEPT_WITH_META_OPS),
2128             "Expected exists");
2129     checkeq(2,
2130             get_int_stat(h, "ep_num_ops_set_meta_res_fail"),
2131             "Expected set meta conflict resolution failure");
2132 
2133     // Check that a higher cas passes
2134     itemMeta.cas = 0xdeadbeff;
2135     checkeq(ENGINE_SUCCESS,
2136             set_with_meta(h,
2137                           "key",
2138                           3,
2139                           NULL,
2140                           0,
2141                           Vbid(0),
2142                           &itemMeta,
2143                           0,
2144                           FORCE_ACCEPT_WITH_META_OPS),
2145             "Expected item to be stored");
2146     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
2147 
2148     // Check that we fail requests if the force flag is not set
2149     itemMeta.cas = 0xdeadbeff + 1;
2150     checkeq(ENGINE_EINVAL,
2151             set_with_meta(
2152                     h, "key", 3, NULL, 0, Vbid(0), &itemMeta, 0, 0 /*options*/),
2153             "Expected EINVAL");
2154 
2155     return SUCCESS;
2156 }
2157 
test_del_meta_conflict_resolution(EngineIface* h)2158 static enum test_result test_del_meta_conflict_resolution(EngineIface* h) {
2159     checkeq(ENGINE_SUCCESS,
2160             store(h, NULL, OPERATION_SET, "key", "somevalue"),
2161             "Failed set.");
2162     wait_for_flusher_to_settle(h);
2163 
2164     // put some random metadata
2165     ItemMetaData itemMeta;
2166     itemMeta.revSeqno = 10;
2167     itemMeta.cas = 0xdeadbeef;
2168     itemMeta.exptime = 0;
2169     itemMeta.flags = 0xdeadbeef;
2170 
2171     checkeq(ENGINE_SUCCESS,
2172             del_with_meta(h, "key", 3, Vbid(0), &itemMeta),
2173             "Expected delete ok");
2174     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
2175     wait_for_flusher_to_settle(h);
2176     wait_for_stat_to_be(h, "curr_items", 0);
2177 
2178     // Check all meta data is the same
2179     checkeq(ENGINE_KEY_EEXISTS,
2180             del_with_meta(h, "key", 3, Vbid(0), &itemMeta),
2181             "Expected exists");
2182     checkeq(1,
2183             get_int_stat(h, "ep_num_ops_del_meta_res_fail"),
2184             "Expected delete meta conflict resolution failure");
2185 
2186     // Check has older flags fails
2187     itemMeta.flags = 0xdeadbeee;
2188     checkeq(ENGINE_KEY_EEXISTS,
2189             del_with_meta(h, "key", 3, Vbid(0), &itemMeta),
2190             "Expected exists");
2191     checkeq(2,
2192             get_int_stat(h, "ep_num_ops_del_meta_res_fail"),
2193             "Expected delete meta conflict resolution failure");
2194 
2195     // Check that smaller exptime loses
2196     itemMeta.exptime = 0;
2197     checkeq(ENGINE_KEY_EEXISTS,
2198             del_with_meta(h, "key", 3, Vbid(0), &itemMeta),
2199             "Expected exists");
2200     checkeq(3,
2201             get_int_stat(h, "ep_num_ops_del_meta_res_fail"),
2202             "Expected delete meta conflict resolution failure");
2203 
2204     // Check testing with old seqno
2205     itemMeta.revSeqno--;
2206     checkeq(ENGINE_KEY_EEXISTS,
2207             del_with_meta(h, "key", 3, Vbid(0), &itemMeta),
2208             "Expected exists");
2209     checkeq(4, get_int_stat(h, "ep_num_ops_del_meta_res_fail"),
2210           "Expected delete meta conflict resolution failure");
2211 
2212     itemMeta.revSeqno += 10;
2213     checkeq(ENGINE_SUCCESS,
2214             del_with_meta(h, "key", 3, Vbid(0), &itemMeta),
2215             "Expected delete OK");
2216     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
2217     checkeq(4, get_int_stat(h, "ep_num_ops_del_meta_res_fail"),
2218           "Expected delete meta conflict resolution failure");
2219 
2220     return SUCCESS;
2221 }
2222 
test_del_meta_lww_conflict_resolution(EngineIface* h)2223 static enum test_result test_del_meta_lww_conflict_resolution(EngineIface* h) {
2224     item *i = NULL;
2225     item_info info;
2226 
2227     checkeq(ENGINE_SUCCESS,
2228             store(h, NULL, OPERATION_SET, "key", "somevalue", &i),
2229             "Failed set.");
2230 
2231     h->get_item_info(i, &info);
2232     wait_for_flusher_to_settle(h);
2233     h->release(i);
2234 
2235     // put some random metadata
2236     ItemMetaData itemMeta;
2237     itemMeta.revSeqno = 10;
2238     itemMeta.cas = info.cas + 1;
2239     itemMeta.exptime = 0;
2240     itemMeta.flags = 0xdeadbeef;
2241 
2242     // first check the command fails if no force is set
2243     checkeq(ENGINE_EINVAL,
2244             del_with_meta(h, "key", 3, Vbid(0), &itemMeta, 0, 0 /*options*/),
2245             "Expected EINVAL");
2246 
2247     checkeq(ENGINE_SUCCESS,
2248             del_with_meta(h,
2249                           "key",
2250                           3,
2251                           Vbid(0),
2252                           &itemMeta,
2253                           0,
2254                           FORCE_ACCEPT_WITH_META_OPS),
2255             "Expected delete OK");
2256     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
2257     wait_for_flusher_to_settle(h);
2258     wait_for_stat_to_be(h, "curr_items", 0);
2259 
2260     // Check all meta data is the same
2261     checkeq(ENGINE_KEY_EEXISTS,
2262             del_with_meta(h,
2263                           "key",
2264                           3,
2265                           Vbid(0),
2266                           &itemMeta,
2267                           0,
2268                           FORCE_ACCEPT_WITH_META_OPS),
2269             "Expected exists");
2270     checkeq(1,
2271             get_int_stat(h, "ep_num_ops_del_meta_res_fail"),
2272             "Expected delete meta conflict resolution failure");
2273 
2274     // Check that higher rev seqno but lower cas fails
2275     itemMeta.cas = info.cas;
2276     itemMeta.revSeqno = 11;
2277     checkeq(ENGINE_KEY_EEXISTS,
2278             del_with_meta(h,
2279                           "key",
2280                           3,
2281                           Vbid(0),
2282                           &itemMeta,
2283                           0,
2284                           FORCE_ACCEPT_WITH_META_OPS),
2285             "Expected exists");
2286     checkeq(2,
2287             get_int_stat(h, "ep_num_ops_del_meta_res_fail"),
2288             "Expected delete meta conflict resolution failure");
2289 
2290     // Check that a higher cas and lower rev seqno passes
2291     itemMeta.cas = info.cas + 2;
2292     itemMeta.revSeqno = 9;
2293     checkeq(ENGINE_SUCCESS,
2294             del_with_meta(h,
2295                           "key",
2296                           3,
2297                           Vbid(0),
2298                           &itemMeta,
2299                           0,
2300                           FORCE_ACCEPT_WITH_META_OPS),
2301             "Expected delete OK");
2302     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected sucess");
2303 
2304     return SUCCESS;
2305 }
2306 
test_getMeta_with_item_eviction(EngineIface* h)2307 static enum test_result test_getMeta_with_item_eviction(EngineIface* h) {
2308     char const *key = "test_get_meta";
2309     item *i = NULL;
2310     checkeq(ENGINE_SUCCESS,
2311             store(h, NULL, OPERATION_SET, key, "somevalue", &i),
2312             "Failed set.");
2313     wait_for_flusher_to_settle(h);
2314     evict_key(h, key, Vbid(0), "Ejected.");
2315 
2316     Item *it = reinterpret_cast<Item*>(i);
2317 
2318     cb::EngineErrorMetadataPair errorMetaPair;
2319     check(get_meta(h, key, errorMetaPair), "Expected to get meta");
2320     ItemMetaData metadata(it->getCas(), it->getRevSeqno(),
2321                           it->getFlags(), it->getExptime());
2322     verifyMetaData(metadata, errorMetaPair.second);
2323 
2324     h->release(i);
2325     return SUCCESS;
2326 }
2327 
test_set_with_meta_and_check_drift_stats( EngineIface* h)2328 static enum test_result test_set_with_meta_and_check_drift_stats(
2329         EngineIface* h) {
2330     // Activate n vbuckets (vb 0 is already)
2331     const int n_vbuckets = 10;
2332     for (int ii = 1; ii < n_vbuckets; ii++) {
2333         check(set_vbucket_state(h, Vbid(ii), vbucket_state_active),
2334               "Failed to set vbucket state.");
2335     }
2336 
2337     // Let's make vbucket n/2 be the one who is ahead, n/3 is behind
2338     const int aheadVb = n_vbuckets/2;
2339     const int behindVb = n_vbuckets/3;
2340     checkne(aheadVb, behindVb, "Cannot have the same VB as ahead/behind");
2341 
2342     HLC hlc(0 /*init HLC*/,
2343             HlcCasSeqnoUninitialised,
2344             std::chrono::microseconds(0) /*ahead threshold*/,
2345             std::chrono::microseconds(0) /*behind threshold*/);
2346 
2347     // grab the drift behind threshold
2348     uint64_t driftBehindThreshold =
2349             get_ull_stat(h, "ep_hlc_drift_ahead_threshold_us", nullptr);
2350     // Create n keys
2351     const int n_keys = 5;
2352     for (int ii = 0 ; ii < n_vbuckets; ii++) {
2353         for (int k = 0; k < n_keys; k++) {
2354             std::string key = "key_" + std::to_string(k);
2355             ItemMetaData itm_meta;
2356             itm_meta.cas = hlc.nextHLC();
2357             if (ii == aheadVb) {
2358                 // Push this guy *far* ahead (1 year)
2359                 itm_meta.cas += 3154E10;
2360             } else if(ii == behindVb) {
2361                 // just be sure it was already greater then 1 + driftthreshold
2362                 checkge(itm_meta.cas, uint64_t(1) + driftBehindThreshold,
2363                         "HLC was already zero");
2364                 // set to be way way behind...
2365                 itm_meta.cas = 1;
2366             }
2367             checkeq(ENGINE_SUCCESS,
2368                     set_with_meta(h,
2369                                   key.data(),
2370                                   key.size(),
2371                                   NULL,
2372                                   0,
2373                                   Vbid(ii),
2374                                   &itm_meta,
2375                                   0,
2376                                   FORCE_ACCEPT_WITH_META_OPS),
2377                     "Expected item to be stored");
2378             checkeq(cb::mcbp::Status::Success, last_status.load(),
2379                     "Expected success");
2380         }
2381     }
2382 
2383     // Bucket stats should report drift
2384     checkge(get_ull_stat(h, "ep_active_hlc_drift"),
2385             uint64_t(0),
2386             "Expected drift above zero");
2387     checkeq(uint64_t(n_keys * n_vbuckets),
2388             get_ull_stat(h, "ep_active_hlc_drift_count"),
2389             "Expected ahead counter to match mutations");
2390 
2391     // Victim VBs should have exceptions
2392     {
2393         std::string vbAheadName = "vb_" + std::to_string(aheadVb);
2394         std::string ahead_threshold_exceeded = vbAheadName + ":drift_ahead_threshold_exceeded";
2395         std::string behind_threshold_exceeded = vbAheadName + ":drift_behind_threshold_exceeded";
2396         std::string total_abs_drift = vbAheadName + ":total_abs_drift";
2397         std::string details = "vbucket-details " + std::to_string(aheadVb);
2398         checkeq(uint64_t(n_keys),
2399                 get_ull_stat(
2400                         h, ahead_threshold_exceeded.data(), details.data()),
2401                 "Expected ahead threshold to match mutations");
2402         checkeq(uint64_t(0),
2403                 get_ull_stat(
2404                         h, behind_threshold_exceeded.data(), details.data()),
2405                 "Expected no behind exceptions");
2406         checkge(get_ull_stat(h, total_abs_drift.data(), details.data()),
2407                 uint64_t(0),
2408                 "Expected some drift");
2409     }
2410 
2411     {
2412         std::string vbBehindName = "vb_" + std::to_string(behindVb);
2413         std::string ahead_threshold_exceeded = vbBehindName + ":drift_ahead_threshold_exceeded";
2414         std::string behind_threshold_exceeded = vbBehindName + ":drift_behind_threshold_exceeded";
2415         std::string total_abs_drift = vbBehindName + ":total_abs_drift";
2416         std::string details = "vbucket-details " + std::to_string(behindVb);
2417         checkeq(uint64_t(n_keys),
2418                 get_ull_stat(
2419                         h, behind_threshold_exceeded.data(), details.data()),
2420                 "Expected behind threshold to match mutations");
2421         checkeq(uint64_t(0),
2422                 get_ull_stat(
2423                         h, ahead_threshold_exceeded.data(), details.data()),
2424                 "Expected no ahead exceptions");
2425         checkge(get_ull_stat(h, total_abs_drift.data(), details.data()),
2426                 uint64_t(0),
2427                 "Expected some drift");
2428     }
2429 
2430 
2431     return SUCCESS;
2432 }
2433 
test_del_with_meta_and_check_drift_stats( EngineIface* h)2434 static enum test_result test_del_with_meta_and_check_drift_stats(
2435         EngineIface* h) {
2436     // Activate n vbuckets (vb 0 is already)
2437     const int n_vbuckets = 10;
2438     for (int ii = 1; ii < n_vbuckets; ii++) {
2439         check(set_vbucket_state(h, Vbid(ii), vbucket_state_active),
2440               "Failed to set vbucket state.");
2441     }
2442 
2443     // Let's make vbucket n/2 be the one who is ahead, n/3 is behind
2444     const int aheadVb = n_vbuckets/2;
2445     const int behindVb = n_vbuckets/3;
2446     checkne(aheadVb, behindVb, "Cannot have the same VB as ahead/behind");
2447 
2448     HLC hlc(0 /*init HLC*/,
2449             HlcCasSeqnoUninitialised,
2450             std::chrono::microseconds(0) /*ahead threshold*/,
2451             std::chrono::microseconds(0) /*behind threshold*/);
2452 
2453     // grab the drift behind threshold
2454     uint64_t driftBehindThreshold =
2455             get_ull_stat(h, "ep_hlc_drift_ahead_threshold_us", nullptr);
2456     // Create n keys * n_vbuckets
2457     const int n_keys = 5;
2458     for (int ii = 0 ; ii < n_vbuckets; ii++) {
2459         for (int k = 0; k < n_keys; k++) {
2460             std::string key = "key_" + std::to_string(k);
2461 
2462             // In the del_with_meta test we want to pretend a del_wm came from
2463             // the past, so we want to ensure a delete doesn't get rejected
2464             // by LWW conflict resolution, thus write all documents that are
2465             // going to be deleted with set_with_meta, and write them way in the past.
2466             // This will trigger threshold and increment drift stats... so we
2467             // account for these later
2468             ItemMetaData itm_meta;
2469             itm_meta.cas = 1; // set to 1
2470             checkeq(ENGINE_SUCCESS,
2471                     set_with_meta(h,
2472                                   key.data(),
2473                                   key.size(),
2474                                   NULL,
2475                                   0,
2476                                   Vbid(ii),
2477                                   &itm_meta,
2478                                   0,
2479                                   FORCE_ACCEPT_WITH_META_OPS),
2480                     "Expected item to be stored");
2481             checkeq(cb::mcbp::Status::Success, last_status.load(),
2482                     "Expected success");
2483         }
2484     }
2485 
2486     checkeq(uint64_t(0),
2487             get_ull_stat(h, "ep_active_ahead_exceptions"),
2488             "Expected ahead counter to match mutations");
2489     checkeq(uint64_t(n_keys * n_vbuckets),
2490             get_ull_stat(h, "ep_active_behind_exceptions"),
2491             "Expected behind counter to match mutations");
2492 
2493     // Del_with_meta n_keys to n_vbuckets
2494     for (int ii = 0 ; ii < n_vbuckets; ii++) {
2495         for (int k = 0; k < n_keys; k++) {
2496             std::string key = "key_" + std::to_string(k);
2497             ItemMetaData itm_meta;
2498             itm_meta.cas = hlc.nextHLC();
2499             if (ii == aheadVb) {
2500                 // Push this guy *far* ahead (1 year)
2501                 itm_meta.cas += 3154E10;
2502             } else if(ii == behindVb) {
2503                 // just be sure it was already greater than 1 + driftthreshold
2504                 checkge(itm_meta.cas, uint64_t(1) + driftBehindThreshold,
2505                         "HLC was already zero");
2506                 // set to be way way behind, but ahead of the documents we have set
2507                 itm_meta.cas = 2;
2508             }
2509             checkeq(ENGINE_SUCCESS,
2510                     del_with_meta(h,
2511                                   key.data(),
2512                                   key.size(),
2513                                   Vbid(ii),
2514                                   &itm_meta,
2515                                   1,
2516                                   FORCE_ACCEPT_WITH_META_OPS),
2517                     "Expected delete OK");
2518             checkeq(cb::mcbp::Status::Success, last_status.load(),
2519                     "Expected success");
2520         }
2521     }
2522 
2523     // Bucket stats should report drift
2524     checkge(get_ull_stat(h, "ep_active_hlc_drift"),
2525             uint64_t(0),
2526             "Expected drift above zero");
2527     checkeq(2 * uint64_t(n_keys * n_vbuckets),
2528             get_ull_stat(h, "ep_active_hlc_drift_count"),
2529             "Expected ahead counter to match mutations");
2530 
2531     // and should report total exception of all VBs
2532     checkeq(uint64_t(n_keys),
2533             get_ull_stat(h, "ep_active_ahead_exceptions"),
2534             "Expected ahead counter to match mutations");
2535     checkeq(uint64_t(n_keys + (n_keys * n_vbuckets)),
2536             get_ull_stat(h, "ep_active_behind_exceptions"),
2537             "Expected behind counter to match mutations");
2538 
2539     // Victim VBs should have exceptions
2540     {
2541         std::string vbAheadName = "vb_" + std::to_string(aheadVb);
2542         std::string ahead_threshold_exceeded = vbAheadName + ":drift_ahead_threshold_exceeded";
2543         std::string behind_threshold_exceeded = vbAheadName + ":drift_behind_threshold_exceeded";
2544         std::string total_abs_drift = vbAheadName + ":total_abs_drift";
2545         std::string details = "vbucket-details " + std::to_string(aheadVb);
2546 
2547         checkeq(uint64_t(n_keys),
2548                 get_ull_stat(
2549                         h, ahead_threshold_exceeded.data(), details.data()),
2550                 "Expected ahead threshold to match mutations");
2551         checkge(get_ull_stat(h, total_abs_drift.data(), details.data()),
2552                 uint64_t(0),
2553                 "Expected some drift");
2554     }
2555 
2556     {
2557         std::string vbBehindName = "vb_" + std::to_string(behindVb);
2558         std::string ahead_threshold_exceeded = vbBehindName + ":drift_ahead_threshold_exceeded";
2559         std::string behind_threshold_exceeded = vbBehindName + ":drift_behind_threshold_exceeded";
2560         std::string total_abs_drift = vbBehindName + ":total_abs_drift";
2561         std::string details = "vbucket-details " + std::to_string(behindVb);
2562 
2563         // *2 behind due to the initial set_with_meta
2564         checkeq(uint64_t(n_keys * 2),
2565                 get_ull_stat(
2566                         h, behind_threshold_exceeded.data(), details.data()),
2567                 "Expected behind threshold to match mutations");
2568         checkeq(uint64_t(0),
2569                 get_ull_stat(
2570                         h, ahead_threshold_exceeded.data(), details.data()),
2571                 "Expected no ahead exceptions");
2572         checkge(get_ull_stat(h, total_abs_drift.data(), details.data()),
2573                 uint64_t(0),
2574                 "Expected some drift");
2575     }
2576 
2577 
2578     return SUCCESS;
2579 }
2580 
test_setting_drift_threshold(EngineIface* h)2581 static enum test_result test_setting_drift_threshold(EngineIface* h) {
2582     std::vector<std::tuple<std::string, std::string, std::string> > configData =
2583         {std::make_tuple("ep_hlc_drift_ahead_threshold_us",
2584                          "hlc_drift_ahead_threshold_us",
2585                          "vb_0:drift_ahead_threshold"),
2586          std::make_tuple("ep_hlc_drift_behind_threshold_us",
2587                          "hlc_drift_behind_threshold_us",
2588                          "vb_0:drift_behind_threshold")};
2589 
2590     std::vector<std::pair<std::string, std::chrono::microseconds> > values = {
2591         {"0", std::chrono::microseconds(0)},
2592         {"1", std::chrono::microseconds(1)},
2593         {"-1", std::chrono::microseconds(-1)},
2594         {"-0", std::chrono::microseconds(0)},
2595         {"18446744073709551615",
2596          std::chrono::microseconds(18446744073709551615ull)}};
2597 
2598     for (auto data : values) {
2599         for (auto conf : configData) {
2600             check(set_param(h,
2601                             cb::mcbp::request::SetParamPayload::Type::Vbucket,
2602                             std::get<1>(conf).c_str(),
2603                             data.first.data()),
2604                   "Expected set_param success");
2605 
2606             checkeq(int64_t(data.second.count()),
2607                     int64_t(get_ull_stat(
2608                             h, std::get<0>(conf).c_str(), nullptr)),
2609                     "Expected the stat to change to the new value");
2610 
2611             // The VB stat values are in nanoseconds
2612             checkeq(int64_t(std::chrono::nanoseconds(data.second).count()),
2613                     int64_t(get_ull_stat(
2614                             h, std::get<2>(conf).c_str(), "vbucket-details 0")),
2615                     "Expected the VB stats to change to the new value");
2616         }
2617     }
2618     return SUCCESS;
2619 }
2620 
2621 /*
2622  * Perform set_with_meta and check CAS regeneration is ok.
2623  */
test_cas_regeneration(EngineIface* h)2624 static enum test_result test_cas_regeneration(EngineIface* h) {
2625     // First store a key from the past (small CAS).
2626     ItemMetaData itemMeta;
2627     itemMeta.revSeqno = 10;
2628     itemMeta.cas = 0x1;
2629     itemMeta.exptime = 0;
2630     itemMeta.flags = 0xdeadbeef;
2631     int force = 0;
2632 
2633     if (strstr(testHarness->get_current_testcase()->cfg,
2634                "conflict_resolution_type=lww") != nullptr) {
2635         force = FORCE_ACCEPT_WITH_META_OPS;
2636     }
2637 
2638     // Set the key with a low CAS value
2639     checkeq(ENGINE_SUCCESS,
2640             set_with_meta(
2641                     h, "key", 3, nullptr, 0, Vbid(0), &itemMeta, 0, force),
2642             "Expected item to be stored");
2643     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
2644 
2645     cb::EngineErrorMetadataPair errorMetaPair;
2646 
2647     check(get_meta(h, "key", errorMetaPair), "Failed to get_meta");
2648 
2649     // CAS must be what we set.
2650     checkeq(itemMeta.cas,
2651             errorMetaPair.second.cas,
2652             "CAS is not the value we stored");
2653 
2654     itemMeta.cas++;
2655 
2656     // Check that the code requires skip
2657     checkeq(ENGINE_EINVAL,
2658             set_with_meta(h,
2659                           "key",
2660                           3,
2661                           nullptr,
2662                           0,
2663                           Vbid(0),
2664                           &itemMeta,
2665                           0,
2666                           REGENERATE_CAS /*but no skip*/),
2667             "Expected EINVAL");
2668 
2669     checkeq(ENGINE_SUCCESS,
2670             set_with_meta(h,
2671                           "key",
2672                           3,
2673                           nullptr,
2674                           0,
2675                           Vbid(0),
2676                           &itemMeta,
2677                           0,
2678                           REGENERATE_CAS | SKIP_CONFLICT_RESOLUTION_FLAG),
2679             "Expected item to be stored");
2680 
2681     checkeq(cb::mcbp::Status::Success, last_status.load(),
2682             "Expected success");
2683 
2684     check(get_meta(h, "key", errorMetaPair), "Failed to get_meta");
2685 
2686     uint64_t cas = errorMetaPair.second.cas;
2687     // Check item has a new CAS
2688     checkne(itemMeta.cas, cas, "CAS was not regenerated");
2689 
2690     itemMeta.cas++;
2691     // All flags set should still regen the cas (lww and seqno)
2692     checkeq(ENGINE_SUCCESS,
2693             set_with_meta(
2694                     h,
2695                     "key",
2696                     3,
2697                     nullptr,
2698                     0,
2699                     Vbid(0),
2700                     &itemMeta,
2701                     0,
2702                     REGENERATE_CAS | SKIP_CONFLICT_RESOLUTION_FLAG | force),
2703             "Expected item to be stored");
2704 
2705     checkeq(cb::mcbp::Status::Success, last_status.load(),
2706             "Expected success");
2707 
2708     check(get_meta(h, "key", errorMetaPair), "Failed to get_meta");
2709     // Check item has a new CAS
2710     checkne(itemMeta.cas, errorMetaPair.second.cas, "CAS was not regenerated");
2711     checkne(cas, errorMetaPair.second.cas, "CAS was not regenerated");
2712     return SUCCESS;
2713 }
2714 
2715 /*
2716  * Perform del_with_meta and check CAS regeneration is ok.
2717  */
test_cas_regeneration_del_with_meta(EngineIface* h)2718 static enum test_result test_cas_regeneration_del_with_meta(EngineIface* h) {
2719     const std::string key("key");
2720     // First store a key from the past (small CAS).
2721     ItemMetaData itemMeta;
2722     itemMeta.revSeqno = 10;
2723     itemMeta.cas = 0x1;
2724     itemMeta.exptime = 0;
2725     itemMeta.flags = 0xdeadbeef;
2726     int force = 0;
2727 
2728     if (strstr(testHarness->get_current_testcase()->cfg,
2729                "conflict_resolution_type=lww") != nullptr) {
2730         force = FORCE_ACCEPT_WITH_META_OPS;
2731     }
2732 
2733     // Set the key with a low CAS value
2734     checkeq(ENGINE_SUCCESS,
2735             set_with_meta(h,
2736                           key.c_str(),
2737                           key.length(),
2738                           nullptr,
2739                           0,
2740                           Vbid(0),
2741                           &itemMeta,
2742                           0,
2743                           force),
2744             "Expected item to be stored");
2745     checkeq(cb::mcbp::Status::Success,
2746             last_status.load(),
2747             "Expected success");
2748 
2749     cb::EngineErrorMetadataPair errorMetaPair;
2750     check(get_meta(h, key.c_str(), errorMetaPair), "Failed to get_meta");
2751     // CAS must be what we set.
2752     checkeq(itemMeta.cas,
2753             errorMetaPair.second.cas,
2754             "CAS is not the value we stored");
2755 
2756     itemMeta.cas++;
2757 
2758     // Check that the code requires skip
2759     checkeq(ENGINE_EINVAL,
2760             del_with_meta(h,
2761                           key.c_str(),
2762                           key.length(),
2763                           Vbid(0),
2764                           &itemMeta,
2765                           0,
2766                           REGENERATE_CAS /*but no skip*/),
2767             "Expected EINVAL");
2768 
2769     checkeq(ENGINE_SUCCESS,
2770             del_with_meta(h,
2771                           key.c_str(),
2772                           key.length(),
2773                           Vbid(0),
2774                           &itemMeta,
2775                           0,
2776                           REGENERATE_CAS | SKIP_CONFLICT_RESOLUTION_FLAG),
2777             "Expected delete OK");
2778     checkeq(cb::mcbp::Status::Success,
2779             last_status.load(),
2780             "Expected success");
2781 
2782     check(get_meta(h, key.c_str(), errorMetaPair), "Failed to get_meta");
2783     uint64_t cas = errorMetaPair.second.cas;
2784     // Check item has a new CAS
2785     checkne(itemMeta.cas, cas, "CAS was not regenerated");
2786 
2787     itemMeta.cas++;
2788     // All flags set should still regen the cas (lww and seqno)
2789     checkeq(ENGINE_SUCCESS,
2790             del_with_meta(
2791                     h,
2792                     key.c_str(),
2793                     key.length(),
2794                     Vbid(0),
2795                     &itemMeta,
2796                     0,
2797                     REGENERATE_CAS | SKIP_CONFLICT_RESOLUTION_FLAG | force),
2798             "Expected delete OK");
2799     checkeq(cb::mcbp::Status::Success,
2800             last_status.load(),
2801             "Expected success");
2802 
2803     check(get_meta(h, key.c_str(), errorMetaPair), "Failed to get_meta");
2804     // Check item has a new CAS
2805     checkne(itemMeta.cas, errorMetaPair.second.cas, "CAS was not regenerated");
2806     checkne(cas, errorMetaPair.second.cas, "CAS was not regenerated");
2807 
2808     return SUCCESS;
2809 }
2810 
2811 /*
2812  * Test that we can send options and nmeta
2813  * The nmeta is just going to be ignored though, but should not fail
2814  */
test_cas_options_and_nmeta(EngineIface* h)2815 static enum test_result test_cas_options_and_nmeta(EngineIface* h) {
2816     ItemMetaData itemMeta;
2817     itemMeta.revSeqno = 10;
2818     itemMeta.cas = 0x1;
2819     itemMeta.exptime = 0;
2820     itemMeta.flags = 0xdeadbeef;
2821 
2822     // Watson (4.6) accepts valid encodings, but ignores them
2823     std::vector<char> junkMeta = {-2,-1,2,3};
2824 
2825     int force = 0;
2826 
2827     if (strstr(testHarness->get_current_testcase()->cfg,
2828                "conflict_resolution_type=lww") != nullptr) {
2829         force = FORCE_ACCEPT_WITH_META_OPS;
2830     }
2831 
2832     // Set the key and junk nmeta
2833     checkeq(ENGINE_EINVAL,
2834             set_with_meta(h,
2835                           "key",
2836                           3,
2837                           NULL,
2838                           0,
2839                           Vbid(0),
2840                           &itemMeta,
2841                           0,
2842                           force,
2843                           PROTOCOL_BINARY_RAW_BYTES,
2844                           nullptr,
2845                           junkMeta),
2846             "Expected EINVAL");
2847 
2848     // Set the key and junk nmeta that's quite large
2849     junkMeta.resize(std::numeric_limits<uint16_t>::max());
2850     checkeq(ENGINE_EINVAL,
2851             set_with_meta(h,
2852                           "key",
2853                           3,
2854                           NULL,
2855                           0,
2856                           Vbid(0),
2857                           &itemMeta,
2858                           0,
2859                           force,
2860                           PROTOCOL_BINARY_RAW_BYTES,
2861                           nullptr,
2862                           junkMeta),
2863             "Expected EINVAL");
2864 
2865     // Test that valid meta can be sent. It should be ignored and success
2866     // returned
2867     // Encodings which should not fail, see ext_meta_parser.cc
2868 #pragma pack(1)
2869     struct adjusted_time_metadata {
2870         uint8_t type;
2871         uint16_t length;
2872         int64_t value;
2873     };
2874     struct conf_res_metadata {
2875         uint8_t type;
2876         uint16_t length;
2877         uint8_t value;
2878     };
2879     struct with_cas_metadata1 {
2880         uint8_t version;
2881         adjusted_time_metadata adjusted_time;
2882     };
2883     struct with_cas_metadata2 {
2884         uint8_t version;
2885         conf_res_metadata conf_res;
2886     };
2887     struct with_cas_metadata3 {
2888         uint8_t version;
2889         conf_res_metadata conf_res;
2890         adjusted_time_metadata adjusted_time;
2891     };
2892     struct with_cas_metadata4 {
2893         uint8_t version;
2894         adjusted_time_metadata adjusted_time;
2895         conf_res_metadata conf_res;
2896     };
2897 #pragma pack()
2898 
2899     {
2900         with_cas_metadata1 validMetaData = {META_EXT_VERSION_ONE,
2901                                             {CMD_META_ADJUSTED_TIME,
2902                                              htons(sizeof(int64_t)), -1}};
2903         std::vector<char> validMetaVector(reinterpret_cast<char*>(&validMetaData),
2904                                           reinterpret_cast<char*>(&validMetaData) +
2905                                           sizeof(validMetaData));
2906 
2907         // Set the key with a low CAS value and real nmeta
2908         checkeq(ENGINE_SUCCESS,
2909                 set_with_meta(h,
2910                               "key1",
2911                               4,
2912                               nullptr,
2913                               0,
2914                               Vbid(0),
2915                               &itemMeta,
2916                               0,
2917                               force,
2918                               PROTOCOL_BINARY_RAW_BYTES,
2919                               nullptr,
2920                               validMetaVector),
2921                 "Expected item to be stored");
2922         checkeq(cb::mcbp::Status::Success, last_status.load(),
2923                 "Expected success");
2924 
2925         itemMeta.cas++;
2926         checkeq(ENGINE_SUCCESS,
2927                 del_with_meta(h,
2928                               "key1",
2929                               4,
2930                               Vbid(0),
2931                               &itemMeta,
2932                               0,
2933                               force,
2934                               nullptr,
2935                               validMetaVector),
2936                 "Expected delete success");
2937         checkeq(cb::mcbp::Status::Success, last_status.load(),
2938                 "Expected success");
2939     }
2940 
2941     {
2942         with_cas_metadata2 validMetaData = {META_EXT_VERSION_ONE,
2943                                             {CMD_META_CONFLICT_RES_MODE,
2944                                              htons(sizeof(uint8_t)), 0xff}};
2945         std::vector<char> validMetaVector(reinterpret_cast<char*>(&validMetaData),
2946                                           reinterpret_cast<char*>(&validMetaData) +
2947                                           sizeof(validMetaData));
2948 
2949         // Set the key with a low CAS value and real nmeta
2950         checkeq(ENGINE_SUCCESS,
2951                 set_with_meta(h,
2952                               "key2",
2953                               4,
2954                               nullptr,
2955                               0,
2956                               Vbid(0),
2957                               &itemMeta,
2958                               0,
2959                               force,
2960                               PROTOCOL_BINARY_RAW_BYTES,
2961                               nullptr,
2962                               validMetaVector),
2963                 "Expected item to be stored");
2964         checkeq(cb::mcbp::Status::Success, last_status.load(),
2965                 "Expected success");
2966 
2967         itemMeta.cas++;
2968         checkeq(ENGINE_SUCCESS,
2969                 del_with_meta(h,
2970                               "key2",
2971                               4,
2972                               Vbid(0),
2973                               &itemMeta,
2974                               0,
2975                               force,
2976                               nullptr,
2977                               validMetaVector),
2978                 "Expected delete success");
2979         checkeq(cb::mcbp::Status::Success, last_status.load(),
2980                 "Expected success");
2981     }
2982 
2983     {
2984         with_cas_metadata3 validMetaData = {META_EXT_VERSION_ONE,
2985                                             {CMD_META_CONFLICT_RES_MODE,
2986                                              htons(sizeof(uint8_t)), 0xff},
2987                                             {CMD_META_ADJUSTED_TIME,
2988                                              htons(sizeof(int64_t)), -1}};
2989         std::vector<char> validMetaVector(reinterpret_cast<char*>(&validMetaData),
2990                                           reinterpret_cast<char*>(&validMetaData) +
2991                                           sizeof(validMetaData));
2992 
2993         // Set the key with a low CAS value and real nmeta
2994         checkeq(ENGINE_SUCCESS,
2995                 set_with_meta(h,
2996                               "key3",
2997                               4,
2998                               nullptr,
2999                               0,
3000                               Vbid(0),
3001                               &itemMeta,
3002                               0,
3003                               force,
3004                               PROTOCOL_BINARY_RAW_BYTES,
3005                               nullptr,
3006                               validMetaVector),
3007                 "Expected item to be stored");
3008         checkeq(cb::mcbp::Status::Success, last_status.load(),
3009                 "Expected success");
3010 
3011         itemMeta.cas++;
3012         checkeq(ENGINE_SUCCESS,
3013                 del_with_meta(h,
3014                               "key3",
3015                               4,
3016                               Vbid(0),
3017                               &itemMeta,
3018                               0,
3019                               force,
3020                               nullptr,
3021                               validMetaVector),
3022                 "Expected delete OK");
3023         checkeq(cb::mcbp::Status::Success, last_status.load(),
3024                 "Expected success");
3025     }
3026 
3027     {
3028         with_cas_metadata4 validMetaData = {META_EXT_VERSION_ONE,
3029                                             {CMD_META_ADJUSTED_TIME,
3030                                              htons(sizeof(int64_t)), -1},
3031                                             {CMD_META_CONFLICT_RES_MODE,
3032                                              htons(sizeof(uint8_t)), 0xff}};
3033         std::vector<char> validMetaVector(reinterpret_cast<char*>(&validMetaData),
3034                                           reinterpret_cast<char*>(&validMetaData) +
3035                                           sizeof(validMetaData));
3036 
3037         // Set the key with a low CAS value and real nmeta
3038         checkeq(ENGINE_SUCCESS,
3039                 set_with_meta(h,
3040                               "key4",
3041                               4,
3042                               NULL,
3043                               0,
3044                               Vbid(0),
3045                               &itemMeta,
3046                               0,
3047                               force,
3048                               PROTOCOL_BINARY_RAW_BYTES,
3049                               nullptr,
3050                               validMetaVector),
3051                 "Expected item to be stored");
3052         checkeq(cb::mcbp::Status::Success, last_status.load(),
3053                 "Expected success");
3054 
3055         itemMeta.cas++;
3056         checkeq(ENGINE_SUCCESS,
3057                 del_with_meta(h,
3058                               "key4",
3059                               4,
3060                               Vbid(0),
3061                               &itemMeta,
3062                               0,
3063                               force,
3064                               nullptr,
3065                               validMetaVector),
3066                 "Expected delete ok");
3067         checkeq(cb::mcbp::Status::Success, last_status.load(),
3068                 "Expected success");
3069     }
3070 
3071     return SUCCESS;
3072 }
3073 
3074 // A delete_with_meta with a large seqno (upper 16-bits dirty) should trigger
3075 // a conflict if evicted and a second identical delete_with_meta occurs
test_MB29119(EngineIface* h)3076 static enum test_result test_MB29119(EngineIface* h) {
3077     const char* key1 = "delete_with_meta_key1";
3078     const size_t keylen = strlen(key1);
3079     RawItemMetaData itemMeta;
3080 
3081     // Overflow seqno from 48-bits
3082     itemMeta.revSeqno = 0x0080a80000000001;
3083     itemMeta.cas = 0xdeadbeef;
3084     itemMeta.exptime = 0;
3085     itemMeta.flags = 0xdeadbeef;
3086 
3087     const void* cookie = testHarness->create_cookie();
3088 
3089     // delete an item with meta data
3090     checkeq(ENGINE_SUCCESS,
3091             del_with_meta(h,
3092                           key1,
3093                           keylen,
3094                           Vbid(0),
3095                           &itemMeta,
3096                           0 /*cas*/,
3097                           0 /*options*/,
3098                           cookie),
3099             "Expected delete OK");
3100 
3101     checkeq(cb::mcbp::Status::Success,
3102             last_status.load(),
3103             "Expected success");
3104 
3105     wait_for_flusher_to_settle(h);
3106     // evict the key
3107     evict_key(h, key1);
3108 
3109     // Same key
3110     checkeq(ENGINE_KEY_EEXISTS,
3111             del_with_meta(h,
3112                           key1,
3113                           keylen,
3114                           Vbid(0),
3115                           &itemMeta,
3116                           0 /*cas*/,
3117                           0 /*options*/,
3118                           cookie),
3119             "Expected EEXISTS");
3120 
3121     testHarness->destroy_cookie(cookie);
3122     return SUCCESS;
3123 }
3124 
3125 /*
3126  * Test that we can send option of IS_EXPIRATION
3127  */
test_expiration_options(EngineIface* h)3128 static enum test_result test_expiration_options(EngineIface* h) {
3129     const char* key = "delete_with_meta_key";
3130     const size_t keylen = strlen(key);
3131     ItemMetaData itemMeta;
3132     itemMeta.revSeqno = 10;
3133     itemMeta.cas = 0x1;
3134     itemMeta.exptime = 10;
3135     itemMeta.flags = 0xdeadbeef;
3136 
3137     // store an item
3138     checkeq(ENGINE_SUCCESS,
3139             store(h,
3140                   NULL,
3141                   OPERATION_SET,
3142                   key,
3143                   "somevalue",
3144                   nullptr,
3145                   0,
3146                   Vbid(0),
3147                   100),
3148             "Failed set.");
3149     wait_for_flusher_to_settle(h);
3150 
3151     // delete an item with meta data indicating expiration
3152     checkeq(ENGINE_SUCCESS,
3153             del_with_meta(h, key, keylen, Vbid(0), &itemMeta, 0, IS_EXPIRATION),
3154             "Expected delete success");
3155     checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
3156 
3157     return SUCCESS;
3158 }
3159 
3160 // Test manifest //////////////////////////////////////////////////////////////
3161 
3162 const char *default_dbname = "./ep_testsuite_xdcr";
3163 
3164 BaseTestCase testsuite_testcases[] = {
3165 
3166         // XDCR unit tests
3167         TestCase("get meta",
3168                  test_get_meta,
3169                  test_setup,
3170                  teardown,
3171                  NULL,
3172                  prepare,
3173                  cleanup),
3174         TestCase("get meta with extras",
3175                  test_get_meta_with_extras,
3176                  test_setup,
3177                  teardown,
3178                  NULL,
3179                  prepare,
3180                  cleanup),
3181         TestCase("get meta deleted",
3182                  test_get_meta_deleted,
3183                  test_setup,
3184                  teardown,
3185                  NULL,
3186                  prepare,
3187                  cleanup),
3188         TestCase("get meta nonexistent",
3189                  test_get_meta_nonexistent,
3190                  test_setup,
3191                  teardown,
3192                  NULL,
3193                  prepare,
3194                  cleanup),
3195         TestCase("get meta followed by get",
3196                  test_get_meta_with_get,
3197                  test_setup,
3198                  teardown,
3199                  NULL,
3200                  prepare,
3201                  cleanup),
3202         TestCase("get meta followed by set",
3203                  test_get_meta_with_set,
3204                  test_setup,
3205                  teardown,
3206                  nullptr,
3207                  /* TODO RDB: curr_items not correct under Rocks */
3208                  prepare_skip_broken_under_rocks,
3209                  cleanup),
3210         TestCase("get meta followed by delete",
3211                  test_get_meta_with_delete,
3212                  test_setup,
3213                  teardown,
3214                  NULL,
3215                  prepare,
3216                  cleanup),
3217         TestCase("get meta with xattr",
3218                  test_get_meta_with_xattr,
3219                  test_setup,
3220                  teardown,
3221                  NULL,
3222                  prepare,
3223                  cleanup),
3224         TestCase("add with meta",
3225                  test_add_with_meta,
3226                  test_setup,
3227                  teardown,
3228                  NULL,
3229                  prepare,
3230                  cleanup),
3231         TestCase("delete with meta",
3232                  test_delete_with_meta,
3233                  test_setup,
3234                  teardown,
3235                  NULL,
3236                  prepare,
3237                  cleanup),
3238         TestCase("delete with meta deleted",
3239                  test_delete_with_meta_deleted,
3240                  test_setup,
3241                  teardown,
3242                  nullptr,
3243                  /* TODO RDB: curr_items not correct under Rocks */
3244                  prepare_skip_broken_under_rocks,
3245                  cleanup),
3246         TestCase("delete with meta nonexistent",
3247                  test_delete_with_meta_nonexistent,
3248                  test_setup,
3249                  teardown,
3250                  nullptr,
3251                  prepare,
3252                  cleanup),
3253         TestCase("delete with meta nonexistent no temp",
3254                  test_delete_with_meta_nonexistent_no_temp,
3255                  test_setup,
3256                  teardown,
3257                  nullptr,
3258                  prepare,
3259                  cleanup),
3260         TestCase("delete_with_meta race with concurrent delete",
3261                  test_delete_with_meta_race_with_delete,
3262                  test_setup,
3263                  teardown,
3264                  NULL,
3265                  prepare,
3266                  cleanup),
3267         TestCase("delete_with_meta race with concurrent set",
3268                  test_delete_with_meta_race_with_set,
3269                  test_setup,
3270                  teardown,
3271                  NULL,
3272                  prepare,
3273                  cleanup),
3274         TestCase("set with meta",
3275                  test_set_with_meta,
3276                  test_setup,
3277                  teardown,
3278                  NULL,
3279                  /* TODO RDB: curr_items not correct under Rocks when full
3280                   * eviction */
3281                  prepare_ep_bucket_skip_broken_under_rocks_full_eviction,
3282                  cleanup),
3283         TestCase("set with meta by force",
3284                  test_set_with_meta_by_force,
3285                  test_setup,
3286                  teardown,
3287                  NULL,
3288                  prepare,
3289                  cleanup),
3290         TestCase("set with meta deleted",
3291                  test_set_with_meta_deleted,
3292                  test_setup,
3293                  teardown,
3294                  nullptr,
3295                  /* TODO RDB: curr_items not correct under Rocks when full
3296                   * eviction */
3297                  prepare_skip_broken_under_rocks_full_eviction,
3298                  cleanup),
3299         TestCase("set with meta nonexistent",
3300                  test_set_with_meta_nonexistent,
3301                  test_setup,
3302                  teardown,
3303                  NULL,
3304                  prepare,
3305                  cleanup),
3306         TestCase("set_with_meta race with concurrent set",
3307                  test_set_with_meta_race_with_set,
3308                  test_setup,
3309                  teardown,
3310                  NULL,
3311                  prepare,
3312                  cleanup),
3313         TestCase("set_with_meta race with concurrent delete",
3314                  test_set_with_meta_race_with_delete,
3315                  test_setup,
3316                  teardown,
3317                  NULL,
3318                  prepare,
3319                  cleanup),
3320         TestCase("test set_with_meta exp persisted",
3321                  test_exp_persisted_set_del,
3322                  test_setup,
3323                  teardown,
3324                  "exp_pager_stime=3",
3325                  /* TODO RDB: curr_items not correct under Rocks in full
3326                   * eviction */
3327                  /* Requires persistence */
3328                  prepare_ep_bucket_skip_broken_under_rocks_full_eviction,
3329                  cleanup),
3330         TestCase("test del meta conflict resolution",
3331                  test_del_meta_conflict_resolution,
3332                  test_setup,
3333                  teardown,
3334                  nullptr,
3335                  /* TODO RDB: curr_items not correct under Rocks in full
3336                   * eviction */
3337                  prepare_skip_broken_under_rocks_full_eviction,
3338                  cleanup),
3339         TestCase("test add meta conflict resolution",
3340                  test_add_meta_conflict_resolution,
3341                  test_setup,
3342                  teardown,
3343                  NULL,
3344                  /* TODO RDB: curr_items not correct under Rocks in full
3345                   * eviction */
3346                  prepare_skip_broken_under_rocks_full_eviction,
3347                  cleanup),
3348         TestCase("test set meta conflict resolution",
3349                  test_set_meta_conflict_resolution,
3350                  test_setup,
3351                  teardown,
3352                  NULL,
3353                  prepare,
3354                  cleanup),
3355         TestCase("test del meta lww conflict resolution",
3356                  test_del_meta_lww_conflict_resolution,
3357                  test_setup,
3358                  teardown,
3359                  "conflict_resolution_type=lww",
3360                  /* TODO RDB: curr_items not correct under Rocks in full
3361                   * eviction */
3362                  prepare_skip_broken_under_rocks_full_eviction,
3363                  cleanup),
3364         TestCase("test set meta lww conflict resolution",
3365                  test_set_meta_lww_conflict_resolution,
3366                  test_setup,
3367                  teardown,
3368                  "conflict_resolution_type=lww",
3369                  prepare,
3370                  cleanup),
3371         TestCase("set with meta xattr",
3372                  test_set_with_meta_xattr,
3373                  test_setup,
3374                  teardown,
3375                  nullptr,
3376                  prepare,
3377                  cleanup),
3378         TestCase("set with meta lww xattr",
3379                  test_set_with_meta_xattr,
3380                  test_setup,
3381                  teardown,
3382                  "conflict_resolution_type=lww",
3383                  prepare,
3384                  cleanup),
3385         TestCase("delete with meta xattr",
3386                  test_delete_with_meta_xattr,
3387                  test_setup,
3388                  teardown,
3389                  nullptr,
3390                  prepare,
3391                  cleanup),
3392         TestCase("delete with meta lww xattr",
3393                  test_delete_with_meta_xattr,
3394                  test_setup,
3395                  teardown,
3396                  "conflict_resolution_type=lww",
3397                  prepare,
3398                  cleanup),
3399         TestCase("temp item deletion",
3400                  test_temp_item_deletion,
3401                  test_setup,
3402