1/* -*- MODE: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/*
3 *     Copyright 2016 Couchbase, Inc
4 *
5 *   Licensed under the Apache License, Version 2.0 (the "License");
6 *   you may not use this file except in compliance with the License.
7 *   You may obtain a copy of the License at
8 *
9 *       http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *   Unless required by applicable law or agreed to in writing, software
12 *   distributed under the License is distributed on an "AS IS" BASIS,
13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *   See the License for the specific language governing permissions and
15 *   limitations under the License.
16 */
17
18/*
19 * Testsuite for XDCR-related functionality in ep-engine.
20 */
21#include "ep_test_apis.h"
22#include "ep_testsuite_common.h"
23#include "hlc.h"
24#include <platform/cb_malloc.h>
25#include <string_utilities.h>
26#include <xattr/blob.h>
27#include <xattr/utils.h>
28
29// Helper functions ///////////////////////////////////////////////////////////
30
31static void verifyMetaData(const ItemMetaData& imd, const item_info& metadata) {
32    checkeq(uint64_t(imd.revSeqno), metadata.seqno, "Seqno didn't match");
33    checkeq(imd.cas, metadata.cas, "Cas didn't match");
34    checkeq(imd.exptime,
35            static_cast<time_t>(metadata.exptime),
36            "Expiration time didn't match");
37    checkeq(imd.flags, metadata.flags, "Flags didn't match");
38}
39
40/**
41 * Create an XATTR document using the supplied string as the body
42 * @returns vector containing the body bytes
43 */
44static std::vector<char> createXattrValue(const std::string& body) {
45    cb::xattr::Blob blob;
46
47    //Add a few XAttrs
48    blob.set("user", "{\"author\":\"bubba\"}");
49    blob.set("_sync", "{\"cas\":\"0xdeadbeefcafefeed\"}");
50    blob.set("meta", "{\"content-type\":\"text\"}");
51
52    auto xattr_value = blob.finalize();
53
54    // append body to the xattrs and store in data
55    std::vector<char> data;
56    std::copy(xattr_value.buf, xattr_value.buf + xattr_value.len,
57              std::back_inserter(data));
58    std::copy(body.c_str(), body.c_str() + body.size(),
59              std::back_inserter(data));
60
61    return data;
62}
63
64
65// Testcases //////////////////////////////////////////////////////////////////
66
67static enum test_result test_get_meta(EngineIface* h) {
68    char const *key = "test_get_meta";
69    item *i = NULL;
70    checkeq(ENGINE_SUCCESS,
71            store(h, NULL, OPERATION_SET, key, "somevalue", &i),
72            "Failed set.");
73    Item *it = reinterpret_cast<Item*>(i);
74    // check the stat
75    auto temp = get_int_stat(h, "ep_num_ops_get_meta");
76    checkeq(0, temp, "Expect zero getMeta ops");
77
78    cb::EngineErrorMetadataPair errorMetaPair;
79    check(get_meta(h, key, errorMetaPair), "Expected to get meta");
80
81    ItemMetaData metadata(it->getCas(), it->getRevSeqno(),
82                          it->getFlags(), it->getExptime());
83    verifyMetaData(metadata, errorMetaPair.second);
84
85    // check the stat again
86    temp = get_int_stat(h, "ep_num_ops_get_meta");
87    checkeq(1, temp, "Expect one getMeta op");
88
89    h->release(i);
90    return SUCCESS;
91}
92
93static enum test_result test_get_meta_with_extras(EngineIface* h) {
94    const char *key1 = "test_getm_one";
95    item *i = NULL;
96    checkeq(ENGINE_SUCCESS,
97            store(h, NULL, OPERATION_SET, key1, "somevalue", &i),
98            "Failed set.");
99
100    wait_for_flusher_to_settle(h);
101
102    Item *it1 = reinterpret_cast<Item*>(i);
103    // check the stat
104    auto temp = get_int_stat(h, "ep_num_ops_get_meta");
105    checkeq(0, temp, "Expect zero getMeta ops");
106
107    cb::EngineErrorMetadataPair errorMetaPair;
108
109    check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
110    ItemMetaData metadata1(it1->getCas(), it1->getRevSeqno(),
111                           it1->getFlags(), it1->getExptime());
112    verifyMetaData(metadata1, errorMetaPair.second);
113    // check the stat again
114    temp = get_int_stat(h, "ep_num_ops_get_meta");
115    checkeq(1, temp, "Expect one getMeta op");
116    h->release(i);
117
118    if (isWarmupEnabled(h)) {
119        // restart
120        testHarness->reload_engine(&h,
121                                   testHarness->engine_path,
122                                   testHarness->get_current_testcase()->cfg,
123                                   true,
124                                   true);
125
126        wait_for_warmup_complete(h);
127
128        check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
129        verifyMetaData(metadata1, errorMetaPair.second);
130    }
131
132    return SUCCESS;
133}
134
135static enum test_result test_get_meta_deleted(EngineIface* h) {
136    char const *key = "k1";
137    item *i = NULL;
138
139    checkeq(ENGINE_SUCCESS,
140            store(h, NULL, OPERATION_SET, key, "somevalue"),
141            "Failed set.");
142    checkeq(ENGINE_SUCCESS,
143            store(h, NULL, OPERATION_SET, key, "somevalue", &i),
144            "Failed set.");
145
146    Item *it = reinterpret_cast<Item*>(i);
147    wait_for_flusher_to_settle(h);
148
149    checkeq(ENGINE_SUCCESS,
150            del(h, key, it->getCas(), Vbid(0)),
151            "Delete failed");
152    wait_for_flusher_to_settle(h);
153
154    // check the stat
155    int temp = get_int_stat(h, "ep_num_ops_get_meta");
156   checkeq(0, temp, "Expect zero getMeta ops");
157
158    cb::EngineErrorMetadataPair errorMetaPair;
159    check(get_meta(h, key, errorMetaPair), "Expected to get meta");
160    checkeq(DocumentState::Deleted,
161            errorMetaPair.second.document_state,
162            "Expected deleted flag to be set");
163    checkeq(errorMetaPair.second.seqno, (it->getRevSeqno() + 1),
164          "Expected seqno to match");
165    checkne(errorMetaPair.second.cas, it->getCas(),
166          "Expected cas to be different");
167    checkeq(errorMetaPair.second.flags, it->getFlags(),
168          "Expected flags to match");
169
170    // check the stat again
171    temp = get_int_stat(h, "ep_num_ops_get_meta");
172    checkeq(1, temp, "Expect one getMeta op");
173
174    h->release(i);
175    return SUCCESS;
176}
177
178static enum test_result test_get_meta_nonexistent(EngineIface* h) {
179    char const *key = "k1";
180
181    // check the stat
182    int temp = get_int_stat(h, "ep_num_ops_get_meta");
183   checkeq(0, temp, "Expect zero getMeta ops");
184
185    cb::EngineErrorMetadataPair errorMetaPair;
186    check(!get_meta(h, key, errorMetaPair),
187          "Expected get meta to return false");
188    checkeq(cb::engine_errc::no_such_key,
189            errorMetaPair.first,
190            "Expected no_such_key");
191
192    // check the stat again
193    temp = get_int_stat(h, "ep_num_ops_get_meta");
194    checkeq(1, temp, "Expect one getMeta ops");
195
196    return SUCCESS;
197}
198
199static enum test_result test_get_meta_with_get(EngineIface* h) {
200    char const *key1 = "key1";
201    char const *key2 = "key2";
202
203    // test get_meta followed by get for an existing key. should pass.
204    checkeq(ENGINE_SUCCESS,
205            store(h, NULL, OPERATION_SET, key1, "somevalue"),
206            "Failed set.");
207    wait_for_flusher_to_settle(h);
208    // check the stat
209    int temp = get_int_stat(h, "ep_num_ops_get_meta");
210   checkeq(0, temp, "Expect zero getMeta ops");
211
212    cb::EngineErrorMetadataPair errorMetaPair;
213
214    check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
215    auto ret = get(h, NULL, key1, Vbid(0));
216    checkeq(cb::engine_errc::success, ret.first,
217            "Expected get success");
218    ret.second.reset();
219    // check the stat again
220    temp = get_int_stat(h, "ep_num_ops_get_meta");
221   checkeq(1, temp, "Expect one getMeta op");
222
223    // test get_meta followed by get for a deleted key. should fail.
224    checkeq(ENGINE_SUCCESS, del(h, key1, 0, Vbid(0)), "Delete failed");
225    wait_for_flusher_to_settle(h);
226    check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
227    checkeq(DocumentState::Deleted,
228            errorMetaPair.second.document_state,
229            "Expected deleted flag to be set");
230    checkeq(cb::engine_errc::no_such_key,
231            get(h, NULL, key1, Vbid(0)).first,
232            "Expected enoent");
233    // check the stat again
234    temp = get_int_stat(h, "ep_num_ops_get_meta");
235    checkeq(2, temp, "Expect more getMeta ops");
236
237    // test get_meta followed by get for a nonexistent key. should fail.
238    check(!get_meta(h, key2, errorMetaPair),
239          "Expected get meta to return false");
240    checkeq(cb::engine_errc::no_such_key,
241            errorMetaPair.first,
242            "Expected no_such_key");
243    checkeq(cb::engine_errc::no_such_key,
244            get(h, NULL, key2, Vbid(0)).first,
245            "Expected enoent");
246    // check the stat again
247    temp = get_int_stat(h, "ep_num_ops_get_meta");
248    checkeq(3, temp, "Expected one extra getMeta ops");
249
250    return SUCCESS;
251}
252
253static enum test_result test_get_meta_with_set(EngineIface* h) {
254    char const *key1 = "key1";
255    char const *key2 = "key2";
256
257    ItemMetaData itm_meta;
258
259    // test get_meta followed by set for an existing key. should pass.
260    checkeq(ENGINE_SUCCESS,
261            store(h, NULL, OPERATION_SET, key1, "somevalue"),
262            "Failed set.");
263    wait_for_flusher_to_settle(h);
264    wait_for_stat_to_be(h, "curr_items", 1);
265
266    // check the stat
267    checkeq(0,
268            get_int_stat(h, "ep_num_ops_get_meta"),
269            "Expect zero getMeta ops");
270
271    cb::EngineErrorMetadataPair errorMetaPair;
272
273    check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
274    checkeq(ENGINE_SUCCESS,
275            store(h, NULL, OPERATION_SET, key1, "someothervalue"),
276            "Failed set.");
277    // check the stat
278    checkeq(1, get_int_stat(h, "ep_num_ops_get_meta"), "Expect one getMeta op");
279    checkeq(1, get_int_stat(h, "curr_items"), "Expected single curr_items");
280    checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
281
282    // check curr, temp item counts
283    checkeq(1, get_int_stat(h, "curr_items"), "Expected single curr_items");
284    checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
285
286    // test get_meta followed by set for a deleted key. should pass.
287    checkeq(ENGINE_SUCCESS, del(h, key1, 0, Vbid(0)), "Delete failed");
288    wait_for_flusher_to_settle(h);
289
290    wait_for_stat_to_be(h, "curr_items", 0);
291    check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
292    checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
293    checkPersistentBucketTempItems(h, 1);
294
295    checkeq(DocumentState::Deleted,
296            errorMetaPair.second.document_state,
297            "Expected deleted flag to be set");
298    checkeq(ENGINE_SUCCESS,
299            store(h, NULL, OPERATION_SET, key1, "someothervalue"),
300            "Failed set.");
301    wait_for_flusher_to_settle(h);
302
303    checkeq(1, get_int_stat(h, "curr_items"), "Expected single curr_items");
304    checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
305
306    // check the stat
307    checkeq(2,
308            get_int_stat(h, "ep_num_ops_get_meta"),
309            "Expect more getMeta ops");
310
311    // test get_meta followed by set for a nonexistent key. should pass.
312    check(!get_meta(h, key2, errorMetaPair),
313          "Expected get meta to return false");
314    checkeq(cb::engine_errc::no_such_key,
315            errorMetaPair.first,
316            "Expected no_such_key");
317    checkeq(ENGINE_SUCCESS,
318            store(h, NULL, OPERATION_SET, key2, "someothervalue"),
319            "Failed set.");
320    // check the stat again
321    checkeq(3,
322            get_int_stat(h, "ep_num_ops_get_meta"),
323            "Expected one extra getMeta ops");
324
325    return SUCCESS;
326}
327
328static enum test_result test_get_meta_with_delete(EngineIface* h) {
329    char const *key1 = "key1";
330    char const *key2 = "key2";
331
332    // test get_meta followed by delete for an existing key. should pass.
333    checkeq(ENGINE_SUCCESS,
334            store(h, NULL, OPERATION_SET, key1, "somevalue"),
335            "Failed set.");
336    wait_for_flusher_to_settle(h);
337    // check the stat
338    int temp = get_int_stat(h, "ep_num_ops_get_meta");
339   checkeq(0, temp, "Expect zero getMeta ops");
340
341    cb::EngineErrorMetadataPair errorMetaPair;
342
343    check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
344    checkeq(ENGINE_SUCCESS, del(h, key1, 0, Vbid(0)), "Delete failed");
345    // check the stat
346    temp = get_int_stat(h, "ep_num_ops_get_meta");
347   checkeq(1, temp, "Expect one getMeta op");
348
349    // test get_meta followed by delete for a deleted key. should fail.
350    wait_for_flusher_to_settle(h);
351    check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
352    checkeq(DocumentState::Deleted,
353            errorMetaPair.second.document_state,
354            "Expected deleted flag to be set");
355    checkeq(ENGINE_KEY_ENOENT, del(h, key1, 0, Vbid(0)), "Expected enoent");
356    // check the stat
357    temp = get_int_stat(h, "ep_num_ops_get_meta");
358    checkeq(2, temp, "Expect more getMeta op");
359
360    // test get_meta followed by delete for a nonexistent key. should fail.
361    check(!get_meta(h, key2, errorMetaPair),
362          "Expected get meta to return false");
363    checkeq(cb::engine_errc::no_such_key,
364            errorMetaPair.first,
365            "Expected no_such_key");
366    checkeq(ENGINE_KEY_ENOENT, del(h, key2, 0, Vbid(0)), "Expected enoent");
367    // check the stat again
368    temp = get_int_stat(h, "ep_num_ops_get_meta");
369    checkeq(3, temp, "Expected one extra getMeta ops");
370
371    return SUCCESS;
372}
373
374static enum test_result test_get_meta_with_xattr(EngineIface* h) {
375    const char* key = "get_meta_key";
376    std::vector<char> data = createXattrValue({"test_expiry_value"});
377
378    const void* cookie = testHarness->create_cookie();
379
380    checkeq(cb::engine_errc::success,
381            storeCasVb11(h,
382                         cookie,
383                         OPERATION_SET,
384                         key,
385                         reinterpret_cast<char*>(data.data()),
386                         data.size(),
387                         9258,
388                         0,
389                         Vbid(0),
390                         0,
391                         PROTOCOL_BINARY_DATATYPE_XATTR)
392                    .first,
393            "Failed to store xattr document");
394
395    if (isPersistentBucket(h)) {
396        wait_for_flusher_to_settle(h);
397    }
398
399    cb::EngineErrorMetadataPair errorMetaPair;
400
401    // Check that the datatype is XATTR (at engine level the datatype is always
402    // returned).
403    check(get_meta(h, key, errorMetaPair, cookie), "Get meta command failed");
404    checkeq(PROTOCOL_BINARY_DATATYPE_XATTR,
405            errorMetaPair.second.datatype,
406            "Datatype is not XATTR");
407
408    if (isPersistentBucket(h)) {
409        //Evict the key
410        evict_key(h, key);
411
412        // This should result in a bg fetch
413        check(get_meta(h, key, errorMetaPair, cookie),
414              "Get meta command failed");
415        checkeq(PROTOCOL_BINARY_DATATYPE_XATTR,
416                errorMetaPair.second.datatype,
417                "Datatype is not XATTR");
418    }
419
420    testHarness->destroy_cookie(cookie);
421
422    return SUCCESS;
423}
424
425/**
426 * Test that we can still get datatype of the deleted item after compaction
427 */
428static enum test_result test_get_meta_mb23905(EngineIface* h) {
429    const char* key = "get_meta_key";
430    std::vector<char> data = createXattrValue({"test_expiry_value"});
431
432    const void* cookie = testHarness->create_cookie();
433
434    checkeq(cb::engine_errc::success,
435            storeCasVb11(h,
436                         cookie,
437                         OPERATION_SET,
438                         key,
439                         reinterpret_cast<char*>(data.data()),
440                         data.size(),
441                         9258,
442                         0,
443                         Vbid(0),
444                         0,
445                         PROTOCOL_BINARY_DATATYPE_XATTR)
446                    .first,
447            "Failed to store xattr document");
448
449    if (isPersistentBucket(h)) {
450        wait_for_flusher_to_settle(h);
451    }
452
453    if (isPersistentBucket(h)) {
454        cb::xattr::Blob systemXattrBlob;
455        systemXattrBlob.set("_sync", "{\"cas\":\"0xdeadbeefcafefeed\"}");
456        auto deletedValue = systemXattrBlob.finalize();
457
458        checkeq(ENGINE_SUCCESS,
459                delete_with_value(h,
460                                  cookie,
461                                  0,
462                                  key,
463                                  {reinterpret_cast<char*>(deletedValue.data()),
464                                   deletedValue.size()},
465                                  cb::mcbp::Datatype::Xattr),
466                "delete_with_value() failed");
467
468        // Run compaction to start using the bloomfilter
469        useconds_t sleepTime = 128;
470        compact_db(h, Vbid(0), Vbid(0), 1, 1, 0);
471        while (get_int_stat(h, "ep_pending_compactions") != 0) {
472            decayingSleep(&sleepTime);
473        }
474
475        cb::EngineErrorMetadataPair errorMetaPair;
476        check(get_meta(h, key, errorMetaPair, cookie),
477              "Get meta command failed");
478        checkeq(PROTOCOL_BINARY_DATATYPE_XATTR,
479                errorMetaPair.second.datatype,
480                "Datatype is not XATTR");
481        checkeq(DocumentState::Deleted,
482                errorMetaPair.second.document_state,
483                "Expected deleted flag to be set");
484    }
485
486    testHarness->destroy_cookie(cookie);
487
488    return SUCCESS;
489}
490
491static enum test_result test_add_with_meta(EngineIface* h) {
492    const char *key = "mykey";
493    const size_t keylen = strlen(key);
494    ItemMetaData itemMeta;
495    int temp = 0;
496
497    // put some random metadata
498    itemMeta.revSeqno = 10;
499    itemMeta.cas = 0xdeadbeef;
500    itemMeta.exptime = 0;
501    itemMeta.flags = 0xdeadbeef;
502    // check the stat
503    temp = get_int_stat(h, "ep_num_ops_set_meta");
504    checkeq(0, temp, "Expect zero setMeta ops");
505
506    // store an item with meta data
507    checkeq(ENGINE_SUCCESS,
508            add_with_meta(h, key, keylen, NULL, 0, Vbid(0), &itemMeta),
509            "Expected to add item");
510    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
511
512    // store the item again, expect key exists
513    checkeq(ENGINE_KEY_EEXISTS,
514            add_with_meta(h, key, keylen, NULL, 0, Vbid(0), &itemMeta, true),
515            "Expected add to fail when the item exists already");
516    // check the stat
517    temp = get_int_stat(h, "ep_num_ops_set_meta");
518    checkeq(1, temp, "Failed op does not count");
519
520    return SUCCESS;
521}
522
523static enum test_result test_delete_with_meta(EngineIface* h) {
524    const char *key1 = "delete_with_meta_key1";
525    const char *key2 = "delete_with_meta_key2";
526    const char *key3 = "delete_with_meta_key3";
527    const size_t keylen = strlen(key1);
528    ItemMetaData itemMeta;
529    uint64_t vb_uuid;
530    uint32_t high_seqno;
531    // check the stat
532    auto temp = get_int_stat(h, "ep_num_ops_del_meta");
533    checkeq(0, temp, "Expect zero setMeta ops");
534
535    // put some random meta data
536    itemMeta.revSeqno = 10;
537    itemMeta.cas = 0xdeadbeef;
538    itemMeta.exptime = 0;
539    itemMeta.flags = 0xdeadbeef;
540
541    // store an item
542    checkeq(ENGINE_SUCCESS,
543            store(h, NULL, OPERATION_SET, key1, "somevalue"),
544            "Failed set.");
545
546    checkeq(ENGINE_SUCCESS,
547            store(h, NULL, OPERATION_SET, key2, "somevalue2"),
548            "Failed set.");
549
550    checkeq(ENGINE_SUCCESS,
551            store(h, NULL, OPERATION_SET, key3, "somevalue3"),
552            "Failed set.");
553
554    vb_uuid = get_ull_stat(h, "vb_0:0:id", "failovers");
555    high_seqno = get_ull_stat(h, "vb_0:high_seqno", "vbucket-seqno");
556
557    const void* cookie = testHarness->create_cookie();
558
559    // delete an item with meta data
560    checkeq(ENGINE_SUCCESS,
561            del_with_meta(h,
562                          key1,
563                          keylen,
564                          Vbid(0),
565                          &itemMeta,
566                          0 /*cas*/,
567                          0 /*options*/,
568                          cookie),
569            "Expected delete OK");
570
571    checkeq(last_uuid.load(), vb_uuid, "Expected valid vbucket uuid");
572    checkeq(last_seqno.load(),
573            static_cast<uint64_t>(high_seqno + 1),
574            "Expected valid sequence number");
575    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
576    // check the stat
577    temp = get_int_stat(h, "ep_num_ops_del_meta");
578    checkeq(1, temp, "Expect more setMeta ops");
579
580    testHarness->set_mutation_extras_handling(cookie, false);
581
582    // delete an item with meta data
583    checkeq(ENGINE_SUCCESS,
584            del_with_meta(h,
585                          key2,
586                          keylen,
587                          Vbid(0),
588                          &itemMeta,
589                          0 /*cas*/,
590                          0 /*options*/,
591                          cookie),
592            "Expected delete OK");
593
594    checkeq(last_uuid.load(), vb_uuid, "Expected valid vbucket uuid");
595    checkeq(last_seqno.load(),
596            static_cast<uint64_t>(high_seqno + 1),
597            "Expected valid sequence number");
598    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
599
600    // delete an item with meta data
601    checkeq(ENGINE_SUCCESS,
602            del_with_meta(h, key3, keylen, Vbid(0), &itemMeta),
603            "Expected delete OK");
604
605    checkeq(last_uuid.load(), vb_uuid, "Expected valid vbucket uuid");
606    checkeq(last_seqno.load(),
607            static_cast<uint64_t>(high_seqno + 3),
608            "Expected valid sequence number");
609    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
610
611    testHarness->destroy_cookie(cookie);
612    return SUCCESS;
613}
614
615static enum test_result test_delete_with_meta_deleted(EngineIface* h) {
616    const char *key = "delete_with_meta_key";
617    const size_t keylen = strlen(key);
618
619    // check the stat
620    checkeq(0,
621            get_int_stat(h, "ep_num_ops_del_meta"),
622            "Expect zero setMeta ops");
623
624    // add a key
625    checkeq(ENGINE_SUCCESS,
626            store(h, NULL, OPERATION_SET, key, "somevalue"),
627            "Failed set.");
628    wait_for_flusher_to_settle(h);
629
630    // delete the key
631    checkeq(ENGINE_SUCCESS, del(h, key, 0, Vbid(0)), "Delete failed");
632    wait_for_flusher_to_settle(h);
633    wait_for_stat_to_be(h, "curr_items", 0);
634
635    cb::EngineErrorMetadataPair errorMetaPair;
636
637    // get metadata of deleted key
638    check(get_meta(h, key, errorMetaPair), "Expected to get meta");
639    checkeq(DocumentState::Deleted,
640            errorMetaPair.second.document_state,
641            "Expected deleted flag to be set");
642    checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
643    checkPersistentBucketTempItems(h, 1);
644
645    // this is the cas to be used with a subsequent delete with meta
646    uint64_t valid_cas = last_cas;
647    uint64_t invalid_cas = 2012;
648    // put some random metadata and delete the item with new meta data
649    ItemMetaData itm_meta(
650            0xdeadbeef, 10, 0xdeadbeef, 1735689600); // expires in 2025
651
652    // do delete with meta with an incorrect cas value. should fail.
653    checkeq(ENGINE_KEY_EEXISTS,
654            del_with_meta(h, key, keylen, Vbid(0), &itm_meta, invalid_cas),
655            "Expected invalid cas error");
656    checkeq(0,
657            get_int_stat(h, "ep_num_ops_del_meta"),
658            "Faild ops does not count");
659    checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
660    checkPersistentBucketTempItems(h, 1);
661
662    // do delete with meta with the correct cas value. should pass.
663    checkeq(ENGINE_SUCCESS,
664            del_with_meta(h, key, keylen, Vbid(0), &itm_meta, valid_cas),
665            "Expected delete oK");
666    wait_for_flusher_to_settle(h);
667
668    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
669    checkeq(1, get_int_stat(h, "ep_num_ops_del_meta"), "Expect some ops");
670    wait_for_stat_to_be(h, "curr_items", 0);
671    checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
672
673    // get metadata again to verify that delete with meta was successful
674    check(get_meta(h, key, errorMetaPair), "Expected to get meta");
675    checkeq(DocumentState::Deleted,
676            errorMetaPair.second.document_state,
677            "Expected deleted flag to be set");
678    checkeq(static_cast<uint64_t>(itm_meta.revSeqno), errorMetaPair.second.seqno,
679            "Expected seqno to match");
680    checkeq(itm_meta.cas, errorMetaPair.second.cas, "Expected cas to match");
681    checkeq(itm_meta.flags, errorMetaPair.second.flags,
682            "Expected flags to match");
683
684    checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
685    checkPersistentBucketTempItems(h, 1);
686
687    return SUCCESS;
688}
689
690static enum test_result test_delete_with_meta_nonexistent(EngineIface* h) {
691    const char *key = "delete_with_meta_key";
692    const size_t keylen = strlen(key);
693
694    // check the stat
695    checkeq(0,
696            get_int_stat(h, "ep_num_ops_del_meta"),
697            "Expect zero setMeta ops");
698
699    cb::EngineErrorMetadataPair errorMetaPair;
700
701    // get metadata of nonexistent key
702    check(!get_meta(h, key, errorMetaPair),
703          "Expected get meta to return false");
704    checkeq(cb::engine_errc::no_such_key,
705            errorMetaPair.first,
706            "Expected no_such_key");
707    checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
708
709    // this is the cas to be used with a subsequent delete with meta
710    uint64_t valid_cas = last_cas;
711    uint64_t invalid_cas = 2012;
712
713    // do delete with meta
714    // put some random metadata and delete the item with new meta data
715    ItemMetaData itm_meta(
716            0xdeadbeef, 10, 0xdeadbeef, 1735689600); // expires in 2025
717
718    // do delete with meta with an incorrect cas value. should fail.
719    checkeq(ENGINE_KEY_EEXISTS,
720            del_with_meta(h, key, keylen, Vbid(0), &itm_meta, invalid_cas),
721            "Expected invalid cas error");
722    // check the stat
723    checkeq(0,
724            get_int_stat(h, "ep_num_ops_del_meta"),
725            "Failed op does not count");
726    checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
727    checkPersistentBucketTempItems(h, 1);
728
729    // do delete with meta with the correct cas value. should pass.
730    checkeq(ENGINE_SUCCESS,
731            del_with_meta(h, key, keylen, Vbid(0), &itm_meta, valid_cas),
732            "Expected delete OK");
733    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
734    wait_for_flusher_to_settle(h);
735
736    // check the stat
737    checkeq(1, get_int_stat(h, "ep_num_ops_del_meta"), "Expect one op");
738    wait_for_stat_to_be(h, "curr_items", 0);
739    checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
740
741    // get metadata again to verify that delete with meta was successful
742    check(get_meta(h, key, errorMetaPair), "Expected to get meta");
743    checkeq(DocumentState::Deleted,
744            errorMetaPair.second.document_state,
745            "Expected deleted flag to be set");
746    checkeq(static_cast<uint64_t>(itm_meta.revSeqno),
747            errorMetaPair.second.seqno,
748            "Expected seqno to match");
749    checkeq(itm_meta.cas, errorMetaPair.second.cas, "Expected cas to match");
750    checkeq(itm_meta.flags, errorMetaPair.second.flags,
751            "Expected flags to match");
752
753    checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
754    checkPersistentBucketTempItems(h, 1);
755
756    return SUCCESS;
757}
758
759static enum test_result test_delete_with_meta_nonexistent_no_temp(
760        EngineIface* h) {
761    const char *key1 = "delete_with_meta_no_temp_key1";
762    const size_t keylen1 = strlen(key1);
763    ItemMetaData itm_meta1;
764
765    // Run compaction to start using the bloomfilter
766    useconds_t sleepTime = 128;
767    compact_db(h, Vbid(0), Vbid(0), 1, 1, 0);
768    while (get_int_stat(h, "ep_pending_compactions") != 0) {
769        decayingSleep(&sleepTime);
770    }
771
772    // put some random metadata and delete the item with new meta data
773    itm_meta1.revSeqno = 10;
774    itm_meta1.cas = 0xdeadbeef;
775    itm_meta1.exptime = 1735689600; // expires in 2025
776    itm_meta1.flags = 0xdeadbeef;
777
778    // do delete with meta with the correct cas value.
779    // skipConflictResolution false
780    checkeq(ENGINE_SUCCESS,
781            del_with_meta(h, key1, keylen1, Vbid(0), &itm_meta1, 0, false),
782            "Expected delete OK");
783    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
784    wait_for_flusher_to_settle(h);
785
786    checkeq(1, get_int_stat(h, "ep_num_ops_del_meta"), "Expect one op");
787    wait_for_stat_to_be(h, "curr_items", 0);
788    checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
789
790    // do delete with meta with the correct cas value.
791    // skipConflictResolution true
792    const char *key2 = "delete_with_meta_no_temp_key2";
793    const size_t keylen2 = strlen(key2);
794    ItemMetaData itm_meta2;
795
796    // put some random metadata and delete the item with new meta data
797    itm_meta2.revSeqno = 10;
798    itm_meta2.cas = 0xdeadbeef;
799    itm_meta2.exptime = 1735689600; // expires in 2025
800    itm_meta2.flags = 0xdeadbeef;
801
802    checkeq(ENGINE_SUCCESS,
803            del_with_meta(h, key2, keylen2, Vbid(0), &itm_meta2, 0, true),
804            "Expected delete OK");
805    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
806    wait_for_flusher_to_settle(h);
807
808    checkeq(2, get_int_stat(h, "ep_num_ops_del_meta"), "Expect one op");
809    wait_for_stat_to_be(h, "curr_items", 0);
810    checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
811
812    return SUCCESS;
813}
814
815static enum test_result test_delete_with_meta_race_with_set(EngineIface* h) {
816    char const *key1 = "key1";
817    const size_t keylen1 = strlen(key1);
818
819    ItemMetaData itm_meta;
820    itm_meta.revSeqno = 10;
821    itm_meta.cas = 0xdeadbeef;
822    itm_meta.exptime = 1735689600; // expires in 2025
823    itm_meta.flags = 0xdeadbeef;
824    // check the stat
825    size_t temp = get_int_stat(h, "ep_num_ops_del_meta");
826    checkeq(size_t{0}, temp, "Expect zero ops");
827
828    //
829    // test race with a concurrent set for an existing key. should fail.
830    //
831
832    // create a new key and do get_meta
833    checkeq(ENGINE_SUCCESS,
834            store(h, NULL, OPERATION_SET, key1, "somevalue"),
835            "Failed set.");
836    wait_for_flusher_to_settle(h);
837
838    cb::EngineErrorMetadataPair errorMetaPair;
839
840    check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
841
842    // do a concurrent set that changes the cas
843    checkeq(ENGINE_SUCCESS,
844            store(h, NULL, OPERATION_SET, key1, "someothervalue"),
845            "Failed set.");
846
847    // attempt delete_with_meta. should fail since cas is no longer valid.
848    checkeq(ENGINE_KEY_EEXISTS,
849            del_with_meta(h,
850                          key1,
851                          keylen1,
852                          Vbid(0),
853                          &itm_meta,
854                          errorMetaPair.second.cas),
855            "Expected invalid cas error");
856    // check the stat
857    temp = get_int_stat(h, "ep_num_ops_del_meta");
858    checkeq(size_t{0}, temp, "Failed op does not count");
859
860    //
861    // test race with a concurrent set for a deleted key. should fail.
862    //
863
864    // do get_meta for the deleted key
865    checkeq(ENGINE_SUCCESS, del(h, key1, 0, Vbid(0)), "Delete failed");
866    wait_for_flusher_to_settle(h);
867
868    check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
869    checkeq(DocumentState::Deleted,
870            errorMetaPair.second.document_state,
871            "Expected deleted flag to be set");
872
873    // do a concurrent set that changes the cas
874    checkeq(ENGINE_SUCCESS,
875            store(h, NULL, OPERATION_SET, key1, "someothervalue"),
876            "Failed set.");
877
878    checkeq(ENGINE_KEY_EEXISTS,
879            del_with_meta(h,
880                          key1,
881                          keylen1,
882                          Vbid(0),
883                          &itm_meta,
884                          errorMetaPair.second.cas),
885            "Expected invalid cas error");
886    // check the stat
887    temp = get_int_stat(h, "ep_num_ops_del_meta");
888    checkeq(size_t{0}, temp, "Failed op does not count");
889
890    return SUCCESS;
891}
892
893static enum test_result test_delete_with_meta_race_with_delete(EngineIface* h) {
894    char const *key1 = "key1";
895    uint16_t keylen1 = (uint16_t)strlen(key1);
896    char const *key2 = "key2";
897    uint16_t keylen2 = (uint16_t)strlen(key2);
898
899    // check the stat
900    size_t temp = get_int_stat(h, "ep_num_ops_del_meta");
901    checkeq(size_t{0}, temp, "Expect zero ops");
902
903    //
904    // test race with a concurrent delete for an existing key. should fail.
905    //
906
907    // create a new key and do get_meta
908    checkeq(ENGINE_SUCCESS,
909            store(h, NULL, OPERATION_SET, key1, "somevalue"),
910            "Failed set.");
911    wait_for_flusher_to_settle(h);
912
913    cb::EngineErrorMetadataPair errorMetaPair;
914
915    check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
916
917    //Store the CAS. This will be used in a subsequent delete_with_meta call
918    uint64_t cas_from_store = errorMetaPair.second.cas;
919
920    //Do a concurrent delete. This should modify the CAS
921    checkeq(ENGINE_SUCCESS, del(h, key1, 0, Vbid(0)), "Delete failed");
922
923    //Get the latest meta data
924    check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
925
926    //Populate the item meta data in such a way, so that we will pass
927    //conflict resolution
928    ItemMetaData itm_meta(errorMetaPair.second.cas,
929                          errorMetaPair.second.seqno + 1,
930                          errorMetaPair.second.flags,
931                          errorMetaPair.second.exptime);
932
933    // attempt delete_with_meta. should fail since cas is no longer valid.
934    checkeq(ENGINE_KEY_EEXISTS,
935            del_with_meta(h, key1, keylen1, Vbid(0), &itm_meta, cas_from_store),
936            "Expected invalid cas error");
937    // check the stat
938    temp = get_int_stat(h, "ep_num_ops_del_meta");
939    checkeq(size_t{0}, temp, "Failed op does not count");
940
941    //
942    // test race with a concurrent delete for a deleted key. should pass since
943    // the delete itself will fail.
944    //
945
946    // do get_meta for the deleted key
947    wait_for_flusher_to_settle(h);
948    check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
949    checkeq(DocumentState::Deleted,
950            errorMetaPair.second.document_state,
951            "Expected deleted flag to be set");
952
953    // do a concurrent delete
954    checkeq(ENGINE_KEY_ENOENT, del(h, key1, 0, Vbid(0)), "Delete failed");
955
956    // attempt delete_with_meta. should pass.
957    checkeq(ENGINE_SUCCESS,
958            del_with_meta(h, key1, keylen1, Vbid(0), &itm_meta, last_cas),
959            "Expected delete OK");
960    checkeq(cb::mcbp::Status::Success, last_status.load(),
961          "Expected delete_with_meta success");
962    // check the stat
963    temp = get_int_stat(h, "ep_num_ops_del_meta");
964    checkeq(size_t{1}, temp, "Expect some ops");
965
966    //
967    // test race with a concurrent delete for a nonexistent key. should pass
968    // since the delete itself will fail.
969    //
970
971    // do get_meta for a nonexisting key
972    check(!get_meta(h, key2, errorMetaPair),
973          "Expected get meta to return false");
974    checkeq(cb::engine_errc::no_such_key,
975            errorMetaPair.first,
976            "Expected no_such_key");
977
978    // do a concurrent delete
979    checkeq(ENGINE_KEY_ENOENT, del(h, key1, 0, Vbid(0)), "Delete failed");
980
981    // attempt delete_with_meta. should pass.
982    checkeq(ENGINE_SUCCESS,
983            del_with_meta(h,
984                          key2,
985                          keylen2,
986                          Vbid(0),
987                          &itm_meta,
988                          errorMetaPair.second.cas),
989            "Expected delete OK");
990    checkeq(cb::mcbp::Status::Success, last_status.load(),
991          "Expected delete_with_meta success");
992    // check the stat
993    temp = get_int_stat(h, "ep_num_ops_del_meta");
994    checkeq(size_t{2}, temp, "Expect some ops");
995
996    return SUCCESS;
997}
998
999static enum test_result test_set_with_meta(EngineIface* h) {
1000    const char* key = "set_with_meta_key";
1001    size_t keylen = strlen(key);
1002    const char* val = "somevalue";
1003    const char* newVal = R"({"json":"yes"})";
1004    size_t newValLen = strlen(newVal);
1005    uint64_t vb_uuid;
1006    uint32_t high_seqno;
1007
1008    // check the stat
1009    checkeq(0, get_int_stat(h, "ep_num_ops_set_meta"), "Expect zero ops");
1010    checkeq(0,
1011            get_int_stat(h, "ep_num_ops_get_meta_on_set_meta"),
1012            "Expect zero ops");
1013    checkeq(0, get_int_stat(h, "curr_items"), "Expect zero items");
1014    checkeq(0, get_int_stat(h, "curr_temp_items"), "Expect zero temp items");
1015
1016    // create a new key
1017    checkeq(ENGINE_SUCCESS,
1018            store(h, NULL, OPERATION_SET, key, val),
1019            "Failed set.");
1020    wait_for_flusher_to_settle(h);
1021
1022    // get metadata for the key
1023    cb::EngineErrorMetadataPair errorMetaPair;
1024    check(get_meta(h, key, errorMetaPair), "Expected to get meta");
1025    checkeq(1, get_int_stat(h, "curr_items"), "Expect one item");
1026    checkeq(0, get_int_stat(h, "curr_temp_items"), "Expect zero temp item");
1027
1028    // this is the cas to be used with a subsequent set with meta
1029    uint64_t cas_for_set = errorMetaPair.second.cas;
1030    // init some random metadata
1031    ItemMetaData itm_meta(0xdeadbeef, 10, 0xdeadbeef, time(NULL) + 300);
1032
1033    char *bigValue = new char[32*1024*1024];
1034    // do set with meta with the value size bigger than the max size allowed.
1035    checkeq(ENGINE_E2BIG,
1036            set_with_meta(h,
1037                          key,
1038                          keylen,
1039                          bigValue,
1040                          32 * 1024 * 1024,
1041                          Vbid(0),
1042                          &itm_meta,
1043                          cas_for_set),
1044            "Expected the max value size exceeding error");
1045    delete []bigValue;
1046
1047    // do set with meta with an incorrect cas value. should fail.
1048    checkeq(ENGINE_KEY_EEXISTS,
1049            set_with_meta(h,
1050                          key,
1051                          keylen,
1052                          newVal,
1053                          newValLen,
1054                          Vbid(0),
1055                          &itm_meta,
1056                          1229),
1057            "Expected invalid cas error");
1058    // check the stat
1059    checkeq(0,
1060            get_int_stat(h, "ep_num_ops_set_meta"),
1061            "Failed op does not count");
1062
1063    vb_uuid = get_ull_stat(h, "vb_0:0:id", "failovers");
1064    high_seqno = get_ull_stat(h, "vb_0:high_seqno", "vbucket-seqno");
1065
1066    const void* cookie = testHarness->create_cookie();
1067    // We are explicitly going to test the !datatype paths, so turn it off.
1068    testHarness->set_datatype_support(cookie, false);
1069
1070    // do set with meta with the correct cas value. should pass.
1071    checkeq(ENGINE_SUCCESS,
1072            set_with_meta(h,
1073                          key,
1074                          keylen,
1075                          newVal,
1076                          newValLen,
1077                          Vbid(0),
1078                          &itm_meta,
1079                          cas_for_set,
1080                          0,
1081                          0,
1082                          cookie),
1083            "Expected item to be stored");
1084    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
1085    checkeq(last_uuid.load(), vb_uuid, "Expected valid vbucket uuid");
1086    checkeq(last_seqno.load(),
1087            static_cast<uint64_t>(high_seqno + 1),
1088            "Expected valid sequence number");
1089
1090    // Check that set_with_meta has marked the JSON input as JSON
1091    item_info info;
1092    check(get_item_info(h, &info, key, Vbid(0)), "get_item_info failed");
1093    checkeq(int(PROTOCOL_BINARY_DATATYPE_JSON), int(info.datatype),
1094        "Expected datatype to now include JSON");
1095
1096    // check the stat
1097    checkeq(1, get_int_stat(h, "ep_num_ops_set_meta"), "Expect some ops");
1098    checkeq(1, get_int_stat(h, "curr_items"), "Expect one item");
1099    checkeq(0, get_int_stat(h, "curr_temp_items"), "Expect zero temp item");
1100
1101    // get metadata again to verify that set with meta was successful
1102    check(get_meta(h, key, errorMetaPair), "Expected to get meta");
1103    checkeq(uint64_t{10},
1104            errorMetaPair.second.seqno,
1105            "Expected seqno to match");
1106    checkeq(uint64_t{0xdeadbeef},
1107            errorMetaPair.second.cas,
1108            "Expected cas to match");
1109    checkeq(uint32_t{0xdeadbeef},
1110            errorMetaPair.second.flags,
1111            "Expected flags to match");
1112
1113    //disable getting vb uuid and seqno as extras
1114    testHarness->set_mutation_extras_handling(cookie, false);
1115    itm_meta.revSeqno++;
1116    cas_for_set = errorMetaPair.second.cas;
1117    checkeq(ENGINE_SUCCESS,
1118            set_with_meta(h,
1119                          key,
1120                          keylen,
1121                          newVal,
1122                          newValLen,
1123                          Vbid(0),
1124                          &itm_meta,
1125                          cas_for_set,
1126                          false,
1127                          0,
1128                          cookie),
1129            "Expected item to be stored");
1130    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
1131    checkeq(last_uuid.load(), vb_uuid, "Expected same vbucket uuid");
1132    checkeq(last_seqno.load(),
1133            static_cast<uint64_t>(high_seqno + 1),
1134            "Expected same sequence number");
1135
1136    itm_meta.revSeqno++;
1137    cas_for_set = last_meta.cas;
1138    checkeq(ENGINE_SUCCESS,
1139            set_with_meta(h,
1140                          key,
1141                          keylen,
1142                          newVal,
1143                          newValLen,
1144                          Vbid(0),
1145                          &itm_meta,
1146                          cas_for_set),
1147            "Expected item to be stored");
1148    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
1149    checkeq(last_uuid.load(), vb_uuid, "Expected valid vbucket uuid");
1150    checkeq(last_seqno.load(),
1151            static_cast<uint64_t>(high_seqno + 3),
1152            "Expected valid sequence number");
1153
1154    // Make sure the item expiration was processed correctly
1155    testHarness->time_travel(301);
1156    auto ret = get(h, NULL, key, Vbid(0));
1157    checkeq(cb::engine_errc::no_such_key, ret.first, "Failed to get value.");
1158
1159    testHarness->destroy_cookie(cookie);
1160    return SUCCESS;
1161}
1162
1163static enum test_result test_set_with_meta_by_force(EngineIface* h) {
1164    const char* key = "set_with_meta_key";
1165    size_t keylen = strlen(key);
1166    const char* val = "somevalue";
1167
1168    // init some random metadata
1169    ItemMetaData itm_meta(0xdeadbeef, 10, 0xdeadbeef, time(NULL) + 300);
1170
1171    // Pass true to force SetWithMeta.
1172    checkeq(ENGINE_SUCCESS,
1173            set_with_meta(h,
1174                          key,
1175                          keylen,
1176                          val,
1177                          strlen(val),
1178                          Vbid(0),
1179                          &itm_meta,
1180                          0,
1181                          true),
1182            "Expected item to be stored");
1183    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
1184    wait_for_flusher_to_settle(h);
1185
1186    // get metadata again to verify that the warmup loads an item correctly.
1187    cb::EngineErrorMetadataPair errorMetaPair;
1188    check(get_meta(h, key, errorMetaPair), "Expected to get meta");
1189    checkeq(uint64_t{10},
1190            errorMetaPair.second.seqno,
1191            "Expected seqno to match");
1192    checkeq(uint64_t{0xdeadbeef},
1193            errorMetaPair.second.cas,
1194            "Expected cas to match");
1195    checkeq(uint32_t{0xdeadbeef},
1196            errorMetaPair.second.flags,
1197            "Expected flags to match");
1198
1199    check_key_value(h, key, val, strlen(val));
1200
1201    return SUCCESS;
1202}
1203
1204static enum test_result test_set_with_meta_deleted(EngineIface* h) {
1205    const char* key = "set_with_meta_key";
1206    size_t keylen = strlen(key);
1207    const char* val = "somevalue";
1208    const char* newVal = "someothervalue";
1209    uint16_t newValLen = (uint16_t)strlen(newVal);
1210
1211    // check the stat
1212    checkeq(0, get_int_stat(h, "ep_num_ops_set_meta"), "Expect zero ops");
1213    checkeq(0,
1214            get_int_stat(h, "ep_num_ops_get_meta_on_set_meta"),
1215            "Expect zero ops");
1216
1217    // create a new key
1218    checkeq(ENGINE_SUCCESS,
1219            store(h, NULL, OPERATION_SET, key, val),
1220            "Failed set.");
1221    wait_for_flusher_to_settle(h);
1222    checkeq(1, get_int_stat(h, "curr_items"), "Expected single curr_items");
1223    checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
1224
1225    // delete the key
1226    checkeq(ENGINE_SUCCESS, del(h, key, 0, Vbid(0)), "Delete failed");
1227    wait_for_flusher_to_settle(h);
1228    wait_for_stat_to_be(h, "curr_items", 0);
1229
1230    cb::EngineErrorMetadataPair errorMetaPair;
1231
1232    // get metadata for the key
1233    check(get_meta(h, key, errorMetaPair), "Expected to get meta");
1234    checkeq(DocumentState::Deleted,
1235            errorMetaPair.second.document_state,
1236            "Expected deleted flag to be set");
1237    checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
1238    checkPersistentBucketTempItems(h, 1);
1239
1240    // this is the cas to be used with a subsequent set with meta
1241    uint64_t cas_for_set = errorMetaPair.second.cas;
1242    // init some random metadata
1243    ItemMetaData itm_meta(
1244            0xdeadbeef, 10, 0xdeadbeef, 1735689600); // expires in 2025
1245
1246    // do set_with_meta with an incorrect cas for a deleted item. should fail.
1247    checkeq(ENGINE_KEY_ENOENT,
1248            set_with_meta(h,
1249                          key,
1250                          keylen,
1251                          newVal,
1252                          newValLen,
1253                          Vbid(0),
1254                          &itm_meta,
1255                          1229),
1256            "Expected key_not_found error");
1257    // check the stat
1258    checkeq(0,
1259            get_int_stat(h, "ep_num_ops_set_meta"),
1260            "Failed op does not count");
1261    checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
1262    checkPersistentBucketTempItems(h, 1);
1263
1264    // do set with meta with the correct cas value. should pass.
1265    checkeq(ENGINE_SUCCESS,
1266            set_with_meta(h,
1267                          key,
1268                          keylen,
1269                          newVal,
1270                          newValLen,
1271                          Vbid(0),
1272                          &itm_meta,
1273                          cas_for_set),
1274            "Expected item to be stored");
1275    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
1276    wait_for_flusher_to_settle(h);
1277
1278    // check the stat
1279    checkeq(1, get_int_stat(h, "ep_num_ops_set_meta"), "Expect some ops");
1280    checkeq(0,
1281            get_int_stat(h, "ep_num_ops_get_meta_on_set_meta"),
1282            "Expect some ops");
1283    checkeq(1, get_int_stat(h, "curr_items"), "Expected single curr_items");
1284    checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
1285
1286    // get metadata again to verify that set with meta was successful
1287    check(get_meta(h, key, errorMetaPair), "Expected to get meta");
1288    ItemMetaData metadata(0xdeadbeef, 10, 0xdeadbeef, 1735689600);
1289    verifyMetaData(metadata, errorMetaPair.second);
1290    checkeq(1, get_int_stat(h, "curr_items"), "Expected single curr_items");
1291    checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
1292
1293    return SUCCESS;
1294}
1295
1296static enum test_result test_set_with_meta_nonexistent(EngineIface* h) {
1297    const char* key = "set_with_meta_key";
1298    size_t keylen = strlen(key);
1299    const char* val = "somevalue";
1300    size_t valLen = strlen(val);
1301
1302    // check the stat
1303    checkeq(0, get_int_stat(h, "ep_num_ops_set_meta"), "Expect zero ops");
1304
1305    cb::EngineErrorMetadataPair errorMetaPair;
1306
1307    // get metadata for the key
1308    check(!get_meta(h, key, errorMetaPair),
1309          "Expected get meta to return false");
1310    checkeq(cb::engine_errc::no_such_key,
1311            errorMetaPair.first,
1312            "Expected no_such_key");
1313    checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
1314
1315    // this is the cas to be used with a subsequent set with meta
1316    uint64_t cas_for_set = errorMetaPair.second.cas;
1317    // init some random metadata
1318    ItemMetaData itm_meta(
1319            0xdeadbeef, 10, 0xdeadbeef, 1735689600); // expires in 2025
1320
1321    // do set_with_meta with an incorrect cas for a non-existent item. should fail.
1322    checkeq(ENGINE_KEY_ENOENT,
1323            set_with_meta(
1324                    h, key, keylen, val, valLen, Vbid(0), &itm_meta, 1229),
1325            "Expected key_not_found error");
1326    // check the stat
1327    checkeq(0,
1328            get_int_stat(h, "ep_num_ops_set_meta"),
1329            "Failed op does not count");
1330    checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
1331
1332    // do set with meta with the correct cas value. should pass.
1333    checkeq(ENGINE_SUCCESS,
1334            set_with_meta(h,
1335                          key,
1336                          keylen,
1337                          val,
1338                          valLen,
1339                          Vbid(0),
1340                          &itm_meta,
1341                          cas_for_set),
1342            "Expected item to be stored");
1343    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
1344    wait_for_flusher_to_settle(h);
1345
1346    // check the stat
1347    checkeq(1, get_int_stat(h, "ep_num_ops_set_meta"), "Expect some ops");
1348    checkeq(1, get_int_stat(h, "curr_items"), "Expected single curr_items");
1349    checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
1350
1351    // get metadata again to verify that set with meta was successful
1352    check(get_meta(h, key, errorMetaPair), "Expected to get meta");
1353    ItemMetaData metadata(0xdeadbeef, 10, 0xdeadbeef, 1735689600);
1354    verifyMetaData(metadata, errorMetaPair.second);
1355    checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
1356    checkeq(1, get_int_stat(h, "curr_items"), "Expected single curr_items");
1357
1358    return SUCCESS;
1359}
1360
1361static enum test_result test_set_with_meta_race_with_set(EngineIface* h) {
1362    char const *key1 = "key1";
1363    size_t keylen1 = strlen(key1);
1364    // check the stat
1365    size_t temp = get_int_stat(h, "ep_num_ops_set_meta");
1366    checkeq(size_t{0}, temp, "Expect zero ops");
1367
1368    //
1369    // test race with a concurrent set for an existing key. should fail.
1370    //
1371
1372    // create a new key and do get_meta
1373    checkeq(ENGINE_SUCCESS,
1374            store(h, NULL, OPERATION_SET, key1, "somevalue"),
1375            "Failed set.");
1376    wait_for_flusher_to_settle(h);
1377    cb::EngineErrorMetadataPair errorMetaPair;
1378    check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
1379
1380    // do a concurrent set that changes the cas
1381    checkeq(ENGINE_SUCCESS,
1382            store(h, NULL, OPERATION_SET, key1, "someothervalue"),
1383            "Failed set.");
1384
1385    // attempt set_with_meta. should fail since cas is no longer valid.
1386    ItemMetaData meta(errorMetaPair.second.cas,
1387                      errorMetaPair.second.seqno + 2,
1388                      errorMetaPair.second.flags,
1389                      errorMetaPair.second.exptime);
1390    checkeq(ENGINE_KEY_EEXISTS,
1391            set_with_meta(h,
1392                          key1,
1393                          keylen1,
1394                          NULL,
1395                          0,
1396                          Vbid(0),
1397                          &meta,
1398                          errorMetaPair.second.cas),
1399            "Expected invalid cas error");
1400    // check the stat
1401    temp = get_int_stat(h, "ep_num_ops_set_meta");
1402    checkeq(size_t{0}, temp, "Failed op does not count");
1403
1404    //
1405    // test race with a concurrent set for a deleted key. should fail.
1406    //
1407
1408    // do get_meta for the deleted key
1409    checkeq(ENGINE_SUCCESS, del(h, key1, 0, Vbid(0)), "Delete failed");
1410    wait_for_flusher_to_settle(h);
1411    check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
1412    checkeq(DocumentState::Deleted,
1413            errorMetaPair.second.document_state,
1414            "Expected deleted flag to be set");
1415
1416    // do a concurrent set that changes the cas
1417    checkeq(ENGINE_SUCCESS,
1418            store(h, NULL, OPERATION_SET, key1, "someothervalue"),
1419            "Failed set.");
1420
1421    // attempt set_with_meta. should fail since cas is no longer valid.
1422    meta = ItemMetaData(errorMetaPair.second.cas,
1423                        errorMetaPair.second.seqno + 2,
1424                        errorMetaPair.second.flags,
1425                        errorMetaPair.second.exptime);
1426    checkeq(ENGINE_KEY_EEXISTS,
1427            set_with_meta(h,
1428                          key1,
1429                          keylen1,
1430                          NULL,
1431                          0,
1432                          Vbid(0),
1433                          &meta,
1434                          errorMetaPair.second.cas),
1435            "Expected invalid cas error");
1436    // check the stat
1437    temp = get_int_stat(h, "ep_num_ops_set_meta");
1438    checkeq(size_t{0}, temp, "Failed op does not count");
1439
1440    return SUCCESS;
1441}
1442
1443static enum test_result test_set_with_meta_race_with_delete(EngineIface* h) {
1444    char const *key1 = "key1";
1445    size_t keylen1 = strlen(key1);
1446    char const *key2 = "key2";
1447    size_t keylen2 = strlen(key2);
1448    // check the stat
1449    size_t temp = get_int_stat(h, "ep_num_ops_set_meta");
1450    checkeq(size_t{0}, temp, "Expect zero op");
1451
1452    //
1453    // test race with a concurrent delete for an existing key. should fail.
1454    //
1455
1456    // create a new key and do get_meta
1457    checkeq(ENGINE_SUCCESS,
1458            store(h, NULL, OPERATION_SET, key1, "somevalue"),
1459            "Failed set.");
1460    wait_for_flusher_to_settle(h);
1461    cb::EngineErrorMetadataPair errorMetaPair;
1462    check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
1463
1464    // do a concurrent delete that changes the cas
1465    checkeq(ENGINE_SUCCESS, del(h, key1, 0, Vbid(0)), "Delete failed");
1466
1467    // attempt set_with_meta. should fail since cas is no longer valid.
1468    ItemMetaData meta(errorMetaPair.second.cas,
1469                      errorMetaPair.second.seqno,
1470                      errorMetaPair.second.flags,
1471                      errorMetaPair.second.exptime);
1472    checkeq(ENGINE_KEY_ENOENT,
1473            set_with_meta(h,
1474                          key1,
1475                          keylen1,
1476                          NULL,
1477                          0,
1478                          Vbid(0),
1479                          &meta,
1480                          errorMetaPair.second.cas,
1481                          true),
1482            (std::string{"Expected invalid cas error (KEY_EXISTS or"
1483                         " KEY_ENOENT), got: "} +
1484             ::to_string(last_status.load()))
1485                    .c_str());
1486
1487    // check the stat
1488    temp = get_int_stat(h, "ep_num_ops_set_meta");
1489    checkeq(size_t{0}, temp, "Expect zero op");
1490
1491    //
1492    // test race with a concurrent delete for a deleted key. should pass since
1493    // the delete will fail.
1494    //
1495
1496    // do get_meta for the deleted key
1497    wait_for_flusher_to_settle(h);
1498    check(get_meta(h, key1, errorMetaPair), "Expected to get meta");
1499    checkeq(DocumentState::Deleted,
1500            errorMetaPair.second.document_state,
1501            "Expected deleted flag to be set");
1502
1503    // do a concurrent delete. should fail.
1504    checkeq(ENGINE_KEY_ENOENT, del(h, key1, 0, Vbid(0)), "Delete failed");
1505
1506    // attempt set_with_meta. should pass since cas is still valid.
1507    meta = ItemMetaData(errorMetaPair.second.cas,
1508                        errorMetaPair.second.seqno,
1509                        errorMetaPair.second.flags,
1510                        errorMetaPair.second.exptime);
1511    checkeq(ENGINE_SUCCESS,
1512            set_with_meta(h,
1513                          key1,
1514                          keylen1,
1515                          NULL,
1516                          0,
1517                          Vbid(0),
1518                          &meta,
1519                          errorMetaPair.second.cas,
1520                          true),
1521            "Expected item to be stored");
1522    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
1523    // check the stat
1524    temp = get_int_stat(h, "ep_num_ops_set_meta");
1525    checkeq(size_t{1}, temp, "Expect some op");
1526
1527    //
1528    // test race with a concurrent delete for a nonexistent key. should pass
1529    // since the delete will fail.
1530    //
1531
1532    // do get_meta for a nonexisting key
1533    check(!get_meta(h, key2, errorMetaPair),
1534          "Expected get meta to return false");
1535    checkeq(cb::engine_errc::no_such_key,
1536            errorMetaPair.first,
1537            "Expected no_such_key");
1538
1539    // do a concurrent delete. should fail.
1540    checkeq(ENGINE_KEY_ENOENT, del(h, key2, 0, Vbid(0)), "Delete failed");
1541
1542    // Attempt set_with_meta. This should pass as we set a new key passing 0 as
1543    // command CAS.
1544    checkeq(ENGINE_SUCCESS,
1545            set_with_meta(h, key2, keylen2, NULL, 0, Vbid(0), &meta, 0, true),
1546            "Expected item to be stored");
1547    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
1548    // check the stat
1549    temp = get_int_stat(h, "ep_num_ops_set_meta");
1550    checkeq(size_t{2}, temp, "Expect some ops");
1551
1552    return SUCCESS;
1553}
1554
1555static enum test_result test_set_with_meta_xattr(EngineIface* h) {
1556    const char* key = "set_with_meta_xattr_key";
1557
1558    // Create XATTR doc with JSON body
1559    std::string value_data = R"({"json":"yes"})";
1560    std::vector<char> data = createXattrValue(value_data);
1561
1562    const void* cookie = testHarness->create_cookie();
1563
1564    // store a value (so we can get its metadata)
1565    checkeq(ENGINE_SUCCESS,
1566            store(h, nullptr, OPERATION_SET, key, value_data.c_str()),
1567            "Failed set.");
1568
1569    cb::EngineErrorMetadataPair errorMetaPair;
1570
1571    check(get_meta(h, key, errorMetaPair), "Expected to get meta");
1572
1573    //init the meta data
1574    ItemMetaData itm_meta(errorMetaPair.second.cas,
1575                          errorMetaPair.second.seqno,
1576                          errorMetaPair.second.flags,
1577                          errorMetaPair.second.exptime);
1578
1579    int force = 0;
1580    if (strstr(testHarness->get_current_testcase()->cfg,
1581               "conflict_resolution_type=lww") != nullptr) {
1582        force = FORCE_ACCEPT_WITH_META_OPS;
1583    }
1584
1585    // Only enable XATTR
1586    testHarness->set_datatype_support(cookie, PROTOCOL_BINARY_DATATYPE_XATTR);
1587
1588    // Set with the same meta data but now with the xattr/json value
1589    checkeq(ENGINE_SUCCESS,
1590            set_with_meta(h,
1591                          key,
1592                          strlen(key),
1593                          data.data(),
1594                          data.size(),
1595                          Vbid(0),
1596                          &itm_meta,
1597                          errorMetaPair.second.cas,
1598                          force,
1599                          PROTOCOL_BINARY_DATATYPE_XATTR,
1600                          cookie),
1601            "Expected item to be stored");
1602
1603    checkeq(cb::mcbp::Status::Success, last_status.load(),
1604            "Expected the set_with_meta to be successful");
1605
1606    // set_with_meta will mark JSON input as JSON
1607    item_info info;
1608    check(get_item_info(h, &info, key, Vbid(0)), "get_item_info failed");
1609    checkeq(int(PROTOCOL_BINARY_DATATYPE_JSON|PROTOCOL_BINARY_DATATYPE_XATTR),
1610        int(info.datatype),
1611        "Expected datatype to be JSON and XATTR");
1612
1613    if (isPersistentBucket(h)) {
1614        wait_for_flusher_to_settle(h);
1615        //evict the key
1616        evict_key(h, key);
1617
1618        //set with the same meta data but now as RAW BYTES.
1619        //This should result in a bg fetch
1620        checkeq(ENGINE_KEY_EEXISTS,
1621                set_with_meta(h,
1622                              key,
1623                              strlen(key),
1624                              data.data(),
1625                              data.size(),
1626                              Vbid(0),
1627                              &itm_meta,
1628                              last_meta.cas,
1629                              force,
1630                              PROTOCOL_BINARY_RAW_BYTES,
1631                              cookie),
1632                "Expected return code to be EEXISTS");
1633    }
1634
1635    testHarness->destroy_cookie(cookie);
1636
1637    return SUCCESS;
1638}
1639
1640static enum test_result test_delete_with_meta_xattr(EngineIface* h) {
1641    const char* key1 = "delete_with_meta_xattr_key1";
1642
1643    const void* cookie = testHarness->create_cookie();
1644
1645    // Create XATTR doc with a JSON body
1646    // In practice a del_with_meta should come along with only XATTR, but
1647    // the command should work with a complete xattr/body blob
1648    std::string body = R"({"key1":"value","key2":"value"})";
1649    std::vector<char> data = createXattrValue(body);
1650    cb::xattr::Blob xattr({data.data(), data.size()}, false);
1651    xattr.prune_user_keys();
1652    data.resize(xattr.finalize().size());
1653
1654    checkeq(ENGINE_SUCCESS,
1655            store(h, nullptr, OPERATION_SET, key1, body.data()),
1656            "Failed to store key1.");
1657
1658    if (isPersistentBucket(h)) {
1659        wait_for_flusher_to_settle(h);
1660    }
1661
1662    // Get the metadata so we can build a del_with_meta
1663    cb::EngineErrorMetadataPair errorMetaPair;
1664    check(get_meta(h, key1, errorMetaPair), "Failed get_meta(key1)");
1665
1666    // Init the meta data for a successful delete
1667    ItemMetaData itm_meta(
1668            errorMetaPair.second.cas + 1, // +1 for CAS conflicts
1669            errorMetaPair.second.seqno + 1, // +1 for seqno conflicts
1670            errorMetaPair.second.flags,
1671            errorMetaPair.second.exptime);
1672
1673    int force = 0;
1674    if (strstr(testHarness->get_current_testcase()->cfg,
1675               "conflict_resolution_type=lww") != nullptr) {
1676        force = FORCE_ACCEPT_WITH_META_OPS;
1677    }
1678
1679    // Now enable XATTR
1680    testHarness->set_datatype_support(cookie, PROTOCOL_BINARY_DATATYPE_XATTR);
1681
1682    // Now delete with a value (marked with XATTR)
1683    checkeq(ENGINE_SUCCESS,
1684            del_with_meta(h,
1685                          key1,
1686                          strlen(key1),
1687                          Vbid(0),
1688                          &itm_meta,
1689                          0, // cas
1690                          force,
1691                          cookie,
1692                          {}, // nmeta
1693                          PROTOCOL_BINARY_DATATYPE_XATTR,
1694                          data),
1695            "Expected delete OK");
1696
1697    /* @todo this should fail, but doesn't. We've just deleted the item, but
1698    it remains in the hash-table (marked deleted), the subsequent set finds the
1699    item and is happy to process this SET_CAS operation (returns success).
1700    A delete with no body would of dropped it from the hash-table and the
1701    SET_CAS would return enoent, delete_with_meta /should/ I think have the same
1702    effect.
1703    checkeq(ENGINE_KEY_ENOENT,
1704            store(h,
1705                  h1,
1706                  nullptr,
1707                  OPERATION_SET,
1708                  key1,
1709                  body.data(),
1710                  nullptr,
1711                  last_cas.load()),
1712            "Failed to store key1.");
1713    */
1714
1715    checkeq(cb::mcbp::Status::Success,
1716            last_status.load(),
1717            "Expected delete_with_meta(key1) to succeed");
1718
1719    // Verify the new value is as expected
1720    item_info info;
1721    auto ret = get(h, nullptr, key1, Vbid(0), DocStateFilter::AliveOrDeleted);
1722    checkeq(cb::engine_errc::success, ret.first, "Failed to get(key1)");
1723
1724    check(h->get_item_info(ret.second.get(), &info),
1725          "Failed get_item_info of key1");
1726
1727    checkeq(data.size(), info.value[0].iov_len, "Value length mismatch");
1728    checkeq(0, memcmp(info.value[0].iov_base, data.data(), data.size()),
1729           "New body mismatch");
1730    checkeq(int(DocumentState::Deleted), int(info.document_state),
1731          "document_state is not DocumentState::Deleted");
1732
1733    // The new value is XATTR
1734    checkeq(int(PROTOCOL_BINARY_DATATYPE_XATTR),
1735            int(info.datatype),
1736            "datatype isn't XATTR");
1737
1738    // @todo implement test for the deletion of a value that has xattr using
1739    // a delete that has none (i.e. non-xattr/!spock client)
1740
1741    testHarness->destroy_cookie(cookie);
1742
1743    return SUCCESS;
1744}
1745
1746static enum test_result test_exp_persisted_set_del(EngineIface* h) {
1747    cb::EngineErrorMetadataPair errorMetaPair;
1748
1749    check(!get_meta(h, "key3", errorMetaPair), "Expected get_meta() to fail");
1750
1751    ItemMetaData itm_meta(1, 1, 0, 0);
1752    checkeq(ENGINE_SUCCESS,
1753            set_with_meta(h,
1754                          "key3",
1755                          4,
1756                          "val0",
1757                          4,
1758                          Vbid(0),
1759                          &itm_meta,
1760                          errorMetaPair.second.cas),
1761            "Expected item to be stored");
1762
1763    itm_meta.revSeqno = 2;
1764    itm_meta.cas = 2;
1765    checkeq(ENGINE_SUCCESS,
1766            set_with_meta(
1767                    h, "key3", 4, "val1", 4, Vbid(0), &itm_meta, last_meta.cas),
1768            "Expected item to be stored");
1769
1770    // MB-21725 Depending on how fast the flusher is, we may see 1 or 2.
1771    wait_for_stat_to_be_gte(h, "ep_total_persisted", 1);
1772
1773    itm_meta.revSeqno = 3;
1774    itm_meta.cas = 3;
1775    itm_meta.exptime = 1735689600; // expires in 2025
1776    checkeq(ENGINE_SUCCESS,
1777            set_with_meta(
1778                    h, "key3", 4, "val1", 4, Vbid(0), &itm_meta, last_meta.cas),
1779            "Expected item to be stored");
1780
1781    testHarness->time_travel(500000000);
1782    // Wait for the item to be expired, either by the pager,
1783    // or by access (as part of persistence callback from a
1784    // previous set - slow disk), or the compactor (unlikely).
1785    wait_for_expired_items_to_be(h, 1);
1786
1787    wait_for_flusher_to_settle(h);
1788    wait_for_stat_to_be(h, "curr_items", 0);
1789
1790    check(get_meta(h, "key3", errorMetaPair), "Expected to get meta");
1791    checkeq(uint64_t{4}, errorMetaPair.second.seqno, "Expected seqno to match");
1792    checkne(uint64_t{3},
1793            errorMetaPair.second.cas,
1794            "Expected cas to be different");
1795    checkeq(uint32_t{0}, errorMetaPair.second.flags, "Expected flags to match");
1796
1797    return SUCCESS;
1798}
1799
1800static enum test_result test_temp_item_deletion(EngineIface* h) {
1801    // Do get_meta for an existing key
1802    char const *k1 = "k1";
1803
1804    checkeq(ENGINE_SUCCESS,
1805            store(h, NULL, OPERATION_SET, k1, "somevalue"),
1806            "Failed set.");
1807    wait_for_flusher_to_settle(h);
1808
1809    checkeq(ENGINE_SUCCESS, del(h, k1, 0, Vbid(0)), "Delete failed");
1810    wait_for_flusher_to_settle(h);
1811    wait_for_stat_to_be(h, "curr_items", 0);
1812
1813    // Issue a get_meta for a deleted key. This will need to bring in a temp
1814    // item into the hashtable as a placeholder for the (deleted) metadata
1815    // which needs to be loaded from disk via BG fetch
1816    // We need to temporarily disable the reader threads as to prevent the
1817    // BGfetch from immediately running and removing our temp_item before
1818    // we've had chance to validate its existence.
1819    set_param(h,
1820              cb::mcbp::request::SetParamPayload::Type::Flush,
1821              "num_reader_threads",
1822              "0");
1823
1824    // Disable nonio so that we have better control of the expirypager
1825    set_param(h,
1826              cb::mcbp::request::SetParamPayload::Type::Flush,
1827              "num_nonio_threads",
1828              "0");
1829
1830    // Tell the harness not to handle EWOULDBLOCK for us - we want it to
1831    // be outstanding while we check the below stats.
1832    const void* cookie = testHarness->create_cookie();
1833    testHarness->set_ewouldblock_handling(cookie, false);
1834
1835    cb::EngineErrorMetadataPair errorMetaPair;
1836
1837    check(!get_meta(h, k1, errorMetaPair, cookie),
1838          "Expected get_meta to fail (EWOULDBLOCK)");
1839    checkeq(cb::engine_errc::would_block,
1840            errorMetaPair.first,
1841            "Expected EWOULDBLOCK");
1842
1843    checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
1844    checkeq(1,
1845            get_int_stat(h, "curr_temp_items"),
1846            "Expected single temp_items");
1847
1848    // Re-enable EWOULDBLOCK handling (and reader threads), and re-issue.
1849    testHarness->set_ewouldblock_handling(cookie, true);
1850    set_param(h,
1851              cb::mcbp::request::SetParamPayload::Type::Flush,
1852              "num_reader_threads",
1853              "1");
1854
1855    check(get_meta(h, k1, errorMetaPair, cookie),
1856          "Expected get_meta to succeed");
1857    checkeq(DocumentState::Deleted,
1858            errorMetaPair.second.document_state,
1859            "Expected deleted flag to be set");
1860
1861    // Even though 2 get_meta calls are made, we may have one or two bg fetches
1862    // done. That is if first bgfetch restores in HT, the deleted item from
1863    // disk, before the second get_meta call tries to find that item in HT,
1864    // we will have only 1 bgfetch
1865    wait_for_stat_to_be_gte(h, "ep_bg_meta_fetched", 1);
1866    int exp_get_meta_ops = get_int_stat(h, "ep_num_ops_get_meta");
1867
1868    // Do get_meta for a non-existing key.
1869    char const *k2 = "k2";
1870    check(!get_meta(h, k2, errorMetaPair), "Expected get meta to return false");
1871    checkeq(cb::engine_errc::no_such_key,
1872            errorMetaPair.first,
1873            "Expected no_such_key");
1874
1875    // This call for get_meta may or may not result in bg fetch because
1876    // bloomfilter may predict that key does not exist.
1877    // However we still must increment the ep_num_ops_get_meta count
1878    checkeq(exp_get_meta_ops + 1,
1879            get_int_stat(h, "ep_num_ops_get_meta"),
1880            "Num get meta ops not as expected");
1881
1882    // Trigger the expiry pager and verify that two temp items are deleted
1883    set_param(h,
1884              cb::mcbp::request::SetParamPayload::Type::Flush,
1885              "num_nonio_threads",
1886              "1");
1887
1888    // When bloom filters are on, it skips 1 of the expired items.
1889    int ep_expired_pager = get_bool_stat(h, "ep_bfilter_enabled") ? 1 : 2;
1890    wait_for_stat_to_be(h, "ep_expired_pager", ep_expired_pager);
1891
1892    checkeq(0, get_int_stat(h, "curr_items"), "Expected zero curr_items");
1893    checkeq(0, get_int_stat(h, "curr_temp_items"), "Expected zero temp_items");
1894
1895    testHarness->destroy_cookie(cookie);
1896
1897    return SUCCESS;
1898}
1899
1900static enum test_result test_add_meta_conflict_resolution(EngineIface* h) {
1901    // put some random metadata
1902    ItemMetaData itemMeta;
1903    itemMeta.revSeqno = 10;
1904    itemMeta.cas = 0xdeadbeef;
1905    itemMeta.exptime = 0;
1906    itemMeta.flags = 0xdeadbeef;
1907
1908    checkeq(ENGINE_SUCCESS,
1909            add_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta),
1910            "Expected to add item");
1911    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
1912
1913    int expected_bg_meta_fetched =
1914            get_bool_stat(h, "ep_bfilter_enabled") ? 0 : 1;
1915    checkeq(expected_bg_meta_fetched,
1916            get_int_stat(h, "ep_bg_meta_fetched"),
1917            "ep_bg_meta_fetched");
1918
1919    checkeq(ENGINE_SUCCESS, del(h, "key", 0, Vbid(0)), "Delete failed");
1920    wait_for_flusher_to_settle(h);
1921    wait_for_stat_to_be(h, "curr_items", 0);
1922
1923    // Check all meta data is the same
1924    itemMeta.revSeqno++;
1925    itemMeta.cas++;
1926    checkeq(ENGINE_KEY_EEXISTS,
1927            add_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta),
1928            "Expected exists");
1929    expected_bg_meta_fetched = 0;
1930    if (isPersistentBucket(h)) {
1931        expected_bg_meta_fetched =
1932                get_bool_stat(h, "ep_bfilter_enabled") ? 1 : 2;
1933    }
1934    checkeq(expected_bg_meta_fetched,
1935            get_int_stat(h, "ep_bg_meta_fetched"),
1936            "ep_bg_meta_fetched");
1937
1938    checkeq(1,
1939            get_int_stat(h, "ep_num_ops_set_meta_res_fail"),
1940            "Expected set meta conflict resolution failure");
1941
1942    // Check has older flags fails
1943    itemMeta.flags = 0xdeadbeee;
1944    checkeq(ENGINE_KEY_EEXISTS,
1945            add_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta),
1946            "Expected exists");
1947    checkeq(2,
1948            get_int_stat(h, "ep_num_ops_set_meta_res_fail"),
1949            "Expected set meta conflict resolution failure");
1950
1951    // Check testing with old seqno
1952    itemMeta.revSeqno--;
1953    checkeq(ENGINE_KEY_EEXISTS,
1954            add_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta),
1955            "Expected exists");
1956    checkeq(3,
1957            get_int_stat(h, "ep_num_ops_set_meta_res_fail"),
1958            "Expected set meta conflict resolution failure");
1959
1960    itemMeta.revSeqno += 10;
1961    checkeq(ENGINE_SUCCESS,
1962            add_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta),
1963            "Expected to add item");
1964    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
1965    checkeq(3,
1966            get_int_stat(h, "ep_num_ops_set_meta_res_fail"),
1967            "Expected set meta conflict resolution failure");
1968
1969    return SUCCESS;
1970}
1971
1972static enum test_result test_set_meta_conflict_resolution(EngineIface* h) {
1973    // put some random metadata
1974    ItemMetaData itemMeta;
1975    itemMeta.revSeqno = 10;
1976    itemMeta.cas = 0xdeadbeef;
1977    itemMeta.exptime = 0;
1978    itemMeta.flags = 0xdeadbeef;
1979
1980    checkeq(0,
1981            get_int_stat(h, "ep_num_ops_set_meta"),
1982            "Expect zero setMeta ops");
1983
1984    checkeq(ENGINE_SUCCESS,
1985            set_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta, 0),
1986            "Expected item to be stored");
1987    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
1988
1989    int expected_bg_meta_fetched = 0;
1990    if (isPersistentBucket(h)) {
1991        expected_bg_meta_fetched =
1992                get_bool_stat(h, "ep_bfilter_enabled") ? 0 : 1;
1993    }
1994    checkeq(expected_bg_meta_fetched,
1995            get_int_stat(h, "ep_bg_meta_fetched"),
1996            "ep_bg_meta_fetched");
1997
1998    // Check all meta data is the same
1999    checkeq(ENGINE_KEY_EEXISTS,
2000            set_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta, 0),
2001            "Expected exists");
2002    checkeq(1,
2003            get_int_stat(h, "ep_num_ops_set_meta_res_fail"),
2004            "Expected set meta conflict resolution failure");
2005
2006    // Check has older flags fails
2007    itemMeta.flags = 0xdeadbeee;
2008    checkeq(ENGINE_KEY_EEXISTS,
2009            set_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta, 0),
2010            "Expected exists");
2011    checkeq(2,
2012            get_int_stat(h, "ep_num_ops_set_meta_res_fail"),
2013            "Expected set meta conflict resolution failure");
2014
2015    // Check has newer flags passes
2016    itemMeta.flags = 0xdeadbeff;
2017    checkeq(ENGINE_SUCCESS,
2018            set_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta, 0),
2019            "Expected item to be stored");
2020    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
2021
2022    // Check that newer exptime wins
2023    itemMeta.exptime = time(NULL) + 10;
2024    checkeq(ENGINE_SUCCESS,
2025            set_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta, 0),
2026            "Expected item to be stored");
2027    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
2028
2029    // Check that smaller exptime loses
2030    itemMeta.exptime = 0;
2031    checkeq(ENGINE_KEY_EEXISTS,
2032            set_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta, 0),
2033            "Expected exists");
2034    checkeq(3,
2035            get_int_stat(h, "ep_num_ops_set_meta_res_fail"),
2036            "Expected set meta conflict resolution failure");
2037
2038    // Check testing with old seqno
2039    itemMeta.revSeqno--;
2040    checkeq(ENGINE_KEY_EEXISTS,
2041            set_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta, 0),
2042            "Expected exists");
2043    checkeq(4,
2044            get_int_stat(h, "ep_num_ops_set_meta_res_fail"),
2045            "Expected set meta conflict resolution failure");
2046
2047    itemMeta.revSeqno += 10;
2048    checkeq(ENGINE_SUCCESS,
2049            set_with_meta(h, "key", 3, NULL, 0, Vbid(0), &itemMeta, 0),
2050            "Expected item to be stored");
2051    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
2052    checkeq(4,
2053            get_int_stat(h, "ep_num_ops_set_meta_res_fail"),
2054            "Expected set meta conflict resolution failure");
2055
2056    expected_bg_meta_fetched =
2057            (!isPersistentBucket(h) || get_bool_stat(h, "ep_bfilter_enabled"))
2058                    ? 0
2059                    : 1;
2060    checkeq(expected_bg_meta_fetched,
2061            get_int_stat(h, "ep_bg_meta_fetched"),
2062            "ep_bg_meta_fetched");
2063
2064    return SUCCESS;
2065}
2066
2067static enum test_result test_set_meta_lww_conflict_resolution(EngineIface* h) {
2068    // put some random metadata
2069    ItemMetaData itemMeta;
2070    itemMeta.revSeqno = 10;
2071    itemMeta.cas = 0xdeadbeef;
2072    itemMeta.exptime = 0;
2073    itemMeta.flags = 0xdeadbeef;
2074
2075    checkeq(0,
2076            get_int_stat(h, "ep_num_ops_set_meta"),
2077            "Expect zero setMeta ops");
2078
2079    checkeq(ENGINE_SUCCESS,
2080            set_with_meta(h,
2081                          "key",
2082                          3,
2083                          NULL,
2084                          0,
2085                          Vbid(0),
2086                          &itemMeta,
2087                          0,
2088                          FORCE_ACCEPT_WITH_META_OPS),
2089            "Expected item to be stored");
2090    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
2091
2092    int expected_bg_meta_fetched =
2093            (!isPersistentBucket(h) || get_bool_stat(h, "ep_bfilter_enabled"))
2094                    ? 0
2095                    : 1;
2096    checkeq(expected_bg_meta_fetched,
2097            get_int_stat(h, "ep_bg_meta_fetched"),
2098            "ep_bg_meta_fetched");
2099
2100    // Check all meta data is the same
2101    checkeq(ENGINE_KEY_EEXISTS,
2102            set_with_meta(h,
2103                          "key",
2104                          3,
2105                          NULL,
2106                          0,
2107                          Vbid(0),
2108                          &itemMeta,
2109                          0,
2110                          FORCE_ACCEPT_WITH_META_OPS),
2111            "Expected exists");
2112    checkeq(1,
2113            get_int_stat(h, "ep_num_ops_set_meta_res_fail"),
2114            "Expected set meta conflict resolution failure");
2115
2116    // Check that an older cas fails
2117    itemMeta.cas = 0xdeadbeee;
2118    checkeq(ENGINE_KEY_EEXISTS,
2119            set_with_meta(h,
2120                          "key",
2121                          3,
2122                          NULL,
2123                          0,
2124                          Vbid(0),
2125                          &itemMeta,
2126                          0,
2127                          FORCE_ACCEPT_WITH_META_OPS),
2128            "Expected exists");
2129    checkeq(2,
2130            get_int_stat(h, "ep_num_ops_set_meta_res_fail"),
2131            "Expected set meta conflict resolution failure");
2132
2133    // Check that a higher cas passes
2134    itemMeta.cas = 0xdeadbeff;
2135    checkeq(ENGINE_SUCCESS,
2136            set_with_meta(h,
2137                          "key",
2138                          3,
2139                          NULL,
2140                          0,
2141                          Vbid(0),
2142                          &itemMeta,
2143                          0,
2144                          FORCE_ACCEPT_WITH_META_OPS),
2145            "Expected item to be stored");
2146    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
2147
2148    // Check that we fail requests if the force flag is not set
2149    itemMeta.cas = 0xdeadbeff + 1;
2150    checkeq(ENGINE_EINVAL,
2151            set_with_meta(
2152                    h, "key", 3, NULL, 0, Vbid(0), &itemMeta, 0, 0 /*options*/),
2153            "Expected EINVAL");
2154
2155    return SUCCESS;
2156}
2157
2158static enum test_result test_del_meta_conflict_resolution(EngineIface* h) {
2159    checkeq(ENGINE_SUCCESS,
2160            store(h, NULL, OPERATION_SET, "key", "somevalue"),
2161            "Failed set.");
2162    wait_for_flusher_to_settle(h);
2163
2164    // put some random metadata
2165    ItemMetaData itemMeta;
2166    itemMeta.revSeqno = 10;
2167    itemMeta.cas = 0xdeadbeef;
2168    itemMeta.exptime = 0;
2169    itemMeta.flags = 0xdeadbeef;
2170
2171    checkeq(ENGINE_SUCCESS,
2172            del_with_meta(h, "key", 3, Vbid(0), &itemMeta),
2173            "Expected delete ok");
2174    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
2175    wait_for_flusher_to_settle(h);
2176    wait_for_stat_to_be(h, "curr_items", 0);
2177
2178    // Check all meta data is the same
2179    checkeq(ENGINE_KEY_EEXISTS,
2180            del_with_meta(h, "key", 3, Vbid(0), &itemMeta),
2181            "Expected exists");
2182    checkeq(1,
2183            get_int_stat(h, "ep_num_ops_del_meta_res_fail"),
2184            "Expected delete meta conflict resolution failure");
2185
2186    // Check has older flags fails
2187    itemMeta.flags = 0xdeadbeee;
2188    checkeq(ENGINE_KEY_EEXISTS,
2189            del_with_meta(h, "key", 3, Vbid(0), &itemMeta),
2190            "Expected exists");
2191    checkeq(2,
2192            get_int_stat(h, "ep_num_ops_del_meta_res_fail"),
2193            "Expected delete meta conflict resolution failure");
2194
2195    // Check that smaller exptime loses
2196    itemMeta.exptime = 0;
2197    checkeq(ENGINE_KEY_EEXISTS,
2198            del_with_meta(h, "key", 3, Vbid(0), &itemMeta),
2199            "Expected exists");
2200    checkeq(3,
2201            get_int_stat(h, "ep_num_ops_del_meta_res_fail"),
2202            "Expected delete meta conflict resolution failure");
2203
2204    // Check testing with old seqno
2205    itemMeta.revSeqno--;
2206    checkeq(ENGINE_KEY_EEXISTS,
2207            del_with_meta(h, "key", 3, Vbid(0), &itemMeta),
2208            "Expected exists");
2209    checkeq(4, get_int_stat(h, "ep_num_ops_del_meta_res_fail"),
2210          "Expected delete meta conflict resolution failure");
2211
2212    itemMeta.revSeqno += 10;
2213    checkeq(ENGINE_SUCCESS,
2214            del_with_meta(h, "key", 3, Vbid(0), &itemMeta),
2215            "Expected delete OK");
2216    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
2217    checkeq(4, get_int_stat(h, "ep_num_ops_del_meta_res_fail"),
2218          "Expected delete meta conflict resolution failure");
2219
2220    return SUCCESS;
2221}
2222
2223static enum test_result test_del_meta_lww_conflict_resolution(EngineIface* h) {
2224    item *i = NULL;
2225    item_info info;
2226
2227    checkeq(ENGINE_SUCCESS,
2228            store(h, NULL, OPERATION_SET, "key", "somevalue", &i),
2229            "Failed set.");
2230
2231    h->get_item_info(i, &info);
2232    wait_for_flusher_to_settle(h);
2233    h->release(i);
2234
2235    // put some random metadata
2236    ItemMetaData itemMeta;
2237    itemMeta.revSeqno = 10;
2238    itemMeta.cas = info.cas + 1;
2239    itemMeta.exptime = 0;
2240    itemMeta.flags = 0xdeadbeef;
2241
2242    // first check the command fails if no force is set
2243    checkeq(ENGINE_EINVAL,
2244            del_with_meta(h, "key", 3, Vbid(0), &itemMeta, 0, 0 /*options*/),
2245            "Expected EINVAL");
2246
2247    checkeq(ENGINE_SUCCESS,
2248            del_with_meta(h,
2249                          "key",
2250                          3,
2251                          Vbid(0),
2252                          &itemMeta,
2253                          0,
2254                          FORCE_ACCEPT_WITH_META_OPS),
2255            "Expected delete OK");
2256    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
2257    wait_for_flusher_to_settle(h);
2258    wait_for_stat_to_be(h, "curr_items", 0);
2259
2260    // Check all meta data is the same
2261    checkeq(ENGINE_KEY_EEXISTS,
2262            del_with_meta(h,
2263                          "key",
2264                          3,
2265                          Vbid(0),
2266                          &itemMeta,
2267                          0,
2268                          FORCE_ACCEPT_WITH_META_OPS),
2269            "Expected exists");
2270    checkeq(1,
2271            get_int_stat(h, "ep_num_ops_del_meta_res_fail"),
2272            "Expected delete meta conflict resolution failure");
2273
2274    // Check that higher rev seqno but lower cas fails
2275    itemMeta.cas = info.cas;
2276    itemMeta.revSeqno = 11;
2277    checkeq(ENGINE_KEY_EEXISTS,
2278            del_with_meta(h,
2279                          "key",
2280                          3,
2281                          Vbid(0),
2282                          &itemMeta,
2283                          0,
2284                          FORCE_ACCEPT_WITH_META_OPS),
2285            "Expected exists");
2286    checkeq(2,
2287            get_int_stat(h, "ep_num_ops_del_meta_res_fail"),
2288            "Expected delete meta conflict resolution failure");
2289
2290    // Check that a higher cas and lower rev seqno passes
2291    itemMeta.cas = info.cas + 2;
2292    itemMeta.revSeqno = 9;
2293    checkeq(ENGINE_SUCCESS,
2294            del_with_meta(h,
2295                          "key",
2296                          3,
2297                          Vbid(0),
2298                          &itemMeta,
2299                          0,
2300                          FORCE_ACCEPT_WITH_META_OPS),
2301            "Expected delete OK");
2302    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected sucess");
2303
2304    return SUCCESS;
2305}
2306
2307static enum test_result test_getMeta_with_item_eviction(EngineIface* h) {
2308    char const *key = "test_get_meta";
2309    item *i = NULL;
2310    checkeq(ENGINE_SUCCESS,
2311            store(h, NULL, OPERATION_SET, key, "somevalue", &i),
2312            "Failed set.");
2313    wait_for_flusher_to_settle(h);
2314    evict_key(h, key, Vbid(0), "Ejected.");
2315
2316    Item *it = reinterpret_cast<Item*>(i);
2317
2318    cb::EngineErrorMetadataPair errorMetaPair;
2319    check(get_meta(h, key, errorMetaPair), "Expected to get meta");
2320    ItemMetaData metadata(it->getCas(), it->getRevSeqno(),
2321                          it->getFlags(), it->getExptime());
2322    verifyMetaData(metadata, errorMetaPair.second);
2323
2324    h->release(i);
2325    return SUCCESS;
2326}
2327
2328static enum test_result test_set_with_meta_and_check_drift_stats(
2329        EngineIface* h) {
2330    // Activate n vbuckets (vb 0 is already)
2331    const int n_vbuckets = 10;
2332    for (int ii = 1; ii < n_vbuckets; ii++) {
2333        check(set_vbucket_state(h, Vbid(ii), vbucket_state_active),
2334              "Failed to set vbucket state.");
2335    }
2336
2337    // Let's make vbucket n/2 be the one who is ahead, n/3 is behind
2338    const int aheadVb = n_vbuckets/2;
2339    const int behindVb = n_vbuckets/3;
2340    checkne(aheadVb, behindVb, "Cannot have the same VB as ahead/behind");
2341
2342    HLC hlc(0 /*init HLC*/,
2343            HlcCasSeqnoUninitialised,
2344            std::chrono::microseconds(0) /*ahead threshold*/,
2345            std::chrono::microseconds(0) /*behind threshold*/);
2346
2347    // grab the drift behind threshold
2348    uint64_t driftBehindThreshold =
2349            get_ull_stat(h, "ep_hlc_drift_ahead_threshold_us", nullptr);
2350    // Create n keys
2351    const int n_keys = 5;
2352    for (int ii = 0 ; ii < n_vbuckets; ii++) {
2353        for (int k = 0; k < n_keys; k++) {
2354            std::string key = "key_" + std::to_string(k);
2355            ItemMetaData itm_meta;
2356            itm_meta.cas = hlc.nextHLC();
2357            if (ii == aheadVb) {
2358                // Push this guy *far* ahead (1 year)
2359                itm_meta.cas += 3154E10;
2360            } else if(ii == behindVb) {
2361                // just be sure it was already greater then 1 + driftthreshold
2362                checkge(itm_meta.cas, uint64_t(1) + driftBehindThreshold,
2363                        "HLC was already zero");
2364                // set to be way way behind...
2365                itm_meta.cas = 1;
2366            }
2367            checkeq(ENGINE_SUCCESS,
2368                    set_with_meta(h,
2369                                  key.data(),
2370                                  key.size(),
2371                                  NULL,
2372                                  0,
2373                                  Vbid(ii),
2374                                  &itm_meta,
2375                                  0,
2376                                  FORCE_ACCEPT_WITH_META_OPS),
2377                    "Expected item to be stored");
2378            checkeq(cb::mcbp::Status::Success, last_status.load(),
2379                    "Expected success");
2380        }
2381    }
2382
2383    // Bucket stats should report drift
2384    checkge(get_ull_stat(h, "ep_active_hlc_drift"),
2385            uint64_t(0),
2386            "Expected drift above zero");
2387    checkeq(uint64_t(n_keys * n_vbuckets),
2388            get_ull_stat(h, "ep_active_hlc_drift_count"),
2389            "Expected ahead counter to match mutations");
2390
2391    // Victim VBs should have exceptions
2392    {
2393        std::string vbAheadName = "vb_" + std::to_string(aheadVb);
2394        std::string ahead_threshold_exceeded = vbAheadName + ":drift_ahead_threshold_exceeded";
2395        std::string behind_threshold_exceeded = vbAheadName + ":drift_behind_threshold_exceeded";
2396        std::string total_abs_drift = vbAheadName + ":total_abs_drift";
2397        std::string details = "vbucket-details " + std::to_string(aheadVb);
2398        checkeq(uint64_t(n_keys),
2399                get_ull_stat(
2400                        h, ahead_threshold_exceeded.data(), details.data()),
2401                "Expected ahead threshold to match mutations");
2402        checkeq(uint64_t(0),
2403                get_ull_stat(
2404                        h, behind_threshold_exceeded.data(), details.data()),
2405                "Expected no behind exceptions");
2406        checkge(get_ull_stat(h, total_abs_drift.data(), details.data()),
2407                uint64_t(0),
2408                "Expected some drift");
2409    }
2410
2411    {
2412        std::string vbBehindName = "vb_" + std::to_string(behindVb);
2413        std::string ahead_threshold_exceeded = vbBehindName + ":drift_ahead_threshold_exceeded";
2414        std::string behind_threshold_exceeded = vbBehindName + ":drift_behind_threshold_exceeded";
2415        std::string total_abs_drift = vbBehindName + ":total_abs_drift";
2416        std::string details = "vbucket-details " + std::to_string(behindVb);
2417        checkeq(uint64_t(n_keys),
2418                get_ull_stat(
2419                        h, behind_threshold_exceeded.data(), details.data()),
2420                "Expected behind threshold to match mutations");
2421        checkeq(uint64_t(0),
2422                get_ull_stat(
2423                        h, ahead_threshold_exceeded.data(), details.data()),
2424                "Expected no ahead exceptions");
2425        checkge(get_ull_stat(h, total_abs_drift.data(), details.data()),
2426                uint64_t(0),
2427                "Expected some drift");
2428    }
2429
2430
2431    return SUCCESS;
2432}
2433
2434static enum test_result test_del_with_meta_and_check_drift_stats(
2435        EngineIface* h) {
2436    // Activate n vbuckets (vb 0 is already)
2437    const int n_vbuckets = 10;
2438    for (int ii = 1; ii < n_vbuckets; ii++) {
2439        check(set_vbucket_state(h, Vbid(ii), vbucket_state_active),
2440              "Failed to set vbucket state.");
2441    }
2442
2443    // Let's make vbucket n/2 be the one who is ahead, n/3 is behind
2444    const int aheadVb = n_vbuckets/2;
2445    const int behindVb = n_vbuckets/3;
2446    checkne(aheadVb, behindVb, "Cannot have the same VB as ahead/behind");
2447
2448    HLC hlc(0 /*init HLC*/,
2449            HlcCasSeqnoUninitialised,
2450            std::chrono::microseconds(0) /*ahead threshold*/,
2451            std::chrono::microseconds(0) /*behind threshold*/);
2452
2453    // grab the drift behind threshold
2454    uint64_t driftBehindThreshold =
2455            get_ull_stat(h, "ep_hlc_drift_ahead_threshold_us", nullptr);
2456    // Create n keys * n_vbuckets
2457    const int n_keys = 5;
2458    for (int ii = 0 ; ii < n_vbuckets; ii++) {
2459        for (int k = 0; k < n_keys; k++) {
2460            std::string key = "key_" + std::to_string(k);
2461
2462            // In the del_with_meta test we want to pretend a del_wm came from
2463            // the past, so we want to ensure a delete doesn't get rejected
2464            // by LWW conflict resolution, thus write all documents that are
2465            // going to be deleted with set_with_meta, and write them way in the past.
2466            // This will trigger threshold and increment drift stats... so we
2467            // account for these later
2468            ItemMetaData itm_meta;
2469            itm_meta.cas = 1; // set to 1
2470            checkeq(ENGINE_SUCCESS,
2471                    set_with_meta(h,
2472                                  key.data(),
2473                                  key.size(),
2474                                  NULL,
2475                                  0,
2476                                  Vbid(ii),
2477                                  &itm_meta,
2478                                  0,
2479                                  FORCE_ACCEPT_WITH_META_OPS),
2480                    "Expected item to be stored");
2481            checkeq(cb::mcbp::Status::Success, last_status.load(),
2482                    "Expected success");
2483        }
2484    }
2485
2486    checkeq(uint64_t(0),
2487            get_ull_stat(h, "ep_active_ahead_exceptions"),
2488            "Expected ahead counter to match mutations");
2489    checkeq(uint64_t(n_keys * n_vbuckets),
2490            get_ull_stat(h, "ep_active_behind_exceptions"),
2491            "Expected behind counter to match mutations");
2492
2493    // Del_with_meta n_keys to n_vbuckets
2494    for (int ii = 0 ; ii < n_vbuckets; ii++) {
2495        for (int k = 0; k < n_keys; k++) {
2496            std::string key = "key_" + std::to_string(k);
2497            ItemMetaData itm_meta;
2498            itm_meta.cas = hlc.nextHLC();
2499            if (ii == aheadVb) {
2500                // Push this guy *far* ahead (1 year)
2501                itm_meta.cas += 3154E10;
2502            } else if(ii == behindVb) {
2503                // just be sure it was already greater than 1 + driftthreshold
2504                checkge(itm_meta.cas, uint64_t(1) + driftBehindThreshold,
2505                        "HLC was already zero");
2506                // set to be way way behind, but ahead of the documents we have set
2507                itm_meta.cas = 2;
2508            }
2509            checkeq(ENGINE_SUCCESS,
2510                    del_with_meta(h,
2511                                  key.data(),
2512                                  key.size(),
2513                                  Vbid(ii),
2514                                  &itm_meta,
2515                                  1,
2516                                  FORCE_ACCEPT_WITH_META_OPS),
2517                    "Expected delete OK");
2518            checkeq(cb::mcbp::Status::Success, last_status.load(),
2519                    "Expected success");
2520        }
2521    }
2522
2523    // Bucket stats should report drift
2524    checkge(get_ull_stat(h, "ep_active_hlc_drift"),
2525            uint64_t(0),
2526            "Expected drift above zero");
2527    checkeq(2 * uint64_t(n_keys * n_vbuckets),
2528            get_ull_stat(h, "ep_active_hlc_drift_count"),
2529            "Expected ahead counter to match mutations");
2530
2531    // and should report total exception of all VBs
2532    checkeq(uint64_t(n_keys),
2533            get_ull_stat(h, "ep_active_ahead_exceptions"),
2534            "Expected ahead counter to match mutations");
2535    checkeq(uint64_t(n_keys + (n_keys * n_vbuckets)),
2536            get_ull_stat(h, "ep_active_behind_exceptions"),
2537            "Expected behind counter to match mutations");
2538
2539    // Victim VBs should have exceptions
2540    {
2541        std::string vbAheadName = "vb_" + std::to_string(aheadVb);
2542        std::string ahead_threshold_exceeded = vbAheadName + ":drift_ahead_threshold_exceeded";
2543        std::string behind_threshold_exceeded = vbAheadName + ":drift_behind_threshold_exceeded";
2544        std::string total_abs_drift = vbAheadName + ":total_abs_drift";
2545        std::string details = "vbucket-details " + std::to_string(aheadVb);
2546
2547        checkeq(uint64_t(n_keys),
2548                get_ull_stat(
2549                        h, ahead_threshold_exceeded.data(), details.data()),
2550                "Expected ahead threshold to match mutations");
2551        checkge(get_ull_stat(h, total_abs_drift.data(), details.data()),
2552                uint64_t(0),
2553                "Expected some drift");
2554    }
2555
2556    {
2557        std::string vbBehindName = "vb_" + std::to_string(behindVb);
2558        std::string ahead_threshold_exceeded = vbBehindName + ":drift_ahead_threshold_exceeded";
2559        std::string behind_threshold_exceeded = vbBehindName + ":drift_behind_threshold_exceeded";
2560        std::string total_abs_drift = vbBehindName + ":total_abs_drift";
2561        std::string details = "vbucket-details " + std::to_string(behindVb);
2562
2563        // *2 behind due to the initial set_with_meta
2564        checkeq(uint64_t(n_keys * 2),
2565                get_ull_stat(
2566                        h, behind_threshold_exceeded.data(), details.data()),
2567                "Expected behind threshold to match mutations");
2568        checkeq(uint64_t(0),
2569                get_ull_stat(
2570                        h, ahead_threshold_exceeded.data(), details.data()),
2571                "Expected no ahead exceptions");
2572        checkge(get_ull_stat(h, total_abs_drift.data(), details.data()),
2573                uint64_t(0),
2574                "Expected some drift");
2575    }
2576
2577
2578    return SUCCESS;
2579}
2580
2581static enum test_result test_setting_drift_threshold(EngineIface* h) {
2582    std::vector<std::tuple<std::string, std::string, std::string> > configData =
2583        {std::make_tuple("ep_hlc_drift_ahead_threshold_us",
2584                         "hlc_drift_ahead_threshold_us",
2585                         "vb_0:drift_ahead_threshold"),
2586         std::make_tuple("ep_hlc_drift_behind_threshold_us",
2587                         "hlc_drift_behind_threshold_us",
2588                         "vb_0:drift_behind_threshold")};
2589
2590    std::vector<std::pair<std::string, std::chrono::microseconds> > values = {
2591        {"0", std::chrono::microseconds(0)},
2592        {"1", std::chrono::microseconds(1)},
2593        {"-1", std::chrono::microseconds(-1)},
2594        {"-0", std::chrono::microseconds(0)},
2595        {"18446744073709551615",
2596         std::chrono::microseconds(18446744073709551615ull)}};
2597
2598    for (auto data : values) {
2599        for (auto conf : configData) {
2600            check(set_param(h,
2601                            cb::mcbp::request::SetParamPayload::Type::Vbucket,
2602                            std::get<1>(conf).c_str(),
2603                            data.first.data()),
2604                  "Expected set_param success");
2605
2606            checkeq(int64_t(data.second.count()),
2607                    int64_t(get_ull_stat(
2608                            h, std::get<0>(conf).c_str(), nullptr)),
2609                    "Expected the stat to change to the new value");
2610
2611            // The VB stat values are in nanoseconds
2612            checkeq(int64_t(std::chrono::nanoseconds(data.second).count()),
2613                    int64_t(get_ull_stat(
2614                            h, std::get<2>(conf).c_str(), "vbucket-details 0")),
2615                    "Expected the VB stats to change to the new value");
2616        }
2617    }
2618    return SUCCESS;
2619}
2620
2621/*
2622 * Perform set_with_meta and check CAS regeneration is ok.
2623 */
2624static enum test_result test_cas_regeneration(EngineIface* h) {
2625    // First store a key from the past (small CAS).
2626    ItemMetaData itemMeta;
2627    itemMeta.revSeqno = 10;
2628    itemMeta.cas = 0x1;
2629    itemMeta.exptime = 0;
2630    itemMeta.flags = 0xdeadbeef;
2631    int force = 0;
2632
2633    if (strstr(testHarness->get_current_testcase()->cfg,
2634               "conflict_resolution_type=lww") != nullptr) {
2635        force = FORCE_ACCEPT_WITH_META_OPS;
2636    }
2637
2638    // Set the key with a low CAS value
2639    checkeq(ENGINE_SUCCESS,
2640            set_with_meta(
2641                    h, "key", 3, nullptr, 0, Vbid(0), &itemMeta, 0, force),
2642            "Expected item to be stored");
2643    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
2644
2645    cb::EngineErrorMetadataPair errorMetaPair;
2646
2647    check(get_meta(h, "key", errorMetaPair), "Failed to get_meta");
2648
2649    // CAS must be what we set.
2650    checkeq(itemMeta.cas,
2651            errorMetaPair.second.cas,
2652            "CAS is not the value we stored");
2653
2654    itemMeta.cas++;
2655
2656    // Check that the code requires skip
2657    checkeq(ENGINE_EINVAL,
2658            set_with_meta(h,
2659                          "key",
2660                          3,
2661                          nullptr,
2662                          0,
2663                          Vbid(0),
2664                          &itemMeta,
2665                          0,
2666                          REGENERATE_CAS /*but no skip*/),
2667            "Expected EINVAL");
2668
2669    checkeq(ENGINE_SUCCESS,
2670            set_with_meta(h,
2671                          "key",
2672                          3,
2673                          nullptr,
2674                          0,
2675                          Vbid(0),
2676                          &itemMeta,
2677                          0,
2678                          REGENERATE_CAS | SKIP_CONFLICT_RESOLUTION_FLAG),
2679            "Expected item to be stored");
2680
2681    checkeq(cb::mcbp::Status::Success, last_status.load(),
2682            "Expected success");
2683
2684    check(get_meta(h, "key", errorMetaPair), "Failed to get_meta");
2685
2686    uint64_t cas = errorMetaPair.second.cas;
2687    // Check item has a new CAS
2688    checkne(itemMeta.cas, cas, "CAS was not regenerated");
2689
2690    itemMeta.cas++;
2691    // All flags set should still regen the cas (lww and seqno)
2692    checkeq(ENGINE_SUCCESS,
2693            set_with_meta(
2694                    h,
2695                    "key",
2696                    3,
2697                    nullptr,
2698                    0,
2699                    Vbid(0),
2700                    &itemMeta,
2701                    0,
2702                    REGENERATE_CAS | SKIP_CONFLICT_RESOLUTION_FLAG | force),
2703            "Expected item to be stored");
2704
2705    checkeq(cb::mcbp::Status::Success, last_status.load(),
2706            "Expected success");
2707
2708    check(get_meta(h, "key", errorMetaPair), "Failed to get_meta");
2709    // Check item has a new CAS
2710    checkne(itemMeta.cas, errorMetaPair.second.cas, "CAS was not regenerated");
2711    checkne(cas, errorMetaPair.second.cas, "CAS was not regenerated");
2712    return SUCCESS;
2713}
2714
2715/*
2716 * Perform del_with_meta and check CAS regeneration is ok.
2717 */
2718static enum test_result test_cas_regeneration_del_with_meta(EngineIface* h) {
2719    const std::string key("key");
2720    // First store a key from the past (small CAS).
2721    ItemMetaData itemMeta;
2722    itemMeta.revSeqno = 10;
2723    itemMeta.cas = 0x1;
2724    itemMeta.exptime = 0;
2725    itemMeta.flags = 0xdeadbeef;
2726    int force = 0;
2727
2728    if (strstr(testHarness->get_current_testcase()->cfg,
2729               "conflict_resolution_type=lww") != nullptr) {
2730        force = FORCE_ACCEPT_WITH_META_OPS;
2731    }
2732
2733    // Set the key with a low CAS value
2734    checkeq(ENGINE_SUCCESS,
2735            set_with_meta(h,
2736                          key.c_str(),
2737                          key.length(),
2738                          nullptr,
2739                          0,
2740                          Vbid(0),
2741                          &itemMeta,
2742                          0,
2743                          force),
2744            "Expected item to be stored");
2745    checkeq(cb::mcbp::Status::Success,
2746            last_status.load(),
2747            "Expected success");
2748
2749    cb::EngineErrorMetadataPair errorMetaPair;
2750    check(get_meta(h, key.c_str(), errorMetaPair), "Failed to get_meta");
2751    // CAS must be what we set.
2752    checkeq(itemMeta.cas,
2753            errorMetaPair.second.cas,
2754            "CAS is not the value we stored");
2755
2756    itemMeta.cas++;
2757
2758    // Check that the code requires skip
2759    checkeq(ENGINE_EINVAL,
2760            del_with_meta(h,
2761                          key.c_str(),
2762                          key.length(),
2763                          Vbid(0),
2764                          &itemMeta,
2765                          0,
2766                          REGENERATE_CAS /*but no skip*/),
2767            "Expected EINVAL");
2768
2769    checkeq(ENGINE_SUCCESS,
2770            del_with_meta(h,
2771                          key.c_str(),
2772                          key.length(),
2773                          Vbid(0),
2774                          &itemMeta,
2775                          0,
2776                          REGENERATE_CAS | SKIP_CONFLICT_RESOLUTION_FLAG),
2777            "Expected delete OK");
2778    checkeq(cb::mcbp::Status::Success,
2779            last_status.load(),
2780            "Expected success");
2781
2782    check(get_meta(h, key.c_str(), errorMetaPair), "Failed to get_meta");
2783    uint64_t cas = errorMetaPair.second.cas;
2784    // Check item has a new CAS
2785    checkne(itemMeta.cas, cas, "CAS was not regenerated");
2786
2787    itemMeta.cas++;
2788    // All flags set should still regen the cas (lww and seqno)
2789    checkeq(ENGINE_SUCCESS,
2790            del_with_meta(
2791                    h,
2792                    key.c_str(),
2793                    key.length(),
2794                    Vbid(0),
2795                    &itemMeta,
2796                    0,
2797                    REGENERATE_CAS | SKIP_CONFLICT_RESOLUTION_FLAG | force),
2798            "Expected delete OK");
2799    checkeq(cb::mcbp::Status::Success,
2800            last_status.load(),
2801            "Expected success");
2802
2803    check(get_meta(h, key.c_str(), errorMetaPair), "Failed to get_meta");
2804    // Check item has a new CAS
2805    checkne(itemMeta.cas, errorMetaPair.second.cas, "CAS was not regenerated");
2806    checkne(cas, errorMetaPair.second.cas, "CAS was not regenerated");
2807
2808    return SUCCESS;
2809}
2810
2811/*
2812 * Test that we can send options and nmeta
2813 * The nmeta is just going to be ignored though, but should not fail
2814 */
2815static enum test_result test_cas_options_and_nmeta(EngineIface* h) {
2816    ItemMetaData itemMeta;
2817    itemMeta.revSeqno = 10;
2818    itemMeta.cas = 0x1;
2819    itemMeta.exptime = 0;
2820    itemMeta.flags = 0xdeadbeef;
2821
2822    // Watson (4.6) accepts valid encodings, but ignores them
2823    std::vector<char> junkMeta = {-2,-1,2,3};
2824
2825    int force = 0;
2826
2827    if (strstr(testHarness->get_current_testcase()->cfg,
2828               "conflict_resolution_type=lww") != nullptr) {
2829        force = FORCE_ACCEPT_WITH_META_OPS;
2830    }
2831
2832    // Set the key and junk nmeta
2833    checkeq(ENGINE_EINVAL,
2834            set_with_meta(h,
2835                          "key",
2836                          3,
2837                          NULL,
2838                          0,
2839                          Vbid(0),
2840                          &itemMeta,
2841                          0,
2842                          force,
2843                          PROTOCOL_BINARY_RAW_BYTES,
2844                          nullptr,
2845                          junkMeta),
2846            "Expected EINVAL");
2847
2848    // Set the key and junk nmeta that's quite large
2849    junkMeta.resize(std::numeric_limits<uint16_t>::max());
2850    checkeq(ENGINE_EINVAL,
2851            set_with_meta(h,
2852                          "key",
2853                          3,
2854                          NULL,
2855                          0,
2856                          Vbid(0),
2857                          &itemMeta,
2858                          0,
2859                          force,
2860                          PROTOCOL_BINARY_RAW_BYTES,
2861                          nullptr,
2862                          junkMeta),
2863            "Expected EINVAL");
2864
2865    // Test that valid meta can be sent. It should be ignored and success
2866    // returned
2867    // Encodings which should not fail, see ext_meta_parser.cc
2868#pragma pack(1)
2869    struct adjusted_time_metadata {
2870        uint8_t type;
2871        uint16_t length;
2872        int64_t value;
2873    };
2874    struct conf_res_metadata {
2875        uint8_t type;
2876        uint16_t length;
2877        uint8_t value;
2878    };
2879    struct with_cas_metadata1 {
2880        uint8_t version;
2881        adjusted_time_metadata adjusted_time;
2882    };
2883    struct with_cas_metadata2 {
2884        uint8_t version;
2885        conf_res_metadata conf_res;
2886    };
2887    struct with_cas_metadata3 {
2888        uint8_t version;
2889        conf_res_metadata conf_res;
2890        adjusted_time_metadata adjusted_time;
2891    };
2892    struct with_cas_metadata4 {
2893        uint8_t version;
2894        adjusted_time_metadata adjusted_time;
2895        conf_res_metadata conf_res;
2896    };
2897#pragma pack()
2898
2899    {
2900        with_cas_metadata1 validMetaData = {META_EXT_VERSION_ONE,
2901                                            {CMD_META_ADJUSTED_TIME,
2902                                             htons(sizeof(int64_t)), -1}};
2903        std::vector<char> validMetaVector(reinterpret_cast<char*>(&validMetaData),
2904                                          reinterpret_cast<char*>(&validMetaData) +
2905                                          sizeof(validMetaData));
2906
2907        // Set the key with a low CAS value and real nmeta
2908        checkeq(ENGINE_SUCCESS,
2909                set_with_meta(h,
2910                              "key1",
2911                              4,
2912                              nullptr,
2913                              0,
2914                              Vbid(0),
2915                              &itemMeta,
2916                              0,
2917                              force,
2918                              PROTOCOL_BINARY_RAW_BYTES,
2919                              nullptr,
2920                              validMetaVector),
2921                "Expected item to be stored");
2922        checkeq(cb::mcbp::Status::Success, last_status.load(),
2923                "Expected success");
2924
2925        itemMeta.cas++;
2926        checkeq(ENGINE_SUCCESS,
2927                del_with_meta(h,
2928                              "key1",
2929                              4,
2930                              Vbid(0),
2931                              &itemMeta,
2932                              0,
2933                              force,
2934                              nullptr,
2935                              validMetaVector),
2936                "Expected delete success");
2937        checkeq(cb::mcbp::Status::Success, last_status.load(),
2938                "Expected success");
2939    }
2940
2941    {
2942        with_cas_metadata2 validMetaData = {META_EXT_VERSION_ONE,
2943                                            {CMD_META_CONFLICT_RES_MODE,
2944                                             htons(sizeof(uint8_t)), 0xff}};
2945        std::vector<char> validMetaVector(reinterpret_cast<char*>(&validMetaData),
2946                                          reinterpret_cast<char*>(&validMetaData) +
2947                                          sizeof(validMetaData));
2948
2949        // Set the key with a low CAS value and real nmeta
2950        checkeq(ENGINE_SUCCESS,
2951                set_with_meta(h,
2952                              "key2",
2953                              4,
2954                              nullptr,
2955                              0,
2956                              Vbid(0),
2957                              &itemMeta,
2958                              0,
2959                              force,
2960                              PROTOCOL_BINARY_RAW_BYTES,
2961                              nullptr,
2962                              validMetaVector),
2963                "Expected item to be stored");
2964        checkeq(cb::mcbp::Status::Success, last_status.load(),
2965                "Expected success");
2966
2967        itemMeta.cas++;
2968        checkeq(ENGINE_SUCCESS,
2969                del_with_meta(h,
2970                              "key2",
2971                              4,
2972                              Vbid(0),
2973                              &itemMeta,
2974                              0,
2975                              force,
2976                              nullptr,
2977                              validMetaVector),
2978                "Expected delete success");
2979        checkeq(cb::mcbp::Status::Success, last_status.load(),
2980                "Expected success");
2981    }
2982
2983    {
2984        with_cas_metadata3 validMetaData = {META_EXT_VERSION_ONE,
2985                                            {CMD_META_CONFLICT_RES_MODE,
2986                                             htons(sizeof(uint8_t)), 0xff},
2987                                            {CMD_META_ADJUSTED_TIME,
2988                                             htons(sizeof(int64_t)), -1}};
2989        std::vector<char> validMetaVector(reinterpret_cast<char*>(&validMetaData),
2990                                          reinterpret_cast<char*>(&validMetaData) +
2991                                          sizeof(validMetaData));
2992
2993        // Set the key with a low CAS value and real nmeta
2994        checkeq(ENGINE_SUCCESS,
2995                set_with_meta(h,
2996                              "key3",
2997                              4,
2998                              nullptr,
2999                              0,
3000                              Vbid(0),
3001                              &itemMeta,
3002                              0,
3003                              force,
3004                              PROTOCOL_BINARY_RAW_BYTES,
3005                              nullptr,
3006                              validMetaVector),
3007                "Expected item to be stored");
3008        checkeq(cb::mcbp::Status::Success, last_status.load(),
3009                "Expected success");
3010
3011        itemMeta.cas++;
3012        checkeq(ENGINE_SUCCESS,
3013                del_with_meta(h,
3014                              "key3",
3015                              4,
3016                              Vbid(0),
3017                              &itemMeta,
3018                              0,
3019                              force,
3020                              nullptr,
3021                              validMetaVector),
3022                "Expected delete OK");
3023        checkeq(cb::mcbp::Status::Success, last_status.load(),
3024                "Expected success");
3025    }
3026
3027    {
3028        with_cas_metadata4 validMetaData = {META_EXT_VERSION_ONE,
3029                                            {CMD_META_ADJUSTED_TIME,
3030                                             htons(sizeof(int64_t)), -1},
3031                                            {CMD_META_CONFLICT_RES_MODE,
3032                                             htons(sizeof(uint8_t)), 0xff}};
3033        std::vector<char> validMetaVector(reinterpret_cast<char*>(&validMetaData),
3034                                          reinterpret_cast<char*>(&validMetaData) +
3035                                          sizeof(validMetaData));
3036
3037        // Set the key with a low CAS value and real nmeta
3038        checkeq(ENGINE_SUCCESS,
3039                set_with_meta(h,
3040                              "key4",
3041                              4,
3042                              NULL,
3043                              0,
3044                              Vbid(0),
3045                              &itemMeta,
3046                              0,
3047                              force,
3048                              PROTOCOL_BINARY_RAW_BYTES,
3049                              nullptr,
3050                              validMetaVector),
3051                "Expected item to be stored");
3052        checkeq(cb::mcbp::Status::Success, last_status.load(),
3053                "Expected success");
3054
3055        itemMeta.cas++;
3056        checkeq(ENGINE_SUCCESS,
3057                del_with_meta(h,
3058                              "key4",
3059                              4,
3060                              Vbid(0),
3061                              &itemMeta,
3062                              0,
3063                              force,
3064                              nullptr,
3065                              validMetaVector),
3066                "Expected delete ok");
3067        checkeq(cb::mcbp::Status::Success, last_status.load(),
3068                "Expected success");
3069    }
3070
3071    return SUCCESS;
3072}
3073
3074// A delete_with_meta with a large seqno (upper 16-bits dirty) should trigger
3075// a conflict if evicted and a second identical delete_with_meta occurs
3076static enum test_result test_MB29119(EngineIface* h) {
3077    const char* key1 = "delete_with_meta_key1";
3078    const size_t keylen = strlen(key1);
3079    RawItemMetaData itemMeta;
3080
3081    // Overflow seqno from 48-bits
3082    itemMeta.revSeqno = 0x0080a80000000001;
3083    itemMeta.cas = 0xdeadbeef;
3084    itemMeta.exptime = 0;
3085    itemMeta.flags = 0xdeadbeef;
3086
3087    const void* cookie = testHarness->create_cookie();
3088
3089    // delete an item with meta data
3090    checkeq(ENGINE_SUCCESS,
3091            del_with_meta(h,
3092                          key1,
3093                          keylen,
3094                          Vbid(0),
3095                          &itemMeta,
3096                          0 /*cas*/,
3097                          0 /*options*/,
3098                          cookie),
3099            "Expected delete OK");
3100
3101    checkeq(cb::mcbp::Status::Success,
3102            last_status.load(),
3103            "Expected success");
3104
3105    wait_for_flusher_to_settle(h);
3106    // evict the key
3107    evict_key(h, key1);
3108
3109    // Same key
3110    checkeq(ENGINE_KEY_EEXISTS,
3111            del_with_meta(h,
3112                          key1,
3113                          keylen,
3114                          Vbid(0),
3115                          &itemMeta,
3116                          0 /*cas*/,
3117                          0 /*options*/,
3118                          cookie),
3119            "Expected EEXISTS");
3120
3121    testHarness->destroy_cookie(cookie);
3122    return SUCCESS;
3123}
3124
3125/*
3126 * Test that we can send option of IS_EXPIRATION
3127 */
3128static enum test_result test_expiration_options(EngineIface* h) {
3129    const char* key = "delete_with_meta_key";
3130    const size_t keylen = strlen(key);
3131    ItemMetaData itemMeta;
3132    itemMeta.revSeqno = 10;
3133    itemMeta.cas = 0x1;
3134    itemMeta.exptime = 10;
3135    itemMeta.flags = 0xdeadbeef;
3136
3137    // store an item
3138    checkeq(ENGINE_SUCCESS,
3139            store(h,
3140                  NULL,
3141                  OPERATION_SET,
3142                  key,
3143                  "somevalue",
3144                  nullptr,
3145                  0,
3146                  Vbid(0),
3147                  100),
3148            "Failed set.");
3149    wait_for_flusher_to_settle(h);
3150
3151    // delete an item with meta data indicating expiration
3152    checkeq(ENGINE_SUCCESS,
3153            del_with_meta(h, key, keylen, Vbid(0), &itemMeta, 0, IS_EXPIRATION),
3154            "Expected delete success");
3155    checkeq(cb::mcbp::Status::Success, last_status.load(), "Expected success");
3156
3157    return SUCCESS;
3158}
3159
3160// Test manifest //////////////////////////////////////////////////////////////
3161
3162const char *default_dbname = "./ep_testsuite_xdcr";
3163
3164BaseTestCase testsuite_testcases[] = {
3165
3166        // XDCR unit tests
3167        TestCase("get meta",
3168                 test_get_meta,
3169                 test_setup,
3170                 teardown,
3171                 NULL,
3172                 prepare,
3173                 cleanup),
3174        TestCase("get meta with extras",
3175                 test_get_meta_with_extras,
3176                 test_setup,
3177                 teardown,
3178                 NULL,
3179                 prepare,
3180                 cleanup),
3181        TestCase("get meta deleted",
3182                 test_get_meta_deleted,
3183                 test_setup,
3184                 teardown,
3185                 NULL,
3186                 prepare,
3187                 cleanup),
3188        TestCase("get meta nonexistent",
3189                 test_get_meta_nonexistent,
3190                 test_setup,
3191                 teardown,
3192                 NULL,
3193                 prepare,
3194                 cleanup),
3195        TestCase("get meta followed by get",
3196                 test_get_meta_with_get,
3197                 test_setup,
3198                 teardown,
3199                 NULL,
3200                 prepare,
3201                 cleanup),
3202        TestCase("get meta followed by set",
3203                 test_get_meta_with_set,
3204                 test_setup,
3205                 teardown,
3206                 nullptr,
3207                 /* TODO RDB: curr_items not correct under Rocks */
3208                 prepare_skip_broken_under_rocks,
3209                 cleanup),
3210        TestCase("get meta followed by delete",
3211                 test_get_meta_with_delete,
3212                 test_setup,
3213                 teardown,
3214                 NULL,
3215                 prepare,
3216                 cleanup),
3217        TestCase("get meta with xattr",
3218                 test_get_meta_with_xattr,
3219                 test_setup,
3220                 teardown,
3221                 NULL,
3222                 prepare,
3223                 cleanup),
3224        TestCase("add with meta",
3225                 test_add_with_meta,
3226                 test_setup,
3227                 teardown,
3228                 NULL,
3229                 prepare,
3230                 cleanup),
3231        TestCase("delete with meta",
3232                 test_delete_with_meta,
3233                 test_setup,
3234                 teardown,
3235                 NULL,
3236                 prepare,
3237                 cleanup),
3238        TestCase("delete with meta deleted",
3239                 test_delete_with_meta_deleted,
3240                 test_setup,
3241                 teardown,
3242                 nullptr,
3243                 /* TODO RDB: curr_items not correct under Rocks */
3244                 prepare_skip_broken_under_rocks,
3245                 cleanup),
3246        TestCase("delete with meta nonexistent",
3247                 test_delete_with_meta_nonexistent,
3248                 test_setup,
3249                 teardown,
3250                 nullptr,
3251                 prepare,
3252                 cleanup),
3253        TestCase("delete with meta nonexistent no temp",
3254                 test_delete_with_meta_nonexistent_no_temp,
3255                 test_setup,
3256                 teardown,
3257                 nullptr,
3258                 prepare,
3259                 cleanup),
3260        TestCase("delete_with_meta race with concurrent delete",
3261                 test_delete_with_meta_race_with_delete,
3262                 test_setup,
3263                 teardown,
3264                 NULL,
3265                 prepare,
3266                 cleanup),
3267        TestCase("delete_with_meta race with concurrent set",
3268                 test_delete_with_meta_race_with_set,
3269                 test_setup,
3270                 teardown,
3271                 NULL,
3272                 prepare,
3273                 cleanup),
3274        TestCase("set with meta",
3275                 test_set_with_meta,
3276                 test_setup,
3277                 teardown,
3278                 NULL,
3279                 /* TODO RDB: curr_items not correct under Rocks when full
3280                  * eviction */
3281                 prepare_ep_bucket_skip_broken_under_rocks_full_eviction,
3282                 cleanup),
3283        TestCase("set with meta by force",
3284                 test_set_with_meta_by_force,
3285                 test_setup,
3286                 teardown,
3287                 NULL,
3288                 prepare,
3289                 cleanup),
3290        TestCase("set with meta deleted",
3291                 test_set_with_meta_deleted,
3292                 test_setup,
3293                 teardown,
3294                 nullptr,
3295                 /* TODO RDB: curr_items not correct under Rocks when full
3296                  * eviction */
3297                 prepare_skip_broken_under_rocks_full_eviction,
3298                 cleanup),
3299        TestCase("set with meta nonexistent",
3300                 test_set_with_meta_nonexistent,
3301                 test_setup,
3302                 teardown,
3303                 NULL,
3304                 prepare,
3305                 cleanup),
3306        TestCase("set_with_meta race with concurrent set",
3307                 test_set_with_meta_race_with_set,
3308                 test_setup,
3309                 teardown,
3310                 NULL,
3311                 prepare,
3312                 cleanup),
3313        TestCase("set_with_meta race with concurrent delete",
3314                 test_set_with_meta_race_with_delete,
3315                 test_setup,
3316                 teardown,
3317                 NULL,
3318                 prepare,
3319                 cleanup),
3320        TestCase("test set_with_meta exp persisted",
3321                 test_exp_persisted_set_del,
3322                 test_setup,
3323                 teardown,
3324                 "exp_pager_stime=3",
3325                 /* TODO RDB: curr_items not correct under Rocks in full
3326                  * eviction */
3327                 /* Requires persistence */
3328                 prepare_ep_bucket_skip_broken_under_rocks_full_eviction,
3329                 cleanup),
3330        TestCase("test del meta conflict resolution",
3331                 test_del_meta_conflict_resolution,
3332                 test_setup,
3333                 teardown,
3334                 nullptr,
3335                 /* TODO RDB: curr_items not correct under Rocks in full
3336                  * eviction */
3337                 prepare_skip_broken_under_rocks_full_eviction,
3338                 cleanup),
3339        TestCase("test add meta conflict resolution",
3340                 test_add_meta_conflict_resolution,
3341                 test_setup,
3342                 teardown,
3343                 NULL,
3344                 /* TODO RDB: curr_items not correct under Rocks in full
3345                  * eviction */
3346                 prepare_skip_broken_under_rocks_full_eviction,
3347                 cleanup),
3348        TestCase("test set meta conflict resolution",
3349                 test_set_meta_conflict_resolution,
3350                 test_setup,
3351                 teardown,
3352                 NULL,
3353                 prepare,
3354                 cleanup),
3355        TestCase("test del meta lww conflict resolution",
3356                 test_del_meta_lww_conflict_resolution,
3357                 test_setup,
3358                 teardown,
3359                 "conflict_resolution_type=lww",
3360                 /* TODO RDB: curr_items not correct under Rocks in full
3361                  * eviction */
3362                 prepare_skip_broken_under_rocks_full_eviction,
3363                 cleanup),
3364        TestCase("test set meta lww conflict resolution",
3365                 test_set_meta_lww_conflict_resolution,
3366                 test_setup,
3367                 teardown,
3368                 "conflict_resolution_type=lww",
3369                 prepare,
3370                 cleanup),
3371        TestCase("set with meta xattr",
3372                 test_set_with_meta_xattr,
3373                 test_setup,
3374                 teardown,
3375                 nullptr,
3376                 prepare,
3377                 cleanup),
3378        TestCase("set with meta lww xattr",
3379                 test_set_with_meta_xattr,
3380                 test_setup,
3381                 teardown,
3382                 "conflict_resolution_type=lww",
3383                 prepare,
3384                 cleanup),
3385        TestCase("delete with meta xattr",
3386                 test_delete_with_meta_xattr,
3387                 test_setup,
3388                 teardown,
3389                 nullptr,
3390                 prepare,
3391                 cleanup),
3392        TestCase("delete with meta lww xattr",
3393                 test_delete_with_meta_xattr,
3394                 test_setup,
3395                 teardown,
3396                 "conflict_resolution_type=lww",
3397                 prepare,
3398                 cleanup),
3399        TestCase("temp item deletion",
3400                 test_temp_item_deletion,
3401                 test_setup,
3402                 teardown,
3403                 "exp_pager_stime=1",
3404                 /* TODO RDB: curr_items not correct under Rocks full eviction*/
3405                 /* related to temp items in hash table */
3406                 prepare_ep_bucket_skip_broken_under_rocks_full_eviction,
3407                 cleanup),
3408        TestCase("test get_meta with item_eviction",
3409                 test_getMeta_with_item_eviction,
3410                 test_setup,
3411                 teardown,
3412                 "item_eviction_policy=full_eviction",
3413                 prepare_full_eviction,
3414                 cleanup),
3415
3416        TestCase("test set_with_meta and drift stats",
3417                 test_set_with_meta_and_check_drift_stats,
3418                 test_setup,
3419                 teardown,
3420                 "hlc_drift_ahead_threshold_us=5000000;"
3421                 "hlc_drift_behind_threshold_us=0;conflict_resolution_type=lww;"
3422                 "max_vbuckets=10;max_num_shards=4",
3423                 prepare,
3424                 cleanup),
3425        TestCase("test del_with_meta and drift stats",
3426                 test_del_with_meta_and_check_drift_stats,
3427                 test_setup,
3428                 teardown,
3429                 "hlc_drift_ahead_threshold_us=0;hlc_drift_behind_threshold_us="
3430                 "5000000;conflict_resolution_type=lww;max_vbuckets=10;max_num_"
3431                 "shards=4",
3432                 prepare,
3433                 cleanup),
3434        TestCase("test setting drift threshold",
3435                 test_setting_drift_threshold,
3436                 test_setup,
3437                 teardown,
3438                 nullptr,
3439                 prepare,
3440                 cleanup),
3441        TestCase("test CAS regeneration lww",
3442                 test_cas_regeneration,
3443                 test_setup,
3444                 teardown,
3445                 "conflict_resolution_type=lww",
3446                 prepare,
3447                 cleanup),
3448        TestCase("test CAS regeneration seqno",
3449                 test_cas_regeneration,
3450                 test_setup,
3451                 teardown,
3452                 "conflict_resolution_type=seqno",
3453                 prepare,
3454                 cleanup),
3455        TestCase("test CAS regeneration seqno del_with_meta lww",
3456                 test_cas_regeneration_del_with_meta,
3457                 test_setup,
3458                 teardown,
3459                 "conflict_resolution_type=lww",
3460                 prepare,
3461                 cleanup),
3462        TestCase("test CAS regeneration seqno del_with_meta seqno",
3463                 test_cas_regeneration_del_with_meta,
3464                 test_setup,
3465                 teardown,
3466                 "conflict_resolution_type=seqno",
3467                 prepare,
3468                 cleanup),
3469        TestCase("test CAS options and nmeta (lww)",
3470                 test_cas_options_and_nmeta,
3471                 test_setup,
3472                 teardown,
3473                 "conflict_resolution_type=lww",
3474                 prepare,
3475                 cleanup),
3476        TestCase("test CAS options and nmeta (seqno)",
3477                 test_cas_options_and_nmeta,
3478                 test_setup,
3479                 teardown,
3480                 "conflict_resolution_type=seqno",
3481                 prepare,
3482                 cleanup),
3483        TestCase("getMetaData mb23905",
3484                 test_get_meta_mb23905,
3485                 test_setup,
3486                 teardown,
3487                 nullptr,
3488                 prepare_ep_bucket,
3489                 cleanup),
3490
3491        TestCase("MB29119",
3492                 test_MB29119,
3493                 test_setup,
3494                 teardown,
3495                 nullptr,
3496                 prepare_full_eviction,
3497                 cleanup),
3498        TestCase("delete with meta with expiration option",
3499                 test_expiration_options,
3500                 test_setup,
3501                 teardown,
3502                 NULL,
3503                 prepare,
3504                 cleanup),
3505
3506        TestCase(NULL, NULL, NULL, NULL, NULL, prepare, cleanup)};
3507