1/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil
2 * -*- */
3/*
4 *     Copyright 2017 Couchbase, Inc
5 *
6 *   Licensed under the Apache License, Version 2.0 (the "License");
7 *   you may not use this file except in compliance with the License.
8 *   You may obtain a copy of the License at
9 *
10 *       http://www.apache.org/licenses/LICENSE-2.0
11 *
12 *   Unless required by applicable law or agreed to in writing, software
13 *   distributed under the License is distributed on an "AS IS" BASIS,
14 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 *   See the License for the specific language governing permissions and
16 *   limitations under the License.
17 */
18
19#include "bgfetcher.h"
20#include "ep_time.h"
21#include "evp_store_single_threaded_test.h"
22#include "tests/mock/mock_global_task.h"
23#include "tests/module_tests/test_helpers.h"
24#include "utilities/protocol2text.h"
25
26#include <string_utilities.h>
27#include <xattr/blob.h>
28#include <xattr/utils.h>
29
30class WithMetaTest : public SingleThreadedEPBucketTest {
31public:
32    void SetUp() override {
33        SingleThreadedEPBucketTest::SetUp();
34        store->setVBucketState(vbid, vbucket_state_active, false);
35        expiry = ep_real_time() + 31557600; // +1 year in seconds
36    }
37
38    void enableLww() {
39        if (!config_string.empty()) {
40            config_string += ";";
41        }
42        config_string += "conflict_resolution_type=lww";
43    }
44
45    /**
46     * Build a *_with_meta packet, defaulting a number of arguments (keeping
47     * some of the test bodies smaller)
48     */
49    std::vector<char> buildWithMeta(protocol_binary_command op,
50                                    ItemMetaData itemMeta,
51                                    const std::string& key,
52                                    const std::string& value) const {
53        return buildWithMetaPacket(op,
54                                   0 /*datatype*/,
55                                   vbid,
56                                   0 /*opaque*/,
57                                   0 /*cas*/,
58                                   itemMeta,
59                                   key,
60                                   value,
61                                   {},
62                                   0);
63    }
64
65    /**
66     * Given a buffer of data representing a with_meta packet, update the meta
67     * Allows test to avoid lots of allocation/copying when creating inputs.
68     */
69    static void updateMeta(std::vector<char>& wm,
70                           uint64_t cas,
71                           uint64_t revSeq,
72                           uint32_t flags,
73                           uint32_t exp) {
74        auto packet = reinterpret_cast<protocol_binary_request_set_with_meta*>(
75                wm.data());
76        packet->message.body.cas = htonll(cas);
77        packet->message.body.seqno = htonll(revSeq);
78        packet->message.body.expiration = htonl(exp);
79        packet->message.body.flags = flags;
80    }
81
82    /**
83     * Given a buffer of data representing a with_meta packet, update the meta
84     * Allows test to avoid lots of allocation/copying when creating inputs.
85     */
86    static void updateMeta(std::vector<char>& wm,
87                           const ItemMetaData& itemMeta) {
88        updateMeta(wm,
89                   itemMeta.cas,
90                   itemMeta.revSeqno,
91                   itemMeta.flags,
92                   uint32_t(itemMeta.exptime));
93    }
94
95    /**
96     * Call the correct engine function for the op (set vs delete)
97     */
98    ENGINE_ERROR_CODE callEngine(protocol_binary_command op,
99                                 std::vector<char>& wm) {
100        if (op == PROTOCOL_BINARY_CMD_DEL_WITH_META ||
101            op == PROTOCOL_BINARY_CMD_DELQ_WITH_META) {
102            return engine->deleteWithMeta(
103                    cookie,
104                    reinterpret_cast<protocol_binary_request_delete_with_meta*>(
105                            wm.data()),
106                    this->addResponse,
107                    DocNamespace::DefaultCollection);
108        } else {
109            return engine->setWithMeta(
110                    cookie,
111                    reinterpret_cast<protocol_binary_request_set_with_meta*>(
112                            wm.data()),
113                    this->addResponse,
114                    DocNamespace::DefaultCollection);
115        }
116    }
117
118    /**
119     * Get the item and check its value, if called for a delete, assuming
120     * delete with value
121     */
122    void checkGetItem(
123            const std::string& key,
124            const std::string& expectedValue,
125            ItemMetaData expectedMeta,
126            ENGINE_ERROR_CODE expectedGetReturnValue = ENGINE_SUCCESS) {
127        auto result = store->get({key, DocNamespace::DefaultCollection},
128                                 vbid,
129                                 cookie,
130                                 GET_DELETED_VALUE);
131
132        ASSERT_EQ(expectedGetReturnValue, result.getStatus());
133
134        if (expectedGetReturnValue == ENGINE_SUCCESS) {
135            EXPECT_EQ(0,
136                      strncmp(expectedValue.data(),
137                              result.item->getData(),
138                              result.item->getNBytes()));
139            EXPECT_EQ(expectedMeta.cas, result.item->getCas());
140            EXPECT_EQ(expectedMeta.revSeqno, result.item->getRevSeqno());
141            EXPECT_EQ(expectedMeta.flags, result.item->getFlags());
142            EXPECT_EQ(expectedMeta.exptime, result.item->getExptime());
143        }
144    }
145
146    void oneOp(protocol_binary_command op,
147               ItemMetaData itemMeta,
148               int options,
149               protocol_binary_response_status expectedResponseStatus,
150               const std::string& key,
151               const std::string& value,
152               const std::vector<char>& emd) {
153        auto swm = buildWithMetaPacket(op,
154                                       0 /*datatype*/,
155                                       vbid,
156                                       0 /*opaque*/,
157                                       0 /*cas*/,
158                                       itemMeta,
159                                       key,
160                                       value,
161                                       emd,
162                                       options);
163        if (expectedResponseStatus == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET) {
164            EXPECT_EQ(ENGINE_NOT_MY_VBUCKET, callEngine(op, swm));
165        } else {
166            EXPECT_EQ(ENGINE_SUCCESS, callEngine(op, swm));
167            EXPECT_EQ(expectedResponseStatus, getAddResponseStatus());
168        }
169    }
170
171    /**
172     * Run one op and check the result
173     */
174    void oneOpAndCheck(protocol_binary_command op,
175                       ItemMetaData itemMeta,
176                       int options,
177                       bool withValue,
178                       protocol_binary_response_status expectedResponseStatus,
179                       ENGINE_ERROR_CODE expectedGetReturnValue,
180                       const std::vector<char>& emd = {}) {
181        std::string key = "mykey";
182        std::string value;
183        if (withValue) {
184            value = createXattrValue("myvalue"); // xattr but stored as raw
185        }
186        oneOp(op, itemMeta, options, expectedResponseStatus, key, value, emd);
187        checkGetItem(key, value, itemMeta, expectedGetReturnValue);
188    }
189
190    // *_with_meta with winning mutations
191    struct TestData {
192        ItemMetaData meta;
193        protocol_binary_response_status expectedStatus;
194    };
195
196    /**
197     * The conflict_win test is reused by seqno/lww and is intended to
198     * test each winning op/meta input
199     */
200    void conflict_win(protocol_binary_command op,
201                      int options,
202                      const std::array<TestData, 4>& testData,
203                      const ItemMetaData& itemMeta);
204    /**
205     * The conflict_lose test is reused by seqno/lww and is intended to
206     * test each winning op/meta input
207     */
208    void conflict_lose(protocol_binary_command op,
209                       int options,
210                       bool withValue,
211                       const std::array<TestData, 4>& testData,
212                       const ItemMetaData& itemMeta);
213
214    /**
215     * The conflict_del_lose_xattr test demonstrates how a delete never gets
216     * to compare xattrs when in conflict.
217     */
218    void conflict_del_lose_xattr(protocol_binary_command op,
219                                 int options,
220                                 bool withValue);
221
222    /**
223     * The conflict_lose_xattr test demonstrates how a set gets
224     * to compare xattrs when in conflict, and the server doc would win.
225     */
226    void conflict_lose_xattr(protocol_binary_command op,
227                             int options,
228                             bool withValue);
229    /**
230     * Initialise an expiry value which allows us to set/get items without them
231     * expiring, i.e. a few years of expiry wiggle room
232     */
233    time_t expiry;
234};
235
236class WithMetaLwwTest : public WithMetaTest {
237public:
238    void SetUp() override {
239        enableLww();
240        WithMetaTest::SetUp();
241    }
242};
243
244class DelWithMetaTest
245        : public WithMetaTest,
246          public ::testing::WithParamInterface<
247                  ::testing::tuple<bool, protocol_binary_command>> {
248public:
249    void SetUp() override {
250        withValue = ::testing::get<0>(GetParam());
251        op = ::testing::get<1>(GetParam());
252        WithMetaTest::SetUp();
253    }
254
255    protocol_binary_command op;
256    bool withValue;
257};
258
259class DelWithMetaLwwTest
260        : public WithMetaTest,
261          public ::testing::WithParamInterface<
262                  ::testing::tuple<bool, protocol_binary_command>> {
263public:
264    void SetUp() override {
265        withValue = ::testing::get<0>(GetParam());
266        op = ::testing::get<1>(GetParam());
267        enableLww();
268        WithMetaTest::SetUp();
269    }
270
271    protocol_binary_command op;
272    bool withValue;
273};
274
275class AllWithMetaTest
276        : public WithMetaTest,
277          public ::testing::WithParamInterface<protocol_binary_command> {};
278
279class AddSetWithMetaTest
280        : public WithMetaTest,
281          public ::testing::WithParamInterface<protocol_binary_command> {};
282
283class AddSetWithMetaLwwTest
284        : public WithMetaTest,
285          public ::testing::WithParamInterface<protocol_binary_command> {
286public:
287    void SetUp() override {
288        enableLww();
289        WithMetaTest::SetUp();
290    }
291};
292
293class XattrWithMetaTest
294        : public WithMetaTest,
295          public ::testing::WithParamInterface<
296                  ::testing::tuple<bool, protocol_binary_command>> {};
297
298class SnappyWithMetaTest : public WithMetaTest,
299                           public ::testing::WithParamInterface<bool> {};
300
301TEST_P(AddSetWithMetaTest, basic) {
302    ItemMetaData itemMeta{0xdeadbeef, 0xf00dcafe, 0xfacefeed, expiry};
303    oneOpAndCheck(GetParam(),
304                  itemMeta,
305                  0, // no-options
306                  true /*set a value*/,
307                  PROTOCOL_BINARY_RESPONSE_SUCCESS,
308                  ENGINE_SUCCESS);
309}
310
311TEST_F(WithMetaTest, basicAdd) {
312    ItemMetaData itemMeta{0xdeadbeef, 0xf00dcafe, 0xfacefeed, expiry};
313    oneOpAndCheck(PROTOCOL_BINARY_CMD_ADD_WITH_META,
314                  itemMeta,
315                  0, // no-options
316                  true /*set a value*/,
317                  PROTOCOL_BINARY_RESPONSE_SUCCESS,
318                  ENGINE_SUCCESS);
319
320    oneOpAndCheck(PROTOCOL_BINARY_CMD_ADD_WITH_META,
321                  itemMeta,
322                  0, // no-options
323                  true /*set a value*/,
324                  PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, // can't do a second add
325                  ENGINE_SUCCESS); // can still get the key
326}
327
328TEST_P(DelWithMetaTest, basic) {
329    ItemMetaData itemMeta{0xdeadbeef, 0xf00dcafe, 0xfacefeed, expiry};
330    // A delete_w_meta against an empty bucket queues a BGFetch (get = ewblock)
331    // A delete_w_meta(with_value) sets the new value (get = success)
332    oneOpAndCheck(op,
333                  itemMeta,
334                  0, // no-options
335                  withValue,
336                  PROTOCOL_BINARY_RESPONSE_SUCCESS,
337                  withValue ? ENGINE_SUCCESS : ENGINE_EWOULDBLOCK);
338}
339
340TEST_P(AllWithMetaTest, invalidCas) {
341    // 0 CAS in the item meta is invalid
342    ItemMetaData itemMeta{0 /*cas*/, 0, 0, 0};
343    oneOpAndCheck(GetParam(),
344                  itemMeta,
345                  0, // no-options
346                  true /*set a value*/,
347                  PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS,
348                  ENGINE_KEY_ENOENT);
349
350    // -1 CAS in the item meta is invalid
351    itemMeta.cas = ~0ull;
352    oneOpAndCheck(GetParam(),
353                  itemMeta,
354                  0, // no-options
355                  true /*set a value*/,
356                  PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS,
357                  ENGINE_KEY_ENOENT);
358}
359
360TEST_P(DelWithMetaTest, invalidCas) {
361    // 0 CAS in the item meta is invalid
362    ItemMetaData itemMeta{0 /*cas*/, 0, 0, 0};
363    oneOpAndCheck(op,
364                  itemMeta,
365                  0, // no-options
366                  withValue /*set a value*/,
367                  PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS,
368                  ENGINE_KEY_ENOENT);
369
370    // -1 CAS in the item meta is invalid
371    itemMeta.cas = ~0ull;
372    oneOpAndCheck(op,
373                  itemMeta,
374                  0, // no-options
375                  withValue /*set a value*/,
376                  PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS,
377                  ENGINE_KEY_ENOENT);
378}
379
380TEST_P(AllWithMetaTest, failForceAccept) {
381    // FORCE_ACCEPT_WITH_META_OPS not allowed unless we're LWW
382    ItemMetaData itemMeta{1, 0, 0, expiry};
383    oneOpAndCheck(GetParam(),
384                  itemMeta,
385                  FORCE_ACCEPT_WITH_META_OPS,
386                  true /*set a value*/,
387                  PROTOCOL_BINARY_RESPONSE_EINVAL,
388                  ENGINE_KEY_ENOENT);
389}
390
391TEST_P(AddSetWithMetaLwwTest, allowForceAccept) {
392    // FORCE_ACCEPT_WITH_META_OPS ok on LWW
393    ItemMetaData itemMeta{1, 0, 0, expiry};
394    oneOpAndCheck(GetParam(),
395                  itemMeta,
396                  FORCE_ACCEPT_WITH_META_OPS,
397                  true /*set a value*/,
398                  PROTOCOL_BINARY_RESPONSE_SUCCESS,
399                  ENGINE_SUCCESS);
400}
401
402TEST_P(DelWithMetaLwwTest, allowForceAccept) {
403    // FORCE_ACCEPT_WITH_META_OPS ok on LWW
404    ItemMetaData itemMeta{1, 0, 0, expiry};
405    oneOpAndCheck(op,
406                  itemMeta,
407                  FORCE_ACCEPT_WITH_META_OPS,
408                  withValue,
409                  PROTOCOL_BINARY_RESPONSE_SUCCESS,
410                  withValue ? ENGINE_SUCCESS : ENGINE_EWOULDBLOCK);
411}
412
413TEST_P(AllWithMetaTest, regenerateCASInvalid) {
414    // REGENERATE_CAS cannot be by itself
415    ItemMetaData itemMeta{1, 0, 0, expiry};
416    oneOpAndCheck(GetParam(),
417                  itemMeta,
418                  REGENERATE_CAS,
419                  true,
420                  PROTOCOL_BINARY_RESPONSE_EINVAL,
421                  ENGINE_KEY_ENOENT);
422}
423
424TEST_P(AllWithMetaTest, forceFail) {
425    store->setVBucketState(vbid, vbucket_state_replica, false);
426    ItemMetaData itemMeta{1, 0, 0, expiry};
427    oneOpAndCheck(GetParam(),
428                  itemMeta,
429                  0 /*no options*/,
430                  true,
431                  PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET,
432                  ENGINE_KEY_ENOENT);
433}
434
435TEST_P(AllWithMetaTest, forceSuccessReplica) {
436    store->setVBucketState(vbid, vbucket_state_replica, false);
437    ItemMetaData itemMeta{1, 0, 0, expiry};
438    oneOpAndCheck(GetParam(),
439                  itemMeta,
440                  FORCE_WITH_META_OP,
441                  true,
442                  PROTOCOL_BINARY_RESPONSE_SUCCESS,
443                  ENGINE_SUCCESS);
444}
445
446TEST_P(AllWithMetaTest, forceSuccessPending) {
447    store->setVBucketState(vbid, vbucket_state_pending, false);
448    ItemMetaData itemMeta{1, 0, 0, expiry};
449    oneOpAndCheck(GetParam(),
450                  itemMeta,
451                  FORCE_WITH_META_OP,
452                  true,
453                  PROTOCOL_BINARY_RESPONSE_SUCCESS,
454                  ENGINE_SUCCESS);
455}
456
457TEST_P(AllWithMetaTest, regenerateCAS) {
458    // Test that
459    uint64_t cas = 1;
460    auto swm =
461            buildWithMetaPacket(GetParam(),
462                                0 /*datatype*/,
463                                vbid /*vbucket*/,
464                                0 /*opaque*/,
465                                0 /*cas*/,
466                                {cas, 0, 0, 0},
467                                "mykey",
468                                "myvalue",
469                                {},
470                                SKIP_CONFLICT_RESOLUTION_FLAG | REGENERATE_CAS);
471
472    EXPECT_EQ(ENGINE_SUCCESS, callEngine(GetParam(), swm));
473    EXPECT_EQ(PROTOCOL_BINARY_RESPONSE_SUCCESS, getAddResponseStatus());
474    auto result = store->get({"mykey", DocNamespace::DefaultCollection},
475                             vbid,
476                             cookie,
477                             GET_DELETED_VALUE);
478    ASSERT_EQ(ENGINE_SUCCESS, result.getStatus());
479    EXPECT_NE(cas, result.item->getCas()) << "CAS didn't change";
480}
481
482TEST_P(AllWithMetaTest, invalid_extlen) {
483    // extlen must be very specific values
484    std::string key = "mykey";
485    std::string value = "myvalue";
486    ItemMetaData itemMeta{1, 0, 0, 0};
487
488    auto swm = buildWithMetaPacket(GetParam(),
489                                   0 /*datatype*/,
490                                   vbid /*vbucket*/,
491                                   0 /*opaque*/,
492                                   0 /*cas*/,
493                                   itemMeta,
494                                   key,
495                                   value);
496
497    auto packet = reinterpret_cast<protocol_binary_request_set_with_meta*>(
498            swm.data());
499    // futz the extlen (yes yes AFL fuzz would be ace)
500    for (uint8_t e = 0; e < 0xff; e++) {
501        if (e == 24 || e == 26 || e == 28 || e == 30) {
502            // Skip the valid sizes
503            continue;
504        }
505        packet->message.header.request.extlen = e;
506        EXPECT_EQ(ENGINE_SUCCESS, callEngine(GetParam(), swm));
507        EXPECT_EQ(PROTOCOL_BINARY_RESPONSE_EINVAL, getAddResponseStatus());
508        checkGetItem(key, value, itemMeta, ENGINE_KEY_ENOENT);
509    }
510}
511
512// Test to verify that a set with meta will store the data
513// as uncompressed in the hash table
514TEST_F(WithMetaTest, storeUncompressedInOffMode) {
515
516    // Return if the bucket compression mode is not 'off'
517    if (engine->getCompressionMode() != BucketCompressionMode::Off) {
518         return;
519    }
520    std::string valueData{R"({"aaaaaaaaa":10000000000})"};
521    auto item = makeCompressibleItem(vbid, makeStoredDocKey("key"), valueData,
522                                     PROTOCOL_BINARY_RAW_BYTES, true);
523
524    item->setCas();
525
526    ASSERT_EQ(PROTOCOL_BINARY_DATATYPE_SNAPPY, item->getDataType());
527
528    cb::const_byte_buffer value{reinterpret_cast<const uint8_t*>(item->getData()),
529                                item->getNBytes()};
530
531    ItemMetaData itemMeta{item->getCas(), item->getRevSeqno(),
532                          item->getFlags(), item->getExptime()};
533
534    mock_set_datatype_support(cookie, PROTOCOL_BINARY_DATATYPE_SNAPPY);
535
536    auto swm = buildWithMetaPacket(PROTOCOL_BINARY_CMD_SET_WITH_META,
537                                   item->getDataType() /*datatype*/,
538                                   vbid /*vbucket*/,
539                                   0 /*opaque*/,
540                                   0 /*cas*/,
541                                   itemMeta,
542                                   std::string("key"),
543                                   std::string(item->getData(), item->getNBytes()),
544                                   {},
545                                   SKIP_CONFLICT_RESOLUTION_FLAG);
546    EXPECT_EQ(ENGINE_SUCCESS, callEngine(PROTOCOL_BINARY_CMD_SET_WITH_META, swm));
547
548    VBucketPtr vb = store->getVBucket(vbid);
549    StoredValue* v(vb->ht.find(makeStoredDocKey("key"), TrackReference::No,
550                               WantsDeleted::No));
551    ASSERT_NE(nullptr, v);
552    EXPECT_EQ(valueData, v->getValue()->to_s());
553    EXPECT_EQ(PROTOCOL_BINARY_DATATYPE_JSON, v->getDatatype());
554}
555
556TEST_P(AllWithMetaTest, nmvb) {
557    std::string key = "mykey";
558    std::string value = "myvalue";
559    auto swm = buildWithMetaPacket(GetParam(),
560                                   0 /*datatype*/,
561                                   vbid + 1 /*vbucket*/,
562                                   0 /*opaque*/,
563                                   0 /*cas*/,
564                                   {1, 0, 0, 0},
565                                   key,
566                                   value);
567    EXPECT_EQ(ENGINE_NOT_MY_VBUCKET, callEngine(GetParam(), swm));
568
569    // Set a dead VB
570    EXPECT_EQ(ENGINE_SUCCESS,
571              store->setVBucketState(vbid + 1, vbucket_state_dead, false));
572    EXPECT_EQ(ENGINE_NOT_MY_VBUCKET, callEngine(GetParam(), swm));
573
574    // update the VB in the packet to the pending one
575    auto packet = reinterpret_cast<protocol_binary_request_header*>(swm.data());
576    packet->request.vbucket = htons(vbid + 2);
577    EXPECT_EQ(ENGINE_SUCCESS,
578              store->setVBucketState(vbid + 2, vbucket_state_pending, false));
579    EXPECT_EQ(ENGINE_EWOULDBLOCK, callEngine(GetParam(), swm));
580    EXPECT_EQ(PROTOCOL_BINARY_RESPONSE_SUCCESS, getAddResponseStatus());
581
582    // Re-run the op now active, else we have a memory leak
583    EXPECT_EQ(ENGINE_SUCCESS,
584              store->setVBucketState(vbid + 2, vbucket_state_active, false));
585    EXPECT_EQ(ENGINE_SUCCESS, callEngine(GetParam(), swm));
586}
587
588TEST_P(AllWithMetaTest, takeoverBackedup) {
589    ItemMetaData itemMeta{1, 0, 0, expiry};
590    auto swm = buildWithMetaPacket(GetParam(),
591                                   0 /*datatype*/,
592                                   vbid /*vbucket*/,
593                                   0 /*opaque*/,
594                                   0 /*cas*/,
595                                   itemMeta,
596                                   "mykey",
597                                   "myvalue");
598
599    store->getVBucket(vbid)->setTakeoverBackedUpState(true);
600    oneOpAndCheck(GetParam(),
601                  itemMeta,
602                  0,
603                  true,
604                  PROTOCOL_BINARY_RESPONSE_ETMPFAIL,
605                  ENGINE_KEY_ENOENT);
606}
607
608TEST_P(AllWithMetaTest, degraded) {
609    ItemMetaData itemMeta{1, 0, 0, expiry};
610    auto swm = buildWithMetaPacket(GetParam(),
611                                   0 /*datatype*/,
612                                   vbid /*vbucket*/,
613                                   0 /*opaque*/,
614                                   0 /*cas*/,
615                                   itemMeta,
616                                   "mykey",
617                                   "myvalue");
618
619    engine->public_enableTraffic(false);
620    oneOpAndCheck(GetParam(),
621                  itemMeta,
622                  0,
623                  true,
624                  PROTOCOL_BINARY_RESPONSE_ETMPFAIL,
625                  ENGINE_KEY_ENOENT);
626}
627
628void WithMetaTest::conflict_lose(protocol_binary_command op,
629                                 int options,
630                                 bool withValue,
631                                 const std::array<TestData, 4>& testData,
632                                 const ItemMetaData& itemMeta) {
633    std::string value;
634    if (withValue) {
635        value = createXattrValue("myvalue");
636    }
637    std::string key = "mykey";
638    // First add a document so we have something to conflict with
639    auto swm = buildWithMetaPacket(PROTOCOL_BINARY_CMD_ADD_WITH_META,
640                                   0,
641                                   vbid /*vbucket*/,
642                                   0 /*opaque*/,
643                                   0 /*cas*/,
644                                   itemMeta,
645                                   key,
646                                   value,
647                                   {},
648                                   options);
649
650    EXPECT_EQ(ENGINE_SUCCESS,
651              callEngine(PROTOCOL_BINARY_CMD_ADD_WITH_META, swm));
652    EXPECT_EQ(PROTOCOL_BINARY_RESPONSE_SUCCESS, getAddResponseStatus());
653
654    for (const auto& td : testData) {
655        oneOp(op, td.meta, options, td.expectedStatus, key, value, {});
656    }
657}
658
659// store a document then <op>_with_meta with equal ItemMeta but xattr on
660void WithMetaTest::conflict_del_lose_xattr(protocol_binary_command op,
661                                           int options,
662                                           bool withValue) {
663    ItemMetaData itemMeta{
664            100 /*cas*/, 100 /*revSeq*/, 100 /*flags*/, expiry /*expiry*/};
665    std::string value;
666    if (withValue) {
667        value = createXattrValue("myvalue");
668    }
669    std::string key = "mykey";
670    // First add a document so we have something to conflict with
671    auto swm = buildWithMetaPacket(PROTOCOL_BINARY_CMD_ADD_WITH_META,
672                                   0 /*xattr off*/,
673                                   vbid /*vbucket*/,
674                                   0 /*opaque*/,
675                                   0 /*cas*/,
676                                   itemMeta,
677                                   key,
678                                   value,
679                                   {},
680                                   options);
681
682    EXPECT_EQ(ENGINE_SUCCESS,
683              callEngine(PROTOCOL_BINARY_CMD_ADD_WITH_META, swm));
684    EXPECT_EQ(PROTOCOL_BINARY_RESPONSE_SUCCESS, getAddResponseStatus());
685
686    // revSeqno/cas/exp/flags equal, xattr on, conflict (a set would win)
687    swm = buildWithMetaPacket(op,
688                              PROTOCOL_BINARY_DATATYPE_XATTR,
689                              vbid /*vbucket*/,
690                              0 /*opaque*/,
691                              0 /*cas*/,
692                              itemMeta,
693                              key,
694                              value,
695                              {},
696                              options);
697    EXPECT_EQ(ENGINE_SUCCESS, callEngine(op, swm));
698    EXPECT_EQ(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, getAddResponseStatus());
699}
700
701void WithMetaTest::conflict_lose_xattr(protocol_binary_command op,
702                                       int options,
703                                       bool withValue) {
704    ItemMetaData itemMeta{
705            100 /*cas*/, 100 /*revSeq*/, 100 /*flags*/, expiry /*expiry*/};
706    std::string value;
707    if (withValue) {
708        value = createXattrValue("myvalue");
709    }
710    std::string key = "mykey";
711    // First add a document so we have something to conflict with
712    auto swm = buildWithMetaPacket(PROTOCOL_BINARY_CMD_ADD_WITH_META,
713                                   PROTOCOL_BINARY_DATATYPE_XATTR,
714                                   vbid /*vbucket*/,
715                                   0 /*opaque*/,
716                                   0 /*cas*/,
717                                   itemMeta,
718                                   key,
719                                   value,
720                                   {},
721                                   options);
722
723    EXPECT_EQ(ENGINE_SUCCESS,
724              callEngine(PROTOCOL_BINARY_CMD_ADD_WITH_META, swm));
725    EXPECT_EQ(PROTOCOL_BINARY_RESPONSE_SUCCESS, getAddResponseStatus());
726
727    // revSeqno/cas/exp/flags equal, xattr off, conflict (a set would win)
728    swm = buildWithMetaPacket(op,
729                              0,
730                              vbid /*vbucket*/,
731                              0 /*opaque*/,
732                              0 /*cas*/,
733                              itemMeta,
734                              key,
735                              value,
736                              {},
737                              options);
738    EXPECT_EQ(ENGINE_SUCCESS, callEngine(op, swm));
739    EXPECT_EQ(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, getAddResponseStatus());
740}
741
742TEST_P(DelWithMetaTest, conflict_lose) {
743    ItemMetaData itemMeta{
744            100 /*cas*/, 100 /*revSeq*/, 100 /*flags*/, expiry /*expiry*/};
745
746    // Conflict test order: 1) seqno 2) cas 3) expiry 4) flags 5) xattr
747    // However deletes only check 1 and 2.
748
749    std::array<TestData, 4> data;
750
751    // 1) revSeqno is less and everything else larger. Expect conflict
752    data[0] = {{101, 99, 101, expiry + 1},
753               PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS};
754    // 2. revSeqno is equal, cas is less, others are larger. Expect conflict
755    data[1] = {{99, 100, 101, expiry + 1},
756               PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS};
757    // 3. revSeqno/cas/flags equal, exp larger. Conflict as exp not checked
758    data[2] = {{100, 100, 100, expiry + 1},
759               PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS};
760    // 4. revSeqno/cas/exp equal, flags larger. Conflict as exp not checked
761    data[3] = {{100, 100, 200, expiry}, PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS};
762
763    conflict_lose(op, 0, withValue, data, itemMeta);
764}
765
766TEST_P(DelWithMetaLwwTest, conflict_lose) {
767    ItemMetaData itemMeta{
768            100 /*cas*/, 100 /*revSeq*/, 100 /*flags*/, expiry /*expiry*/};
769
770    // Conflict test order: 1) cas 2) seqno 3) expiry 4) flags 5) xattr
771    // However deletes only check 1 and 2.
772
773    std::array<TestData, 4> data;
774    // 1) cas is less and everything else larger. Expect conflict
775    data[0] = {{99, 101, 101, expiry + 1},
776               PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS};
777    // 2. cas is equal, revSeqno is less, others are larger. Expect conflict
778    data[1] = {{100, 99, 101, expiry + 1},
779               PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS};
780    // 3. revSeqno/cas/flags equal, exp larger. Conflict as exp not checked
781    data[2] = {{100, 100, 100, expiry + 1},
782               PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS};
783    // 4. revSeqno/cas/exp equal, flags larger. Conflict as exp not checked
784    data[3] = {{100, 100, 200, expiry}, PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS};
785
786    conflict_lose(op, FORCE_ACCEPT_WITH_META_OPS, withValue, data, itemMeta);
787}
788
789TEST_P(DelWithMetaTest, conflict_xattr_lose) {
790    conflict_del_lose_xattr(op, 0, withValue);
791}
792
793TEST_P(DelWithMetaLwwTest, conflict_xattr_lose) {
794    conflict_del_lose_xattr(op, FORCE_ACCEPT_WITH_META_OPS, withValue);
795}
796
797TEST_P(AddSetWithMetaTest, conflict_lose) {
798    ItemMetaData itemMeta{
799            100 /*cas*/, 100 /*revSeq*/, 100 /*flags*/, expiry /*expiry*/};
800
801    // Conflict test order: 1) seqno 2) cas 3) expiry 4) flags 5) xattr
802    std::array<TestData, 4> data;
803    // 1) revSeqno is less and everything else larger. Expect conflict
804    data[0] = {{101, 99, 101, expiry + 1},
805               PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS};
806    // 2. revSeqno is equal, cas is less, others are larger. Expect conflict
807    data[1] = {{99, 100, 101, expiry + 1},
808               PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS};
809    // 3. revSeqno/cas equal, flags larger, exp less, conflict
810    data[2] = {{100, 100, 101, expiry - 1},
811               PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS};
812    // 4. revSeqno/cas/exp equal, flags less, conflict
813    data[3] = {{100, 100, 99, expiry}, PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS};
814
815    conflict_lose(
816            GetParam(), 0 /*options*/, true /*withValue*/, data, itemMeta);
817}
818
819TEST_P(AddSetWithMetaLwwTest, conflict_lose) {
820    ItemMetaData itemMeta{
821            100 /*cas*/, 100 /*revSeq*/, 100 /*flags*/, expiry /*expiry*/};
822
823    // Conflict test order: 1) cas 2) seqno 3) expiry 4) flags 5) xattr
824    std::array<TestData, 4> data;
825    // 1) cas is less and everything else larger. Expect conflict
826    data[0] = {{99, 101, 101, expiry + 1},
827               PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS};
828    // 2. cas is equal, revSeq is less, others are larger. Expect conflict
829    data[1] = {{100, 99, 101, expiry + 1},
830               PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS};
831    // 3. revSeqno/cas equal, flags larger, exp less, conflict
832    data[2] = {{100, 100, 101, expiry - 1},
833               PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS};
834    // 4. revSeqno/cas/exp equal, flags less, conflict
835    data[3] = {{100, 100, 99, expiry}, PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS};
836
837    conflict_lose(GetParam(), FORCE_ACCEPT_WITH_META_OPS, true, data, itemMeta);
838}
839
840TEST_P(AddSetWithMetaTest, conflict_xattr_lose) {
841    conflict_lose_xattr(GetParam(), 0 /*options*/, true /*withvalue*/);
842}
843
844TEST_P(AddSetWithMetaLwwTest, conflict_xattr_lose) {
845    conflict_lose_xattr(
846            GetParam(), FORCE_ACCEPT_WITH_META_OPS, true /*withvalue*/);
847}
848
849// This test will store an item with this meta data then store again
850// using the testData entries
851void WithMetaTest::conflict_win(protocol_binary_command op,
852                                int options,
853                                const std::array<TestData, 4>& testData,
854                                const ItemMetaData& itemMeta) {
855    EXPECT_NE(op, PROTOCOL_BINARY_CMD_ADD_WITH_META);
856    EXPECT_NE(op, PROTOCOL_BINARY_CMD_ADDQ_WITH_META);
857    bool isDelete = op == PROTOCOL_BINARY_CMD_DEL_WITH_META ||
858                    op == PROTOCOL_BINARY_CMD_DELQ_WITH_META;
859    bool isSet = op == PROTOCOL_BINARY_CMD_SET_WITH_META ||
860                 op == PROTOCOL_BINARY_CMD_SETQ_WITH_META;
861
862    int counter = 0;
863    for (auto& td : testData) {
864        // Set our "target" (new key each iteration)
865        std::string key = "mykey" + std::to_string(counter);
866        key.push_back(op); // and the op for test uniqueness
867        std::string value = "newvalue" + std::to_string(counter);
868        auto swm = buildWithMetaPacket(PROTOCOL_BINARY_CMD_SET_WITH_META,
869                                       0 /*datatype*/,
870                                       vbid /*vbucket*/,
871                                       0 /*opaque*/,
872                                       0 /*cas*/,
873                                       itemMeta,
874                                       key,
875                                       "myvalue",
876                                       {},
877                                       options);
878
879        EXPECT_EQ(ENGINE_SUCCESS,
880                  callEngine(PROTOCOL_BINARY_CMD_SET_WITH_META, swm));
881        EXPECT_EQ(PROTOCOL_BINARY_RESPONSE_SUCCESS, getAddResponseStatus())
882                << "Failed to set the target key:" << key;
883
884        // Next the test packet (always with a value).
885        auto wm = buildWithMetaPacket(op,
886                                      0 /*datatype*/,
887                                      vbid /*vbucket*/,
888                                      0 /*opaque*/,
889                                      0 /*cas*/,
890                                      td.meta,
891                                      key,
892                                      value,
893                                      {},
894                                      options);
895
896        // Now set/del against the item using the test iteration metadata
897        EXPECT_EQ(ENGINE_SUCCESS, callEngine(op, wm));
898
899        auto status = getAddResponseStatus();
900        if (isDelete) {
901            EXPECT_EQ(td.expectedStatus, status)
902                    << "Failed deleteWithMeta for iteration " << counter;
903            if (status == PROTOCOL_BINARY_RESPONSE_SUCCESS) {
904                checkGetItem(key, value, td.meta);
905            }
906        } else {
907            EXPECT_TRUE(isSet);
908            EXPECT_EQ(td.expectedStatus, status)
909                    << "Failed setWithMeta for iteration " << counter;
910            checkGetItem(key, value, td.meta);
911        }
912        counter++;
913    }
914
915    // ... Finally give an Item with a datatype (not xattr)
916    std::string key = "mykey" + std::to_string(counter);
917    key.push_back(op); // and the op for test uniqueness
918    auto swm = buildWithMetaPacket(PROTOCOL_BINARY_CMD_ADD_WITH_META,
919                                   PROTOCOL_BINARY_DATATYPE_JSON,
920                                   vbid /*vbucket*/,
921                                   0 /*opaque*/,
922                                   0 /*cas*/,
923                                   itemMeta,
924                                   key,
925                                   "myvalue",
926                                   {},
927                                   options);
928
929    EXPECT_EQ(ENGINE_SUCCESS,
930              callEngine(PROTOCOL_BINARY_CMD_ADD_WITH_META, swm));
931    EXPECT_EQ(PROTOCOL_BINARY_RESPONSE_SUCCESS, getAddResponseStatus());
932
933    // And test same cas/seq/exp/flags but marked with xattr
934    auto xattrValue = createXattrValue("xattr_value");
935    swm = buildWithMetaPacket(op,
936                              PROTOCOL_BINARY_DATATYPE_XATTR,
937                              vbid /*vbucket*/,
938                              0 /*opaque*/,
939                              0 /*cas*/,
940                              itemMeta,
941                              key,
942                              xattrValue,
943                              {},
944                              options);
945    EXPECT_EQ(ENGINE_SUCCESS, callEngine(op, swm));
946
947    if (isSet) {
948        EXPECT_EQ(PROTOCOL_BINARY_RESPONSE_SUCCESS, getAddResponseStatus());
949        checkGetItem(key, xattrValue, itemMeta);
950    } else {
951        EXPECT_TRUE(isDelete);
952        protocol_binary_response_status expected =
953                (options & SKIP_CONFLICT_RESOLUTION_FLAG) != 0
954                        ? PROTOCOL_BINARY_RESPONSE_SUCCESS
955                        : PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS;
956        // del fails as conflict resolution won't get to the XATTR test
957        EXPECT_EQ(expected, getAddResponseStatus());
958    }
959}
960
961// Using test data that should cause a conflict, run the conflict_lose test
962// but with SKIP_CONFLICT_RESOLUTION_FLAG
963TEST_F(WithMetaLwwTest, mutate_conflict_resolve_skipped) {
964    ItemMetaData itemMeta{
965            100 /*cas*/, 100 /*revSeq*/, 100 /*flags*/, expiry /*expiry*/};
966
967    // Conflict test order: 1) cas 2) seqno 3) expiry 4) flags 5) xattr
968    std::array<TestData, 4> data;
969    // 1) cas is less and everything else larger. Expect conflict
970    data[0] = {{99, 101, 101, expiry + 1}, PROTOCOL_BINARY_RESPONSE_SUCCESS};
971    // 2. cas is equal, revSeq is less, others are larger. Expect conflict
972    data[1] = {{100, 99, 101, expiry + 1}, PROTOCOL_BINARY_RESPONSE_SUCCESS};
973    // 3. revSeqno/cas equal, flags larger, exp less, conflict
974    data[2] = {{100, 100, 101, expiry - 1}, PROTOCOL_BINARY_RESPONSE_SUCCESS};
975    // 4. revSeqno/cas/exp equal, flags less, conflict
976    data[3] = {{100, 100, 99, expiry}, PROTOCOL_BINARY_RESPONSE_SUCCESS};
977
978    // Run with SKIP_CONFLICT_RESOLUTION_FLAG
979    conflict_win(PROTOCOL_BINARY_CMD_SET_WITH_META,
980                 FORCE_ACCEPT_WITH_META_OPS | SKIP_CONFLICT_RESOLUTION_FLAG,
981                 data,
982                 itemMeta);
983    conflict_win(PROTOCOL_BINARY_CMD_SETQ_WITH_META,
984                 FORCE_ACCEPT_WITH_META_OPS | SKIP_CONFLICT_RESOLUTION_FLAG,
985                 data,
986                 itemMeta);
987}
988
989// Using test data that should cause a conflict, run the conflict_lose test
990// but with SKIP_CONFLICT_RESOLUTION_FLAG
991TEST_F(WithMetaTest, mutate_conflict_resolve_skipped) {
992    ItemMetaData itemMeta{
993            100 /*cas*/, 100 /*revSeq*/, 100 /*flags*/, expiry /*expiry*/};
994
995    // Conflict test order: 1) cas 2) seqno 3) expiry 4) flags 5) xattr
996    std::array<TestData, 4> data;
997    // 1) cas is less and everything else larger. Expect conflict
998    data[0] = {{99, 101, 101, expiry + 1}, PROTOCOL_BINARY_RESPONSE_SUCCESS};
999    // 2. cas is equal, revSeq is less, others are larger. Expect conflict
1000    data[1] = {{100, 99, 101, expiry + 1}, PROTOCOL_BINARY_RESPONSE_SUCCESS};
1001    // 3. revSeqno/cas equal, flags larger, exp less, conflict
1002    data[2] = {{100, 100, 101, expiry - 1}, PROTOCOL_BINARY_RESPONSE_SUCCESS};
1003    // 4. revSeqno/cas/exp equal, flags less, conflict
1004    data[3] = {{100, 100, 99, expiry}, PROTOCOL_BINARY_RESPONSE_SUCCESS};
1005
1006    // Run with SKIP_CONFLICT_RESOLUTION_FLAG
1007    conflict_win(PROTOCOL_BINARY_CMD_SET_WITH_META,
1008                 SKIP_CONFLICT_RESOLUTION_FLAG,
1009                 data,
1010                 itemMeta);
1011    conflict_win(PROTOCOL_BINARY_CMD_SETQ_WITH_META,
1012                 SKIP_CONFLICT_RESOLUTION_FLAG,
1013                 data,
1014                 itemMeta);
1015}
1016
1017// Using test data that should cause a conflict, run the conflict_lose test
1018// but with SKIP_CONFLICT_RESOLUTION_FLAG
1019TEST_F(WithMetaLwwTest, del_conflict_resolve_skipped) {
1020    ItemMetaData itemMeta{
1021            100 /*cas*/, 100 /*revSeq*/, 100 /*flags*/, expiry /*expiry*/};
1022
1023    // Conflict test order: 1) cas 2) seqno 3) expiry 4) flags 5) xattr
1024    // However deletes only check 1 and 2.
1025
1026    std::array<TestData, 4> data;
1027    // 1) cas is less and everything else larger. Expect conflict
1028    data[0] = {{99, 101, 101, expiry + 1}, PROTOCOL_BINARY_RESPONSE_SUCCESS};
1029    // 2. cas is equal, revSeqno is less, others are larger. Expect conflict
1030    data[1] = {{100, 99, 101, expiry + 1}, PROTOCOL_BINARY_RESPONSE_SUCCESS};
1031    // 3. revSeqno/cas/flags equal, exp larger. Conflict as exp not checked
1032    data[2] = {{100, 100, 100, expiry + 1}, PROTOCOL_BINARY_RESPONSE_SUCCESS};
1033    // 4. revSeqno/cas/exp equal, flags larger. Conflict as exp not checked
1034    data[3] = {{100, 100, 200, expiry}, PROTOCOL_BINARY_RESPONSE_SUCCESS};
1035
1036    // Run with SKIP_CONFLICT_RESOLUTION_FLAG
1037    conflict_win(PROTOCOL_BINARY_CMD_DEL_WITH_META,
1038                 FORCE_ACCEPT_WITH_META_OPS | SKIP_CONFLICT_RESOLUTION_FLAG,
1039                 data,
1040                 itemMeta);
1041    conflict_win(PROTOCOL_BINARY_CMD_DELQ_WITH_META,
1042                 FORCE_ACCEPT_WITH_META_OPS | SKIP_CONFLICT_RESOLUTION_FLAG,
1043                 data,
1044                 itemMeta);
1045}
1046
1047// Using test data that should cause a conflict, run the conflict_lose test
1048// but with SKIP_CONFLICT_RESOLUTION_FLAG
1049TEST_F(WithMetaTest, del_conflict_resolve_skipped) {
1050    ItemMetaData itemMeta{
1051            100 /*cas*/, 100 /*revSeq*/, 100 /*flags*/, expiry /*expiry*/};
1052
1053    // Conflict test order: 1) cas 2) seqno 3) expiry 4) flags 5) xattr
1054    // However deletes only check 1 and 2.
1055
1056    std::array<TestData, 4> data;
1057    // 1) cas is less and everything else larger. Expect conflict
1058    data[0] = {{99, 101, 101, expiry + 1}, PROTOCOL_BINARY_RESPONSE_SUCCESS};
1059    // 2. cas is equal, revSeqno is less, others are larger. Expect conflict
1060    data[1] = {{100, 99, 101, expiry + 1}, PROTOCOL_BINARY_RESPONSE_SUCCESS};
1061    // 3. revSeqno/cas/flags equal, exp larger. Conflict as exp not checked
1062    data[2] = {{100, 100, 100, expiry + 1}, PROTOCOL_BINARY_RESPONSE_SUCCESS};
1063    // 4. revSeqno/cas/exp equal, flags larger. Conflict as exp not checked
1064    data[3] = {{100, 100, 200, expiry}, PROTOCOL_BINARY_RESPONSE_SUCCESS};
1065
1066    // Run with SKIP_CONFLICT_RESOLUTION_FLAG
1067    conflict_win(PROTOCOL_BINARY_CMD_DEL_WITH_META,
1068                 SKIP_CONFLICT_RESOLUTION_FLAG,
1069                 data,
1070                 itemMeta);
1071    conflict_win(PROTOCOL_BINARY_CMD_DELQ_WITH_META,
1072                 SKIP_CONFLICT_RESOLUTION_FLAG,
1073                 data,
1074                 itemMeta);
1075}
1076
1077TEST_F(WithMetaTest, set_conflict_win) {
1078    ItemMetaData itemMeta{
1079            100 /*cas*/, 100 /*revSeq*/, 100 /*flags*/, expiry /*expiry*/};
1080    std::array<TestData, 4> data = {
1081            {{{100, 101, 100, expiry}, // ... mutate with higher seq
1082              PROTOCOL_BINARY_RESPONSE_SUCCESS},
1083             {{101, 100, 100, expiry}, // ... mutate with same but higher cas
1084              PROTOCOL_BINARY_RESPONSE_SUCCESS},
1085             {{100,
1086               100,
1087               100,
1088               expiry + 1}, // ... mutate with same but higher exp
1089              PROTOCOL_BINARY_RESPONSE_SUCCESS},
1090             {{100, 100, 101, expiry}, // ... mutate with same but higher flags
1091              PROTOCOL_BINARY_RESPONSE_SUCCESS}}};
1092
1093    conflict_win(PROTOCOL_BINARY_CMD_SET_WITH_META, 0, data, itemMeta);
1094    conflict_win(PROTOCOL_BINARY_CMD_SETQ_WITH_META, 0, data, itemMeta);
1095}
1096
1097TEST_F(WithMetaTest, del_conflict_win) {
1098    ItemMetaData itemMeta{
1099            100 /*cas*/, 100 /*revSeq*/, 100 /*flags*/, expiry /*expiry*/};
1100    std::array<TestData, 4> data = {{
1101            {{100, 101, 100, expiry}, // ... mutate with higher seq
1102             PROTOCOL_BINARY_RESPONSE_SUCCESS},
1103            {{101, 100, 100, expiry}, // ... mutate with same but higher cas
1104             PROTOCOL_BINARY_RESPONSE_SUCCESS},
1105            {{100, 100, 100, expiry + 1}, // ... mutate with same but higher exp
1106             PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS}, // delete ignores expiry
1107            {{100, 100, 101, expiry}, // ... mutate with same but higher flags
1108             PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS} // delete ignores flags
1109    }};
1110
1111    conflict_win(PROTOCOL_BINARY_CMD_DEL_WITH_META, 0, data, itemMeta);
1112    conflict_win(PROTOCOL_BINARY_CMD_DELQ_WITH_META, 0, data, itemMeta);
1113}
1114
1115TEST_F(WithMetaLwwTest, set_conflict_win) {
1116    ItemMetaData itemMeta{
1117            100 /*cas*/, 100 /*revSeq*/, 100 /*flags*/, expiry /*expiry*/};
1118    std::array<TestData, 4> data = {
1119            {{{101, 100, 100, expiry}, // ... mutate with higher cas
1120              PROTOCOL_BINARY_RESPONSE_SUCCESS},
1121             {{100, 101, 100, expiry}, // ... mutate with same but higher seq
1122              PROTOCOL_BINARY_RESPONSE_SUCCESS},
1123             {{100,
1124               100,
1125               100,
1126               expiry + 1}, // ... mutate with same but higher exp
1127              PROTOCOL_BINARY_RESPONSE_SUCCESS},
1128             {{100, 100, 101, expiry}, // ... mutate with same but higher flags
1129              PROTOCOL_BINARY_RESPONSE_SUCCESS}}};
1130
1131    conflict_win(PROTOCOL_BINARY_CMD_SET_WITH_META,
1132                 FORCE_ACCEPT_WITH_META_OPS,
1133                 data,
1134                 itemMeta);
1135    conflict_win(PROTOCOL_BINARY_CMD_SETQ_WITH_META,
1136                 FORCE_ACCEPT_WITH_META_OPS,
1137                 data,
1138                 itemMeta);
1139}
1140
1141TEST_F(WithMetaLwwTest, del_conflict_win) {
1142    ItemMetaData itemMeta{
1143            100 /*cas*/, 100 /*revSeq*/, 100 /*flags*/, expiry /*expiry*/};
1144    std::array<TestData, 4> data = {{
1145            {{101, 100, 100, expiry}, // ... mutate with higher cas
1146             PROTOCOL_BINARY_RESPONSE_SUCCESS},
1147            {{100, 101, 100, expiry}, // ... mutate with same but higher seq
1148             PROTOCOL_BINARY_RESPONSE_SUCCESS},
1149            {{100, 100, 100, expiry + 1}, // ... mutate with same but higher exp
1150             PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS}, // delete ignores expiry
1151            {{100, 100, 101, expiry}, // ... mutate with same but higher flags
1152             PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS} // delete ignores flags
1153    }};
1154
1155    conflict_win(PROTOCOL_BINARY_CMD_DEL_WITH_META,
1156                 FORCE_ACCEPT_WITH_META_OPS,
1157                 data,
1158                 itemMeta);
1159    conflict_win(PROTOCOL_BINARY_CMD_DELQ_WITH_META,
1160                 FORCE_ACCEPT_WITH_META_OPS,
1161                 data,
1162                 itemMeta);
1163}
1164
1165TEST_P(AllWithMetaTest, markJSON) {
1166    // Write a XATTR doc with JSON body, expect the doc to be marked as JSON
1167    auto value = createXattrValue(
1168            R"({"json":"yesplease"})");
1169    auto swm = buildWithMetaPacket(GetParam(),
1170                                   PROTOCOL_BINARY_DATATYPE_XATTR,
1171                                   vbid /*vbucket*/,
1172                                   0 /*opaque*/,
1173                                   0 /*cas*/,
1174                                   {100, 100, 100, expiry},
1175                                   "json",
1176                                   value);
1177    EXPECT_EQ(ENGINE_SUCCESS, callEngine(GetParam(), swm));
1178    EXPECT_EQ(PROTOCOL_BINARY_RESPONSE_SUCCESS, getAddResponseStatus());
1179
1180    auto result = store->get({"json", DocNamespace::DefaultCollection},
1181                             vbid,
1182                             cookie,
1183                             GET_DELETED_VALUE);
1184    ASSERT_EQ(ENGINE_SUCCESS, result.getStatus());
1185    EXPECT_EQ(0,
1186              strncmp(value.data(),
1187                      result.item->getData(),
1188                      result.item->getNBytes()));
1189    EXPECT_EQ(PROTOCOL_BINARY_DATATYPE_JSON | PROTOCOL_BINARY_DATATYPE_XATTR,
1190              result.item->getDataType());
1191}
1192
1193// Test uses an XATTR body that has 1 system key (see createXattrValue)
1194TEST_P(SnappyWithMetaTest, xattrPruneUserKeysOnDelete1) {
1195    auto value = createXattrValue(
1196            R"({"json":"yesplease"})", true, GetParam());
1197    uint8_t snappy = GetParam() ? PROTOCOL_BINARY_DATATYPE_SNAPPY : 0;
1198    ItemMetaData itemMeta{1, 1, 0, expiry};
1199    std::string mykey = "mykey";
1200    DocKey key{mykey, DocNamespace::DefaultCollection};
1201    auto swm = buildWithMetaPacket(PROTOCOL_BINARY_CMD_SET_WITH_META,
1202                                   PROTOCOL_BINARY_DATATYPE_XATTR | snappy,
1203                                   vbid /*vbucket*/,
1204                                   0 /*opaque*/,
1205                                   0 /*cas*/,
1206                                   itemMeta,
1207                                   mykey,
1208                                   value);
1209
1210    mock_set_datatype_support(cookie, PROTOCOL_BINARY_DATATYPE_SNAPPY);
1211
1212    EXPECT_EQ(ENGINE_SUCCESS,
1213              callEngine(PROTOCOL_BINARY_CMD_SET_WITH_META, swm));
1214    EXPECT_EQ(std::make_pair(false, size_t(1)),
1215              getEPBucket().flushVBucket(vbid));
1216
1217    itemMeta.revSeqno++; // make delete succeed
1218    auto dwm = buildWithMeta(
1219            PROTOCOL_BINARY_CMD_DEL_WITH_META, itemMeta, mykey, {});
1220    EXPECT_EQ(ENGINE_SUCCESS,
1221              callEngine(PROTOCOL_BINARY_CMD_DEL_WITH_META, dwm));
1222
1223    EXPECT_EQ(std::make_pair(false, size_t(1)),
1224              getEPBucket().flushVBucket(vbid));
1225
1226    auto options = get_options_t(QUEUE_BG_FETCH | GET_DELETED_VALUE);
1227    auto result = store->get(key, vbid, cookie, options);
1228    EXPECT_EQ(ENGINE_EWOULDBLOCK, result.getStatus());
1229    runBGFetcherTask();
1230
1231    result = store->get(key, vbid, cookie, options);
1232    ASSERT_EQ(ENGINE_SUCCESS, result.getStatus());
1233
1234    // Now reconstruct a XATTR Blob and validate the user keys are gone
1235    // These code relies on knowing what createXattrValue generates.
1236    auto sz = cb::xattr::get_body_offset(
1237            {result.item->getData(), result.item->getNBytes()});
1238
1239    cb::xattr::Blob blob({const_cast<char*>(result.item->getData()), sz},
1240                         false);
1241
1242    EXPECT_EQ(0, blob.get("user").size());
1243    EXPECT_EQ(0, blob.get("meta").size());
1244    ASSERT_NE(0, blob.get("_sync").size());
1245    EXPECT_STREQ("{\"cas\":\"0xdeadbeefcafefeed\"}",
1246                 reinterpret_cast<char*>(blob.get("_sync").data()));
1247
1248    auto itm = result.item.get();
1249    EXPECT_TRUE(itm->isDeleted()) << "Not deleted " << *itm;
1250    // The meta-data should match the delete_with_meta
1251    EXPECT_EQ(itemMeta.cas, itm->getCas());
1252    EXPECT_EQ(itemMeta.flags, itm->getFlags());
1253    EXPECT_EQ(itemMeta.revSeqno, itm->getRevSeqno());
1254    EXPECT_EQ(itemMeta.exptime, itm->getExptime());
1255
1256}
1257
1258// Test uses an XATTR body that has no system keys
1259TEST_P(XattrWithMetaTest, xattrPruneUserKeysOnDelete2) {
1260    auto value = createXattrValue(
1261            R"({"json":"yesplease"})", false, ::testing::get<0>(GetParam()));
1262    uint8_t snappy =
1263            ::testing::get<0>(GetParam()) ? PROTOCOL_BINARY_DATATYPE_SNAPPY : 0;
1264
1265    ItemMetaData itemMeta{1, 1, 0, expiry};
1266    std::string mykey = "mykey";
1267    DocKey key{mykey, DocNamespace::DefaultCollection};
1268    auto swm = buildWithMetaPacket(PROTOCOL_BINARY_CMD_SET_WITH_META,
1269                                   PROTOCOL_BINARY_DATATYPE_XATTR | snappy,
1270                                   vbid /*vbucket*/,
1271                                   0 /*opaque*/,
1272                                   0 /*cas*/,
1273                                   itemMeta,
1274                                   mykey,
1275                                   value);
1276
1277    mock_set_datatype_support(cookie, PROTOCOL_BINARY_DATATYPE_SNAPPY);
1278
1279    EXPECT_EQ(ENGINE_SUCCESS,
1280              callEngine(PROTOCOL_BINARY_CMD_SET_WITH_META, swm));
1281    EXPECT_EQ(std::make_pair(false, size_t(1)),
1282              getEPBucket().flushVBucket(vbid));
1283
1284    itemMeta.revSeqno++; // make delete succeed
1285    auto dwm = buildWithMeta(
1286            PROTOCOL_BINARY_CMD_DEL_WITH_META, itemMeta, mykey, {});
1287    EXPECT_EQ(ENGINE_SUCCESS,
1288              callEngine(PROTOCOL_BINARY_CMD_DEL_WITH_META, dwm));
1289
1290    EXPECT_EQ(std::make_pair(false, size_t(1)),
1291              getEPBucket().flushVBucket(vbid));
1292
1293    auto options = get_options_t(QUEUE_BG_FETCH | GET_DELETED_VALUE);
1294    auto result = store->get(key, vbid, cookie, options);
1295    EXPECT_EQ(ENGINE_EWOULDBLOCK, result.getStatus());
1296
1297    // Run the BGFetcher task
1298    MockGlobalTask mockTask(engine->getTaskable(), TaskId::MultiBGFetcherTask);
1299    store->getVBucket(vbid)->getShard()->getBgFetcher()->run(&mockTask);
1300
1301    options = get_options_t(options & (~GET_DELETED_VALUE));
1302
1303    // K/V is gone
1304    result = store->get(key, vbid, cookie, options);
1305    EXPECT_EQ(ENGINE_KEY_ENOENT, result.getStatus());
1306}
1307
1308TEST_P(AllWithMetaTest, skipConflicts) {
1309    ItemMetaData itemMeta{1, 0, 0, expiry};
1310    auto swm = buildWithMetaPacket(GetParam(),
1311                                   0 /*datatype*/,
1312                                   vbid /*vbucket*/,
1313                                   0 /*opaque*/,
1314                                   0 /*cas*/,
1315                                   itemMeta,
1316                                   "mykey",
1317                                   "myvalue");
1318
1319    engine->public_enableTraffic(false);
1320    oneOpAndCheck(GetParam(),
1321                  itemMeta,
1322                  0,
1323                  true,
1324                  PROTOCOL_BINARY_RESPONSE_ETMPFAIL,
1325                  ENGINE_KEY_ENOENT);
1326}
1327
1328// Perform a DeleteWithMeta with a deleteTime of 1, verify that time comes back
1329// after a fetch and getMeta
1330TEST_P(DelWithMetaTest, setting_deleteTime) {
1331    ItemMetaData itemMeta{0xdeadbeef, 0xf00dcafe, 0xfacefeed, 1};
1332    oneOpAndCheck(op,
1333                  itemMeta,
1334                  0, // no-options
1335                  withValue,
1336                  PROTOCOL_BINARY_RESPONSE_SUCCESS,
1337                  withValue ? ENGINE_SUCCESS : ENGINE_EWOULDBLOCK);
1338
1339    EXPECT_EQ(std::make_pair(false, size_t(1)),
1340              getEPBucket().flushVBucket(vbid));
1341
1342    ItemMetaData metadata;
1343    uint32_t deleted = 0;
1344    uint8_t datatype = 0;
1345    EXPECT_EQ(ENGINE_EWOULDBLOCK,
1346              store->getMetaData({"mykey", DocNamespace::DefaultCollection},
1347                                 vbid,
1348                                 cookie,
1349                                 metadata,
1350                                 deleted,
1351                                 datatype));
1352    MockGlobalTask mockTask(engine->getTaskable(), TaskId::MultiBGFetcherTask);
1353    store->getVBucket(vbid)->getShard()->getBgFetcher()->run(&mockTask);
1354    EXPECT_EQ(ENGINE_SUCCESS,
1355              store->getMetaData({"mykey", DocNamespace::DefaultCollection},
1356                                 vbid,
1357                                 cookie,
1358                                 metadata,
1359                                 deleted,
1360                                 datatype));
1361    EXPECT_EQ(itemMeta.exptime, metadata.exptime);
1362}
1363
1364// Perform a DeleteWithMeta with a deleteTime of 0 and verify that a time is
1365// generated.
1366TEST_P(DelWithMetaTest, setting_zero_deleteTime) {
1367    ItemMetaData itemMeta{0xdeadbeef, 0xf00dcafe, 0xfacefeed, 0};
1368    oneOpAndCheck(op,
1369                  itemMeta,
1370                  0, // no-options
1371                  withValue,
1372                  PROTOCOL_BINARY_RESPONSE_SUCCESS,
1373                  withValue ? ENGINE_SUCCESS : ENGINE_EWOULDBLOCK);
1374
1375    EXPECT_EQ(std::make_pair(false, size_t(1)),
1376              getEPBucket().flushVBucket(vbid));
1377
1378    ItemMetaData metadata;
1379    uint32_t deleted = 0;
1380    uint8_t datatype = 0;
1381    EXPECT_EQ(ENGINE_EWOULDBLOCK,
1382              store->getMetaData({"mykey", DocNamespace::DefaultCollection},
1383                                 vbid,
1384                                 cookie,
1385                                 metadata,
1386                                 deleted,
1387                                 datatype));
1388    MockGlobalTask mockTask(engine->getTaskable(), TaskId::MultiBGFetcherTask);
1389    store->getVBucket(vbid)->getShard()->getBgFetcher()->run(&mockTask);
1390    EXPECT_EQ(ENGINE_SUCCESS,
1391              store->getMetaData({"mykey", DocNamespace::DefaultCollection},
1392                                 vbid,
1393                                 cookie,
1394                                 metadata,
1395                                 deleted,
1396                                 datatype));
1397    EXPECT_NE(0, metadata.exptime);
1398}
1399
1400TEST_P(DelWithMetaTest, MB_31141) {
1401    ItemMetaData itemMeta{0xdeadbeef, 0xf00dcafe, 0xfacefeed, expiry};
1402    // Do a delete with valid extended meta
1403    // see - ep-engine/docs/protocol/del_with_meta.md
1404    oneOpAndCheck(op,
1405                  itemMeta,
1406                  0, // no-options
1407                  withValue,
1408                  PROTOCOL_BINARY_RESPONSE_SUCCESS,
1409                  withValue ? ENGINE_SUCCESS : ENGINE_EWOULDBLOCK,
1410                  {0x01, 0x01, 0x00, 0x01, 0x01});
1411
1412    EXPECT_EQ(std::make_pair(false, size_t(1)),
1413              getEPBucket().flushVBucket(vbid));
1414
1415    auto options = get_options_t(QUEUE_BG_FETCH | GET_DELETED_VALUE);
1416    auto result = store->get(
1417            {"mykey", DocNamespace::DefaultCollection}, vbid, cookie, options);
1418    EXPECT_EQ(ENGINE_EWOULDBLOCK, result.getStatus());
1419
1420    // Run the BGFetcher task
1421    MockGlobalTask mockTask(engine->getTaskable(), TaskId::MultiBGFetcherTask);
1422    store->getVBucket(vbid)->getShard()->getBgFetcher()->run(&mockTask);
1423
1424    result = store->get(
1425            {"mykey", DocNamespace::DefaultCollection}, vbid, cookie, options);
1426    ASSERT_EQ(ENGINE_SUCCESS, result.getStatus());
1427
1428    // Before the fix 5.0+ could of left the value as 5, the size of the
1429    // extended metadata
1430    int expectedSize = withValue ? 275 : 0;
1431    EXPECT_EQ(expectedSize, result.item->getNBytes());
1432}
1433
1434TEST_P(AddSetWithMetaTest, MB_31141) {
1435    ItemMetaData itemMeta{0xdeadbeef, 0xf00dcafe, 0xfacefeed, expiry};
1436    // Do a set/add with valid extended meta
1437    // see - ep-engine/docs/protocol/del_with_meta.md
1438    oneOpAndCheck(GetParam(),
1439                  itemMeta,
1440                  0, // no-options
1441                  true,
1442                  PROTOCOL_BINARY_RESPONSE_SUCCESS,
1443                  ENGINE_SUCCESS,
1444                  {0x01, 0x01, 0x00, 0x01, 0x01});
1445
1446    EXPECT_EQ(std::make_pair(false, size_t(1)),
1447              getEPBucket().flushVBucket(vbid));
1448
1449    auto options = get_options_t(QUEUE_BG_FETCH);
1450    auto result = store->get(
1451            {"mykey", DocNamespace::DefaultCollection}, vbid, cookie, options);
1452    ASSERT_EQ(ENGINE_SUCCESS, result.getStatus());
1453    // Only the value should come back
1454    EXPECT_EQ(275, result.item->getNBytes());
1455}
1456
1457auto opcodeValues = ::testing::Values(PROTOCOL_BINARY_CMD_SET_WITH_META,
1458                                      PROTOCOL_BINARY_CMD_SETQ_WITH_META,
1459                                      PROTOCOL_BINARY_CMD_ADD_WITH_META,
1460                                      PROTOCOL_BINARY_CMD_ADDQ_WITH_META,
1461                                      PROTOCOL_BINARY_CMD_DEL_WITH_META,
1462                                      PROTOCOL_BINARY_CMD_DELQ_WITH_META);
1463
1464auto addSetOpcodeValues = ::testing::Values(PROTOCOL_BINARY_CMD_SET_WITH_META,
1465                                            PROTOCOL_BINARY_CMD_SETQ_WITH_META,
1466                                            PROTOCOL_BINARY_CMD_ADD_WITH_META,
1467                                            PROTOCOL_BINARY_CMD_ADDQ_WITH_META);
1468
1469auto deleteOpcodeValues = ::testing::Values(PROTOCOL_BINARY_CMD_DEL_WITH_META,
1470                                            PROTOCOL_BINARY_CMD_DELQ_WITH_META);
1471
1472struct PrintToStringCombinedName {
1473    std::string
1474    operator()(const ::testing::TestParamInfo<
1475               ::testing::tuple<bool, protocol_binary_command>>& info) const {
1476        std::string rv = memcached_opcode_2_text(::testing::get<1>(info.param));
1477        if (::testing::get<0>(info.param)) {
1478            rv += "_with_value";
1479        }
1480        return rv;
1481    }
1482};
1483
1484struct PrintToStringCombinedNameSnappyOnOff {
1485    std::string
1486    operator()(const ::testing::TestParamInfo<
1487               ::testing::tuple<bool, protocol_binary_command>>& info) const {
1488        std::string rv = memcached_opcode_2_text(::testing::get<1>(info.param));
1489        if (::testing::get<0>(info.param)) {
1490            rv += "_snappy";
1491        }
1492        return rv;
1493    }
1494};
1495
1496struct PrintOpcode {
1497    std::string operator()(
1498            const ::testing::TestParamInfo<protocol_binary_command>& info)
1499            const {
1500        return memcached_opcode_2_text(info.param);
1501    }
1502};
1503
1504struct PrintSnappyOnOff {
1505    std::string operator()(const ::testing::TestParamInfo<bool>& info) const {
1506        if (info.param) {
1507            return "snappy";
1508        }
1509        return "no_snappy";
1510    }
1511};
1512
1513INSTANTIATE_TEST_CASE_P(DelWithMeta,
1514                        DelWithMetaTest,
1515                        ::testing::Combine(::testing::Bool(),
1516                                           deleteOpcodeValues),
1517                        PrintToStringCombinedName());
1518
1519INSTANTIATE_TEST_CASE_P(DelWithMetaLww,
1520                        DelWithMetaLwwTest,
1521                        ::testing::Combine(::testing::Bool(),
1522                                           deleteOpcodeValues),
1523                        PrintToStringCombinedName());
1524
1525INSTANTIATE_TEST_CASE_P(AddSetWithMeta,
1526                        AddSetWithMetaTest,
1527                        addSetOpcodeValues,
1528                        PrintOpcode());
1529
1530INSTANTIATE_TEST_CASE_P(AddSetWithMetaLww,
1531                        AddSetWithMetaLwwTest,
1532                        addSetOpcodeValues,
1533                        PrintOpcode());
1534
1535INSTANTIATE_TEST_CASE_P(AddSetDelMeta,
1536                        AllWithMetaTest,
1537                        opcodeValues,
1538                        PrintOpcode());
1539
1540INSTANTIATE_TEST_CASE_P(SnappyWithMetaTest,
1541                        SnappyWithMetaTest,
1542                        ::testing::Bool(),
1543                        PrintSnappyOnOff());
1544
1545INSTANTIATE_TEST_CASE_P(AddSetDelXattrMeta,
1546                        XattrWithMetaTest,
1547                        // Bool for snappy on/off
1548                        ::testing::Combine(::testing::Bool(), opcodeValues),
1549                        PrintToStringCombinedNameSnappyOnOff());
1550