1/* -*- MODE: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/*
3 *     Copyright 2016 Couchbase, Inc
4 *
5 *   Licensed under the Apache License, Version 2.0 (the "License");
6 *   you may not use this file except in compliance with the License.
7 *   You may obtain a copy of the License at
8 *
9 *       http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *   Unless required by applicable law or agreed to in writing, software
12 *   distributed under the License is distributed on an "AS IS" BASIS,
13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *   See the License for the specific language governing permissions and
15 *   limitations under the License.
16 */
17
18/*
19 * Testsuite for XDCR-related functionality in ep-engine.
20 */
21
22#include "config.h"
23
24#include "ep_test_apis.h"
25#include "ep_testsuite_common.h"
26#include "hlc.h"
27
28#include <platform/cb_malloc.h>
29
30// Helper functions ///////////////////////////////////////////////////////////
31
32static void verifyLastMetaData(ItemMetaData imd) {
33    checkeq(imd.revSeqno, last_meta.revSeqno, "Seqno didn't match");
34    checkeq(imd.cas, last_meta.cas, "Cas didn't match");
35    checkeq(imd.exptime, last_meta.exptime, "Expiration time didn't match");
36    checkeq(imd.flags, last_meta.flags, "Flags didn't match");
37}
38
39// Testcases //////////////////////////////////////////////////////////////////
40
41static enum test_result test_get_meta(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)
42{
43    char const *key = "test_get_meta";
44    item *i = NULL;
45    checkeq(ENGINE_SUCCESS,
46            store(h, h1, NULL, OPERATION_SET, key, "somevalue", &i),
47            "Failed set.");
48    Item *it = reinterpret_cast<Item*>(i);
49    // check the stat
50    size_t temp = get_int_stat(h, h1, "ep_num_ops_get_meta");
51    check(temp == 0, "Expect zero getMeta ops");
52
53    check(get_meta(h, h1, key), "Expected to get meta");
54    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
55            "Expected success");
56
57    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
58    ItemMetaData metadata(it->getCas(), it->getRevSeqno(),
59                          it->getFlags(), it->getExptime());
60    verifyLastMetaData(metadata);
61
62    // check the stat again
63    temp = get_int_stat(h, h1, "ep_num_ops_get_meta");
64    check(temp == 1, "Expect one getMeta op");
65
66    h1->release(h, NULL, i);
67    return SUCCESS;
68}
69
70static enum test_result test_get_meta_with_extras(ENGINE_HANDLE *h,
71                                                  ENGINE_HANDLE_V1 *h1)
72{
73    const char *key1 = "test_getm_one";
74    item *i = NULL;
75    checkeq(ENGINE_SUCCESS,
76            store(h, h1, NULL, OPERATION_SET, key1, "somevalue", &i),
77            "Failed set.");
78
79    wait_for_flusher_to_settle(h, h1);
80
81    Item *it1 = reinterpret_cast<Item*>(i);
82    // check the stat
83    size_t temp = get_int_stat(h, h1, "ep_num_ops_get_meta");
84    check(temp == 0, "Expect zero getMeta ops");
85
86    check(get_meta(h, h1, key1, true), "Expected to get meta");
87    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
88    ItemMetaData metadata1(it1->getCas(), it1->getRevSeqno(),
89                           it1->getFlags(), it1->getExptime());
90    verifyLastMetaData(metadata1);
91    // check the stat again
92    temp = get_int_stat(h, h1, "ep_num_ops_get_meta");
93    check(temp == 1, "Expect one getMeta op");
94    h1->release(h, NULL, i);
95
96    // restart
97    testHarness.reload_engine(&h, &h1,
98                              testHarness.engine_path,
99                              testHarness.get_current_testcase()->cfg,
100                              true, true);
101    wait_for_warmup_complete(h, h1);
102
103    check(get_meta(h, h1, key1, true), "Expected to get meta");
104    check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS, "Expected success");
105    verifyLastMetaData(metadata1);
106
107    return SUCCESS;
108}
109
110static enum test_result test_get_meta_deleted(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)
111{
112    char const *key = "k1";
113    item *i = NULL;
114
115    checkeq(ENGINE_SUCCESS,
116            store(h, h1, NULL, OPERATION_SET, key, "somevalue", &i),
117            "Failed set.");
118    h1->release(h, NULL, i);
119    checkeq(ENGINE_SUCCESS,
120            store(h, h1, NULL, OPERATION_SET, key, "somevalue", &i),
121            "Failed set.");
122
123    Item *it = reinterpret_cast<Item*>(i);
124    wait_for_flusher_to_settle(h, h1);
125
126    checkeq(ENGINE_SUCCESS, del(h, h1, key, it->getCas(), 0), "Delete failed");
127    wait_for_flusher_to_settle(h, h1);
128
129    // check the stat
130    int temp = get_int_stat(h, h1, "ep_num_ops_get_meta");
131    check(temp == 0, "Expect zero getMeta ops");
132
133    check(get_meta(h, h1, key), "Expected to get meta");
134    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
135    check(last_deleted_flag, "Expected deleted flag to be set");
136    check(last_meta.revSeqno == it->getRevSeqno() + 1, "Expected seqno to match");
137    check(last_meta.cas != it->getCas() , "Expected cas to be different");
138    check(last_meta.flags == it->getFlags(), "Expected flags to match");
139
140    // check the stat again
141    temp = get_int_stat(h, h1, "ep_num_ops_get_meta");
142    checkeq(1, temp, "Expect one getMeta op");
143
144    h1->release(h, NULL, i);
145    return SUCCESS;
146}
147
148static enum test_result test_get_meta_nonexistent(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)
149{
150    char const *key = "k1";
151
152    // check the stat
153    int temp = get_int_stat(h, h1, "ep_num_ops_get_meta");
154    check(temp == 0, "Expect zero getMeta ops");
155    check(!get_meta(h, h1, key), "Expected get meta to return false");
156    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, last_status.load(),
157            "Expected enoent");
158    // check the stat again
159    temp = get_int_stat(h, h1, "ep_num_ops_get_meta");
160    checkeq(1, temp, "Expect one getMeta ops");
161
162    return SUCCESS;
163}
164
165static enum test_result test_get_meta_with_get(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)
166{
167    char const *key1 = "key1";
168    char const *key2 = "key2";
169
170    item *i = NULL;
171    // test get_meta followed by get for an existing key. should pass.
172    checkeq(ENGINE_SUCCESS,
173            store(h, h1, NULL, OPERATION_SET, key1, "somevalue", &i),
174            "Failed set.");
175    h1->release(h, NULL, i);
176    wait_for_flusher_to_settle(h, h1);
177    // check the stat
178    int temp = get_int_stat(h, h1, "ep_num_ops_get_meta");
179    check(temp == 0, "Expect zero getMeta ops");
180    check(get_meta(h, h1, key1), "Expected to get meta");
181    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
182    checkeq(ENGINE_SUCCESS,
183            h1->get(h, NULL, &i, key1, strlen(key1), 0), "Expected get success");
184    h1->release(h, NULL, i);
185    // check the stat again
186    temp = get_int_stat(h, h1, "ep_num_ops_get_meta");
187    check(temp == 1, "Expect one getMeta op");
188
189    // test get_meta followed by get for a deleted key. should fail.
190    checkeq(ENGINE_SUCCESS,
191            del(h, h1, key1, 0, 0), "Delete failed");
192    wait_for_flusher_to_settle(h, h1);
193    check(get_meta(h, h1, key1), "Expected to get meta");
194    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
195            "Expected success");
196    check(last_deleted_flag, "Expected deleted flag to be set");
197    checkeq(ENGINE_KEY_ENOENT,
198            h1->get(h, NULL, &i, key1, strlen(key1), 0), "Expected enoent");
199    // check the stat again
200    temp = get_int_stat(h, h1, "ep_num_ops_get_meta");
201    checkeq(2, temp, "Expect more getMeta ops");
202
203    // test get_meta followed by get for a nonexistent key. should fail.
204    check(!get_meta(h, h1, key2), "Expected get meta to return false");
205    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, last_status.load(),
206            "Expected enoent");
207    checkeq(ENGINE_KEY_ENOENT,
208            h1->get(h, NULL, &i, key2, strlen(key2), 0), "Expected enoent");
209    // check the stat again
210    temp = get_int_stat(h, h1, "ep_num_ops_get_meta");
211    checkeq(3, temp, "Expected one extra getMeta ops");
212
213    return SUCCESS;
214}
215
216static enum test_result test_get_meta_with_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)
217{
218    char const *key1 = "key1";
219    char const *key2 = "key2";
220
221    item *i = NULL;
222    ItemMetaData itm_meta;
223
224    // test get_meta followed by set for an existing key. should pass.
225    checkeq(ENGINE_SUCCESS,
226            store(h, h1, NULL, OPERATION_SET, key1, "somevalue", &i),
227            "Failed set.");
228    h1->release(h, NULL, i);
229    wait_for_flusher_to_settle(h, h1);
230    wait_for_stat_to_be(h, h1, "curr_items", 1);
231
232    // check the stat
233    checkeq(0, get_int_stat(h, h1, "ep_num_ops_get_meta"), "Expect zero getMeta ops");
234    check(get_meta(h, h1, key1), "Expected to get meta");
235    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
236            "Expected success");
237    checkeq(ENGINE_SUCCESS,
238            store(h, h1, NULL, OPERATION_SET, key1, "someothervalue", &i),
239            "Failed set.");
240    // check the stat
241    checkeq(1, get_int_stat(h, h1, "ep_num_ops_get_meta"), "Expect one getMeta op");
242    checkeq(1, get_int_stat(h, h1, "curr_items"), "Expected single curr_items");
243    checkeq(0, get_int_stat(h, h1, "curr_temp_items"), "Expected zero temp_items");
244    h1->release(h, NULL, i);
245
246    // check curr, temp item counts
247    checkeq(1, get_int_stat(h, h1, "curr_items"), "Expected single curr_items");
248    checkeq(0, get_int_stat(h, h1, "curr_temp_items"), "Expected zero temp_items");
249
250    // test get_meta followed by set for a deleted key. should pass.
251    checkeq(ENGINE_SUCCESS, del(h, h1, key1, 0, 0), "Delete failed");
252    wait_for_flusher_to_settle(h, h1);
253
254    wait_for_stat_to_be(h, h1, "curr_items", 0);
255    check(get_meta(h, h1, key1), "Expected to get meta");
256    checkeq(0, get_int_stat(h, h1, "curr_items"), "Expected zero curr_items");
257    checkeq(1, get_int_stat(h, h1, "curr_temp_items"), "Expected single temp_items");
258
259    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
260            "Expected success");
261    check(last_deleted_flag, "Expected deleted flag to be set");
262    checkeq(ENGINE_SUCCESS,
263            store(h, h1, NULL, OPERATION_SET, key1, "someothervalue", &i),
264            "Failed set.");
265
266    checkeq(1, get_int_stat(h, h1, "curr_items"), "Expected single curr_items");
267    checkeq(0, get_int_stat(h, h1, "curr_temp_items"), "Expected zero temp_items");
268
269    // check the stat
270    checkeq(2, get_int_stat(h, h1, "ep_num_ops_get_meta"), "Expect more getMeta ops");
271    h1->release(h, NULL, i);
272
273    // test get_meta followed by set for a nonexistent key. should pass.
274    check(!get_meta(h, h1, key2), "Expected get meta to return false");
275    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, last_status.load(), "Expected enoent");
276    checkeq(ENGINE_SUCCESS,
277            store(h, h1, NULL, OPERATION_SET, key2, "someothervalue", &i),
278            "Failed set.");
279    // check the stat again
280    checkeq(3, get_int_stat(h, h1, "ep_num_ops_get_meta"),
281            "Expected one extra getMeta ops");
282
283    h1->release(h, NULL, i);
284    return SUCCESS;
285}
286
287static enum test_result test_get_meta_with_delete(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)
288{
289    char const *key1 = "key1";
290    char const *key2 = "key2";
291
292    item *i = NULL;
293
294    // test get_meta followed by delete for an existing key. should pass.
295    checkeq(ENGINE_SUCCESS,
296            store(h, h1, NULL, OPERATION_SET, key1, "somevalue", &i),
297            "Failed set.");
298    h1->release(h, NULL, i);
299    wait_for_flusher_to_settle(h, h1);
300    // check the stat
301    int temp = get_int_stat(h, h1, "ep_num_ops_get_meta");
302    check(temp == 0, "Expect zero getMeta ops");
303    check(get_meta(h, h1, key1), "Expected to get meta");
304    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
305    checkeq(ENGINE_SUCCESS, del(h, h1, key1, 0, 0), "Delete failed");
306    // check the stat
307    temp = get_int_stat(h, h1, "ep_num_ops_get_meta");
308    check(temp == 1, "Expect one getMeta op");
309
310    // test get_meta followed by delete for a deleted key. should fail.
311    wait_for_flusher_to_settle(h, h1);
312    check(get_meta(h, h1, key1), "Expected to get meta");
313    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
314    check(last_deleted_flag, "Expected deleted flag to be set");
315    checkeq(ENGINE_KEY_ENOENT, del(h, h1, key1, 0, 0), "Expected enoent");
316    // check the stat
317    temp = get_int_stat(h, h1, "ep_num_ops_get_meta");
318    checkeq(2, temp, "Expect more getMeta op");
319
320    // test get_meta followed by delete for a nonexistent key. should fail.
321    check(!get_meta(h, h1, key2), "Expected get meta to return false");
322    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, last_status.load(), "Expected enoent");
323    checkeq(ENGINE_KEY_ENOENT, del(h, h1, key2, 0, 0), "Expected enoent");
324    // check the stat again
325    temp = get_int_stat(h, h1, "ep_num_ops_get_meta");
326    checkeq(3, temp, "Expected one extra getMeta ops");
327
328    return SUCCESS;
329}
330
331static enum test_result test_add_with_meta(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)
332{
333    const char *key = "mykey";
334    const size_t keylen = strlen(key);
335    ItemMetaData itemMeta;
336    size_t temp = 0;
337
338    // put some random metadata
339    itemMeta.revSeqno = 10;
340    itemMeta.cas = 0xdeadbeef;
341    itemMeta.exptime = 0;
342    itemMeta.flags = 0xdeadbeef;
343    // check the stat
344    temp = get_int_stat(h, h1, "ep_num_ops_set_meta");
345    check(temp == 0, "Expect zero setMeta ops");
346
347    // store an item with meta data
348    add_with_meta(h, h1, key, keylen, NULL, 0, 0, &itemMeta);
349    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
350
351    // store the item again, expect key exists
352    add_with_meta(h, h1, key, keylen, NULL, 0, 0, &itemMeta, true);
353    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, last_status.load(),
354            "Expected add to fail when the item exists already");
355    // check the stat
356    temp = get_int_stat(h, h1, "ep_num_ops_set_meta");
357    check(temp == 1, "Failed op does not count");
358
359    return SUCCESS;
360}
361
362static enum test_result test_delete_with_meta(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
363
364    const char *key1 = "delete_with_meta_key1";
365    const char *key2 = "delete_with_meta_key2";
366    const char *key3 = "delete_with_meta_key3";
367    const size_t keylen = strlen(key1);
368    ItemMetaData itemMeta;
369    uint64_t vb_uuid;
370    uint32_t high_seqno;
371    // check the stat
372    size_t temp = get_int_stat(h, h1, "ep_num_ops_del_meta");
373    check(temp == 0, "Expect zero setMeta ops");
374
375    // put some random meta data
376    itemMeta.revSeqno = 10;
377    itemMeta.cas = 0xdeadbeef;
378    itemMeta.exptime = 0;
379    itemMeta.flags = 0xdeadbeef;
380
381    // store an item
382    item *i = NULL;
383    checkeq(ENGINE_SUCCESS,
384            store(h, h1, NULL, OPERATION_SET, key1,
385                  "somevalue", &i),
386            "Failed set.");
387    h1->release(h, NULL, i);
388
389    checkeq(ENGINE_SUCCESS,
390            store(h, h1, NULL, OPERATION_SET, key2,
391                  "somevalue2", &i),
392            "Failed set.");
393    h1->release(h, NULL, i);
394
395    checkeq(ENGINE_SUCCESS,
396            store(h, h1, NULL, OPERATION_SET, key3,
397                  "somevalue3", &i), "Failed set.");
398    h1->release(h, NULL, i);
399
400    vb_uuid = get_ull_stat(h, h1, "vb_0:0:id", "failovers");
401    high_seqno = get_ull_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno");
402
403    const void *cookie = testHarness.create_cookie();
404
405    // delete an item with meta data
406    del_with_meta(h, h1, key1, keylen, 0, &itemMeta, 0/*cas*/, 0/*options*/, cookie);
407
408    check(last_uuid == vb_uuid, "Expected valid vbucket uuid");
409    check(last_seqno == high_seqno + 1, "Expected valid sequence number");
410    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
411    // check the stat
412    temp = get_int_stat(h, h1, "ep_num_ops_del_meta");
413    check(temp == 1, "Expect more setMeta ops");
414
415    testHarness.set_mutation_extras_handling(cookie, false);
416
417    // delete an item with meta data
418    del_with_meta(h, h1, key2, keylen, 0, &itemMeta, 0/*cas*/, 0/*options*/, cookie);
419
420    check(last_uuid == vb_uuid, "Expected same vbucket uuid");
421    check(last_seqno == high_seqno + 1, "Expected same sequence number");
422    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
423
424    // delete an item with meta data
425    del_with_meta(h, h1, key3, keylen, 0, &itemMeta);
426
427    check(last_uuid == vb_uuid, "Expected valid vbucket uuid");
428    check(last_seqno == high_seqno + 3, "Expected valid sequence number");
429    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
430
431    testHarness.destroy_cookie(cookie);
432    return SUCCESS;
433}
434
435static enum test_result test_delete_with_meta_deleted(ENGINE_HANDLE *h,
436                                                      ENGINE_HANDLE_V1 *h1) {
437    const char *key = "delete_with_meta_key";
438    const size_t keylen = strlen(key);
439    item *i = NULL;
440
441    // check the stat
442    checkeq(0, get_int_stat(h, h1, "ep_num_ops_del_meta"),
443            "Expect zero setMeta ops");
444
445    // add a key
446    checkeq(ENGINE_SUCCESS,
447            store(h, h1, NULL, OPERATION_SET, key, "somevalue", &i),
448            "Failed set.");
449    wait_for_flusher_to_settle(h, h1);
450
451    // delete the key
452    checkeq(ENGINE_SUCCESS, del(h, h1, key, 0, 0),
453            "Delete failed");
454    wait_for_flusher_to_settle(h, h1);
455    wait_for_stat_to_be(h, h1, "curr_items", 0);
456
457    // get metadata of deleted key
458    check(get_meta(h, h1, key), "Expected to get meta");
459    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
460            "Expected success");
461    check(last_deleted_flag, "Expected deleted flag to be set");
462    checkeq(0, get_int_stat(h, h1, "curr_items"), "Expected zero curr_items");
463    checkeq(1,get_int_stat(h, h1, "curr_temp_items"), "Expected single temp_items");
464
465    // this is the cas to be used with a subsequent delete with meta
466    uint64_t valid_cas = last_cas;
467    uint64_t invalid_cas = 2012;
468    // put some random metadata and delete the item with new meta data
469    ItemMetaData itm_meta;
470    itm_meta.revSeqno = 10;
471    itm_meta.cas = 0xdeadbeef;
472    itm_meta.exptime = 1735689600; // expires in 2025
473    itm_meta.flags = 0xdeadbeef;
474
475    // do delete with meta with an incorrect cas value. should fail.
476    del_with_meta(h, h1, key, keylen, 0, &itm_meta, invalid_cas);
477    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, last_status.load(),
478            "Expected invalid cas error");
479    checkeq(0, get_int_stat(h, h1, "ep_num_ops_del_meta"), "Faild ops does not count");
480    checkeq(0, get_int_stat(h, h1, "curr_items"), "Expected zero curr_items");
481    checkeq(1,get_int_stat(h, h1, "curr_temp_items"), "Expected single temp_items");
482
483    // do delete with meta with the correct cas value. should pass.
484    del_with_meta(h, h1, key, keylen, 0, &itm_meta, valid_cas);
485    wait_for_flusher_to_settle(h, h1);
486
487    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
488    checkeq(1, get_int_stat(h, h1, "ep_num_ops_del_meta"), "Expect some ops");
489    wait_for_stat_to_be(h, h1, "curr_items", 0);
490    checkeq(0, get_int_stat(h, h1, "curr_temp_items"), "Expected zero temp_items");
491
492    // get metadata again to verify that delete with meta was successful
493    check(get_meta(h, h1, key), "Expected to get meta");
494    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
495    check(last_deleted_flag, "Expected deleted flag to be set");
496    check(itm_meta.revSeqno == last_meta.revSeqno, "Expected seqno to match");
497    check(itm_meta.cas == last_meta.cas, "Expected cas to match");
498    check(itm_meta.flags == last_meta.flags, "Expected flags to match");
499    checkeq(0, get_int_stat(h, h1, "curr_items"), "Expected zero curr_items");
500    checkeq(1, get_int_stat(h, h1, "curr_temp_items"), "Expected single temp_items");
501
502    h1->release(h, NULL, i);
503    return SUCCESS;
504}
505
506static enum test_result test_delete_with_meta_nonexistent(ENGINE_HANDLE *h,
507                                                          ENGINE_HANDLE_V1 *h1) {
508    const char *key = "delete_with_meta_key";
509    const size_t keylen = strlen(key);
510    ItemMetaData itm_meta;
511
512    // check the stat
513    checkeq(0, get_int_stat(h, h1, "ep_num_ops_del_meta"),
514            "Expect zero setMeta ops");
515
516    // get metadata of nonexistent key
517    check(!get_meta(h, h1, key), "Expected get meta to return false");
518    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, last_status.load(),
519            "Expected enoent");
520    checkeq(0, get_int_stat(h, h1, "curr_items"), "Expected zero curr_items");
521
522    // this is the cas to be used with a subsequent delete with meta
523    uint64_t valid_cas = last_cas;
524    uint64_t invalid_cas = 2012;
525
526    // do delete with meta
527    // put some random metadata and delete the item with new meta data
528    itm_meta.revSeqno = 10;
529    itm_meta.cas = 0xdeadbeef;
530    itm_meta.exptime = 1735689600; // expires in 2025
531    itm_meta.flags = 0xdeadbeef;
532
533    // do delete with meta with an incorrect cas value. should fail.
534    del_with_meta(h, h1, key, keylen, 0, &itm_meta, invalid_cas);
535    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, last_status.load(),
536            "Expected invalid cas error");
537    // check the stat
538    checkeq(0, get_int_stat(h, h1, "ep_num_ops_del_meta"), "Failed op does not count");
539    checkeq(0, get_int_stat(h, h1, "curr_items"), "Expected zero curr_items");
540    checkeq(1, get_int_stat(h, h1, "curr_temp_items"), "Expected single temp_items");
541
542    // do delete with meta with the correct cas value. should pass.
543    del_with_meta(h, h1, key, keylen, 0, &itm_meta, valid_cas);
544    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
545    wait_for_flusher_to_settle(h, h1);
546
547    // check the stat
548    checkeq(1, get_int_stat(h, h1, "ep_num_ops_del_meta"), "Expect one op");
549    wait_for_stat_to_be(h, h1, "curr_items", 0);
550    checkeq(0, get_int_stat(h, h1, "curr_temp_items"), "Expected zero temp_items");
551
552    // get metadata again to verify that delete with meta was successful
553    check(get_meta(h, h1, key), "Expected to get meta");
554    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
555    check(last_deleted_flag, "Expected deleted flag to be set");
556    check(itm_meta.revSeqno == last_meta.revSeqno, "Expected seqno to match");
557    check(itm_meta.cas == last_meta.cas, "Expected cas to match");
558    check(itm_meta.flags == last_meta.flags, "Expected flags to match");
559    checkeq(0, get_int_stat(h, h1, "curr_items"), "Expected zero curr_items");
560    checkeq(1, get_int_stat(h, h1, "curr_temp_items"), "Expected single temp_items");
561
562    return SUCCESS;
563}
564
565static enum test_result test_delete_with_meta_nonexistent_no_temp(ENGINE_HANDLE *h,
566                                                                  ENGINE_HANDLE_V1 *h1) {
567    const char *key1 = "delete_with_meta_no_temp_key1";
568    const size_t keylen1 = strlen(key1);
569    ItemMetaData itm_meta1;
570
571    // Run compaction to start using the bloomfilter
572    useconds_t sleepTime = 128;
573    compact_db(h, h1, 0, 1, 1, 0);
574    while (get_int_stat(h, h1, "ep_pending_compactions") != 0) {
575        decayingSleep(&sleepTime);
576    }
577
578    // put some random metadata and delete the item with new meta data
579    itm_meta1.revSeqno = 10;
580    itm_meta1.cas = 0xdeadbeef;
581    itm_meta1.exptime = 1735689600; // expires in 2025
582    itm_meta1.flags = 0xdeadbeef;
583
584    // do delete with meta with the correct cas value.
585    // skipConflictResolution false
586    del_with_meta(h, h1, key1, keylen1, 0, &itm_meta1, 0, false);
587    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
588    wait_for_flusher_to_settle(h, h1);
589
590    checkeq(1, get_int_stat(h, h1, "ep_num_ops_del_meta"), "Expect one op");
591    wait_for_stat_to_be(h, h1, "curr_items", 0);
592    checkeq(0, get_int_stat(h, h1, "curr_temp_items"), "Expected zero temp_items");
593
594    // do delete with meta with the correct cas value.
595    // skipConflictResolution true
596    const char *key2 = "delete_with_meta_no_temp_key2";
597    const size_t keylen2 = strlen(key2);
598    ItemMetaData itm_meta2;
599
600    // put some random metadata and delete the item with new meta data
601    itm_meta2.revSeqno = 10;
602    itm_meta2.cas = 0xdeadbeef;
603    itm_meta2.exptime = 1735689600; // expires in 2025
604    itm_meta2.flags = 0xdeadbeef;
605
606    del_with_meta(h, h1, key2, keylen2, 0, &itm_meta2, 0, true);
607    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
608    wait_for_flusher_to_settle(h, h1);
609
610    checkeq(2, get_int_stat(h, h1, "ep_num_ops_del_meta"), "Expect one op");
611    wait_for_stat_to_be(h, h1, "curr_items", 0);
612    checkeq(0, get_int_stat(h, h1, "curr_temp_items"), "Expected zero temp_items");
613
614    return SUCCESS;
615}
616
617static enum test_result test_delete_with_meta_race_with_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)
618{
619    char const *key1 = "key1";
620    const size_t keylen1 = strlen(key1);
621
622    item *i = NULL;
623    ItemMetaData itm_meta;
624    itm_meta.revSeqno = 10;
625    itm_meta.cas = 0xdeadbeef;
626    itm_meta.exptime = 1735689600; // expires in 2025
627    itm_meta.flags = 0xdeadbeef;
628    // check the stat
629    size_t temp = get_int_stat(h, h1, "ep_num_ops_del_meta");
630    check(temp == 0, "Expect zero ops");
631
632    //
633    // test race with a concurrent set for an existing key. should fail.
634    //
635
636    // create a new key and do get_meta
637    checkeq(ENGINE_SUCCESS,
638            store(h, h1, NULL, OPERATION_SET, key1, "somevalue", &i),
639            "Failed set.");
640    h1->release(h, NULL, i);
641    wait_for_flusher_to_settle(h, h1);
642    check(get_meta(h, h1, key1), "Expected to get meta");
643    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
644
645    // do a concurrent set that changes the cas
646    checkeq(ENGINE_SUCCESS,
647            store(h, h1, NULL, OPERATION_SET, key1, "someothervalue", &i),
648            "Failed set.");
649    h1->release(h, NULL, i);
650
651    // attempt delete_with_meta. should fail since cas is no longer valid.
652    del_with_meta(h, h1, key1, keylen1, 0, &itm_meta, last_cas);
653    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, last_status.load(),
654          "Expected invalid cas error");
655    // check the stat
656    temp = get_int_stat(h, h1, "ep_num_ops_del_meta");
657    check(temp == 0, "Failed op does not count");
658
659    //
660    // test race with a concurrent set for a deleted key. should fail.
661    //
662
663    // do get_meta for the deleted key
664    checkeq(ENGINE_SUCCESS, del(h, h1, key1, 0, 0), "Delete failed");
665    wait_for_flusher_to_settle(h, h1);
666
667    check(get_meta(h, h1, key1), "Expected to get meta");
668    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
669    check(last_deleted_flag, "Expected deleted flag to be set");
670
671    // do a concurrent set that changes the cas
672    checkeq(ENGINE_SUCCESS,
673            store(h, h1, NULL, OPERATION_SET, key1, "someothervalue", &i),
674            "Failed set.");
675    h1->release(h, NULL, i);
676    del_with_meta(h, h1, key1, keylen1, 0, &itm_meta, last_cas);
677    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, last_status.load(),
678          "Expected invalid cas error");
679    // check the stat
680    temp = get_int_stat(h, h1, "ep_num_ops_del_meta");
681    check(temp == 0, "Failed op does not count");
682
683    return SUCCESS;
684}
685
686static enum test_result test_delete_with_meta_race_with_delete(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)
687{
688    char const *key1 = "key1";
689    uint16_t keylen1 = (uint16_t)strlen(key1);
690    char const *key2 = "key2";
691    uint16_t keylen2 = (uint16_t)strlen(key2);
692    item *i = NULL;
693    ItemMetaData itm_meta;
694    itm_meta.cas = 0x1;
695    // check the stat
696    size_t temp = get_int_stat(h, h1, "ep_num_ops_del_meta");
697    check(temp == 0, "Expect zero ops");
698
699    //
700    // test race with a concurrent delete for an existing key. should fail.
701    //
702
703    // create a new key and do get_meta
704    checkeq(ENGINE_SUCCESS,
705            store(h, h1, NULL, OPERATION_SET, key1, "somevalue", &i),
706            "Failed set.");
707    wait_for_flusher_to_settle(h, h1);
708    check(get_meta(h, h1, key1), "Expected to get meta");
709    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
710
711    // do a concurrent delete
712    checkeq(ENGINE_SUCCESS, del(h, h1, key1, 0, 0), "Delete failed");
713
714    // attempt delete_with_meta. should fail since cas is no longer valid.
715    del_with_meta(h, h1, key1, keylen1, 0, &itm_meta, last_cas, true);
716    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, last_status.load(),
717          "Expected invalid cas error");
718    // check the stat
719    temp = get_int_stat(h, h1, "ep_num_ops_del_meta");
720    check(temp == 0, "Failed op does not count");
721
722    //
723    // test race with a concurrent delete for a deleted key. should pass since
724    // the delete itself will fail.
725    //
726
727    // do get_meta for the deleted key
728    wait_for_flusher_to_settle(h, h1);
729    check(get_meta(h, h1, key1), "Expected to get meta");
730    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
731    check(last_deleted_flag, "Expected deleted flag to be set");
732
733    // do a concurrent delete
734    checkeq(ENGINE_KEY_ENOENT, del(h, h1, key1, 0, 0), "Delete failed");
735
736    // attempt delete_with_meta. should pass.
737    del_with_meta(h, h1, key1, keylen1, 0, &itm_meta, last_cas, true);
738    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
739          "Expected delete_with_meta success");
740    // check the stat
741    temp = get_int_stat(h, h1, "ep_num_ops_del_meta");
742    check(temp == 1, "Expect some ops");
743
744    //
745    // test race with a concurrent delete for a nonexistent key. should pass
746    // since the delete itself will fail.
747    //
748
749    // do get_meta for a nonexisting key
750    check(!get_meta(h, h1, key2), "Expected get meta to return false");
751    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, last_status.load(), "Expected enoent");
752
753    // do a concurrent delete
754    checkeq(ENGINE_KEY_ENOENT, del(h, h1, key1, 0, 0), "Delete failed");
755
756    // attempt delete_with_meta. should pass.
757    del_with_meta(h, h1, key2, keylen2, 0, &itm_meta, last_cas, true);
758    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
759          "Expected delete_with_meta success");
760    // check the stat
761    temp = get_int_stat(h, h1, "ep_num_ops_del_meta");
762    check(temp == 2, "Expect some ops");
763
764    h1->release(h, NULL, i);
765    return SUCCESS;
766}
767
768static enum test_result test_set_with_meta(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
769    const char* key = "set_with_meta_key";
770    size_t keylen = strlen(key);
771    const char* val = "somevalue";
772    const char* newVal = "someothervalue";
773    size_t newValLen = strlen(newVal);
774    uint64_t vb_uuid;
775    uint32_t high_seqno;
776
777    // check the stat
778    checkeq(0, get_int_stat(h, h1, "ep_num_ops_set_meta"), "Expect zero ops");
779    checkeq(0, get_int_stat(h, h1, "ep_num_ops_get_meta_on_set_meta"),
780            "Expect zero ops");
781    checkeq(0, get_int_stat(h, h1, "curr_items"), "Expect zero items");
782    checkeq(0, get_int_stat(h, h1, "curr_temp_items"), "Expect zero temp items");
783
784    // create a new key
785    item *i = NULL;
786    checkeq(ENGINE_SUCCESS,
787            store(h, h1, NULL, OPERATION_SET, key, val, &i),
788            "Failed set.");
789
790    // get metadata for the key
791    check(get_meta(h, h1, key), "Expected to get meta");
792    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
793    checkeq(1, get_int_stat(h, h1, "curr_items"), "Expect one item");
794    checkeq(0, get_int_stat(h, h1, "curr_temp_items"), "Expect zero temp item");
795
796    // this is the cas to be used with a subsequent set with meta
797    uint64_t cas_for_set = last_cas;
798    // init some random metadata
799    ItemMetaData itm_meta;
800    itm_meta.revSeqno = 10;
801    itm_meta.cas = 0xdeadbeef;
802    itm_meta.exptime = time(NULL) + 300;
803    itm_meta.flags = 0xdeadbeef;
804
805    char *bigValue = new char[32*1024*1024];
806    // do set with meta with the value size bigger than the max size allowed.
807    set_with_meta(h, h1, key, keylen, bigValue, 32*1024*1024, 0, &itm_meta, cas_for_set);
808    checkeq(PROTOCOL_BINARY_RESPONSE_E2BIG, last_status.load(),
809          "Expected the max value size exceeding error");
810    delete []bigValue;
811
812    // do set with meta with an incorrect cas value. should fail.
813    set_with_meta(h, h1, key, keylen, newVal, newValLen, 0, &itm_meta, 1229);
814    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, last_status.load(),
815          "Expected invalid cas error");
816    // check the stat
817    checkeq(0, get_int_stat(h, h1, "ep_num_ops_set_meta"), "Failed op does not count");
818
819    vb_uuid = get_ull_stat(h, h1, "vb_0:0:id", "failovers");
820    high_seqno = get_ull_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno");
821
822    const void *cookie = testHarness.create_cookie();
823
824    // do set with meta with the correct cas value. should pass.
825    set_with_meta(h, h1, key, keylen, newVal, newValLen, 0, &itm_meta, cas_for_set,
826                  0, 0, cookie);
827    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
828    check(last_uuid == vb_uuid, "Expected valid vbucket uuid");
829    check(last_seqno == high_seqno + 1, "Expected valid sequence number");
830
831    // check the stat
832    checkeq(1, get_int_stat(h, h1, "ep_num_ops_set_meta"), "Expect some ops");
833    checkeq(1, get_int_stat(h, h1, "curr_items"), "Expect one item");
834    checkeq(0, get_int_stat(h, h1, "curr_temp_items"), "Expect zero temp item");
835
836    // get metadata again to verify that set with meta was successful
837    check(get_meta(h, h1, key), "Expected to get meta");
838    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
839    check(last_meta.revSeqno == 10, "Expected seqno to match");
840    check(last_meta.cas == 0xdeadbeef, "Expected cas to match");
841    check(last_meta.flags == 0xdeadbeef, "Expected flags to match");
842
843    //disable getting vb uuid and seqno as extras
844    testHarness.set_mutation_extras_handling(cookie, false);
845    itm_meta.revSeqno++;
846    cas_for_set = last_meta.cas;
847    set_with_meta(h, h1, key, keylen, newVal, newValLen, 0, &itm_meta, cas_for_set,
848                  false, 0, cookie);
849    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
850    check(last_uuid == vb_uuid, "Expected same vbucket uuid");
851    check(last_seqno == high_seqno + 1, "Expected same sequence number");
852
853    itm_meta.revSeqno++;
854    cas_for_set = last_meta.cas;
855    set_with_meta(h, h1, key, keylen, newVal, newValLen, 0, &itm_meta, cas_for_set);
856    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
857    check(last_uuid == vb_uuid, "Expected valid vbucket uuid");
858    check(last_seqno == high_seqno + 3, "Expected valid sequence number");
859
860    // Make sure the item expiration was processed correctly
861    testHarness.time_travel(301);
862    checkeq(ENGINE_KEY_ENOENT, h1->get(h, NULL, &i, key, keylen, 0),
863            "Failed to get value.");
864
865    h1->release(h, NULL, i);
866    testHarness.destroy_cookie(cookie);
867    return SUCCESS;
868}
869
870static enum test_result test_set_with_meta_by_force(ENGINE_HANDLE *h,
871                                                    ENGINE_HANDLE_V1 *h1) {
872    const char* key = "set_with_meta_key";
873    size_t keylen = strlen(key);
874    const char* val = "somevalue";
875
876    // init some random metadata
877    ItemMetaData itm_meta;
878    itm_meta.revSeqno = 10;
879    itm_meta.cas = 0xdeadbeef;
880    itm_meta.exptime = time(NULL) + 300;
881    itm_meta.flags = 0xdeadbeef;
882
883    // Pass true to force SetWithMeta.
884    set_with_meta(h, h1, key, keylen, val, strlen(val), 0, &itm_meta,
885                  0, true);
886    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
887    wait_for_flusher_to_settle(h, h1);
888
889    // get metadata again to verify that the warmup loads an item correctly.
890    check(get_meta(h, h1, key), "Expected to get meta");
891    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
892    check(last_meta.revSeqno == 10, "Expected seqno to match");
893    check(last_meta.cas == 0xdeadbeef, "Expected cas to match");
894    check(last_meta.flags == 0xdeadbeef, "Expected flags to match");
895
896    check_key_value(h, h1, key, val, strlen(val));
897
898    return SUCCESS;
899}
900
901static enum test_result test_set_with_meta_deleted(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
902    const char* key = "set_with_meta_key";
903    size_t keylen = strlen(key);
904    const char* val = "somevalue";
905    const char* newVal = "someothervalue";
906    uint16_t newValLen = (uint16_t)strlen(newVal);
907
908    // check the stat
909    checkeq(0, get_int_stat(h, h1, "ep_num_ops_set_meta"), "Expect zero ops");
910    checkeq(0, get_int_stat(h, h1, "ep_num_ops_get_meta_on_set_meta"),
911            "Expect zero ops");
912
913    // create a new key
914    item *i = NULL;
915    checkeq(ENGINE_SUCCESS,
916            store(h, h1, NULL, OPERATION_SET, key, val, &i),
917            "Failed set.");
918    wait_for_flusher_to_settle(h, h1);
919    checkeq(1, get_int_stat(h, h1, "curr_items"), "Expected single curr_items");
920    checkeq(0, get_int_stat(h, h1, "curr_temp_items"), "Expected zero temp_items");
921
922    // delete the key
923    checkeq(ENGINE_SUCCESS, del(h, h1, key, 0, 0), "Delete failed");
924    wait_for_flusher_to_settle(h, h1);
925    wait_for_stat_to_be(h, h1, "curr_items", 0);
926
927    // get metadata for the key
928    check(get_meta(h, h1, key), "Expected to get meta");
929    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
930    check(last_deleted_flag, "Expected deleted flag to be set");
931    checkeq(0, get_int_stat(h, h1, "curr_items"), "Expected zero curr_items");
932    checkeq(1, get_int_stat(h, h1, "curr_temp_items"), "Expected single temp_items");
933
934    // this is the cas to be used with a subsequent set with meta
935    uint64_t cas_for_set = last_cas;
936    // init some random metadata
937    ItemMetaData itm_meta;
938    itm_meta.revSeqno = 10;
939    itm_meta.cas = 0xdeadbeef;
940    itm_meta.exptime = 1735689600; // expires in 2025
941    itm_meta.flags = 0xdeadbeef;
942
943    // do set_with_meta with an incorrect cas for a deleted item. should fail.
944    set_with_meta(h, h1, key, keylen, newVal, newValLen, 0, &itm_meta, 1229);
945    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, last_status.load(),
946          "Expected key_not_found error");
947    // check the stat
948    checkeq(0, get_int_stat(h, h1, "ep_num_ops_set_meta"), "Failed op does not count");
949    checkeq(0, get_int_stat(h, h1, "curr_items"), "Expected zero curr_items");
950    checkeq(1, get_int_stat(h, h1, "curr_temp_items"), "Expected single temp_items");
951
952    // do set with meta with the correct cas value. should pass.
953    set_with_meta(h, h1, key, keylen, newVal, newValLen, 0, &itm_meta, cas_for_set);
954    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
955    // check the stat
956    checkeq(1, get_int_stat(h, h1, "ep_num_ops_set_meta"), "Expect some ops");
957    checkeq(0, get_int_stat(h, h1, "ep_num_ops_get_meta_on_set_meta"),
958            "Expect some ops");
959    checkeq(1, get_int_stat(h, h1, "curr_items"), "Expected single curr_items");
960    checkeq(0, get_int_stat(h, h1, "curr_temp_items"), "Expected zero temp_items");
961
962    // get metadata again to verify that set with meta was successful
963    check(get_meta(h, h1, key), "Expected to get meta");
964    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
965    ItemMetaData metadata(0xdeadbeef, 10, 0xdeadbeef, 1735689600);
966    verifyLastMetaData(metadata);
967    checkeq(1, get_int_stat(h, h1, "curr_items"), "Expected single curr_items");
968    checkeq(0, get_int_stat(h, h1, "curr_temp_items"), "Expected zero temp_items");
969
970    h1->release(h, NULL, i);
971    return SUCCESS;
972}
973
974static enum test_result test_set_with_meta_nonexistent(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
975    const char* key = "set_with_meta_key";
976    size_t keylen = strlen(key);
977    const char* val = "somevalue";
978    size_t valLen = strlen(val);
979
980    // check the stat
981    checkeq(0, get_int_stat(h, h1, "ep_num_ops_set_meta"), "Expect zero ops");
982
983    // get metadata for the key
984    check(!get_meta(h, h1, key), "Expected get meta to return false");
985    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, last_status.load(), "Expected enoent");
986    checkeq(0, get_int_stat(h, h1, "curr_items"), "Expected zero curr_items");
987
988    // this is the cas to be used with a subsequent set with meta
989    uint64_t cas_for_set = last_cas;
990    // init some random metadata
991    ItemMetaData itm_meta;
992    itm_meta.revSeqno = 10;
993    itm_meta.cas = 0xdeadbeef;
994    itm_meta.exptime = 1735689600; // expires in 2025
995    itm_meta.flags = 0xdeadbeef;
996
997    // do set_with_meta with an incorrect cas for a non-existent item. should fail.
998    set_with_meta(h, h1, key, keylen, val, valLen, 0, &itm_meta, 1229);
999    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, last_status.load(),
1000          "Expected key_not_found error");
1001    // check the stat
1002    checkeq(0, get_int_stat(h, h1, "ep_num_ops_set_meta"), "Failed op does not count");
1003    checkeq(0, get_int_stat(h, h1, "curr_items"), "Expected zero curr_items");
1004
1005    // do set with meta with the correct cas value. should pass.
1006    set_with_meta(h, h1, key, keylen, val, valLen, 0, &itm_meta, cas_for_set);
1007    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
1008    // check the stat
1009    checkeq(1, get_int_stat(h, h1, "ep_num_ops_set_meta"), "Expect some ops");
1010    checkeq(1, get_int_stat(h, h1, "curr_items"), "Expected single curr_items");
1011    checkeq(0, get_int_stat(h, h1, "curr_temp_items"), "Expected zero temp_items");
1012
1013    // get metadata again to verify that set with meta was successful
1014    check(get_meta(h, h1, key), "Expected to get meta");
1015    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
1016    ItemMetaData metadata(0xdeadbeef, 10, 0xdeadbeef, 1735689600);
1017    verifyLastMetaData(metadata);
1018    checkeq(0, get_int_stat(h, h1, "curr_temp_items"), "Expected zero temp_items");
1019    checkeq(1, get_int_stat(h, h1, "curr_items"), "Expected single curr_items");
1020
1021    return SUCCESS;
1022}
1023
1024static enum test_result test_set_with_meta_race_with_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)
1025{
1026    char const *key1 = "key1";
1027    size_t keylen1 = strlen(key1);
1028    item *i = NULL;
1029    // check the stat
1030    size_t temp = get_int_stat(h, h1, "ep_num_ops_set_meta");
1031    check(temp == 0, "Expect zero ops");
1032
1033    //
1034    // test race with a concurrent set for an existing key. should fail.
1035    //
1036
1037    // create a new key and do get_meta
1038    checkeq(ENGINE_SUCCESS,
1039            store(h, h1, NULL, OPERATION_SET, key1, "somevalue", &i),
1040            "Failed set.");
1041    h1->release(h, NULL, i);
1042    wait_for_flusher_to_settle(h, h1);
1043    check(get_meta(h, h1, key1), "Expected to get meta");
1044    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
1045
1046    // do a concurrent set that changes the cas
1047    checkeq(ENGINE_SUCCESS,
1048            store(h, h1, NULL, OPERATION_SET, key1, "someothervalue", &i),
1049            "Failed set.");
1050    h1->release(h, NULL, i);
1051
1052    // attempt set_with_meta. should fail since cas is no longer valid.
1053    last_meta.revSeqno += 2;
1054    set_with_meta(h, h1, key1, keylen1, NULL, 0, 0, &last_meta, last_cas);
1055    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, last_status.load(),
1056          "Expected invalid cas error");
1057    // check the stat
1058    temp = get_int_stat(h, h1, "ep_num_ops_set_meta");
1059    check(temp == 0, "Failed op does not count");
1060
1061    //
1062    // test race with a concurrent set for a deleted key. should fail.
1063    //
1064
1065    // do get_meta for the deleted key
1066    checkeq(ENGINE_SUCCESS, del(h, h1, key1, 0, 0), "Delete failed");
1067    wait_for_flusher_to_settle(h, h1);
1068    check(get_meta(h, h1, key1), "Expected to get meta");
1069    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
1070    check(last_deleted_flag, "Expected deleted flag to be set");
1071
1072    // do a concurrent set that changes the cas
1073    checkeq(ENGINE_SUCCESS,
1074            store(h, h1, NULL, OPERATION_SET, key1, "someothervalue", &i),
1075            "Failed set.");
1076    h1->release(h, NULL, i);
1077
1078    // attempt set_with_meta. should fail since cas is no longer valid.
1079    last_meta.revSeqno += 2;
1080    set_with_meta(h, h1, key1, keylen1, NULL, 0, 0, &last_meta, last_cas);
1081    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, last_status.load(),
1082          "Expected invalid cas error");
1083    // check the stat
1084    temp = get_int_stat(h, h1, "ep_num_ops_set_meta");
1085    check(temp == 0, "Failed op does not count");
1086
1087    return SUCCESS;
1088}
1089
1090static enum test_result test_set_with_meta_race_with_delete(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)
1091{
1092    char const *key1 = "key1";
1093    size_t keylen1 = strlen(key1);
1094    char const *key2 = "key2";
1095    size_t keylen2 = strlen(key2);
1096    item *i = NULL;
1097    // check the stat
1098    size_t temp = get_int_stat(h, h1, "ep_num_ops_set_meta");
1099    check(temp == 0, "Expect zero op");
1100
1101    //
1102    // test race with a concurrent delete for an existing key. should fail.
1103    //
1104
1105    // create a new key and do get_meta
1106    checkeq(ENGINE_SUCCESS,
1107            store(h, h1, NULL, OPERATION_SET, key1, "somevalue", &i),
1108            "Failed set.");
1109    wait_for_flusher_to_settle(h, h1);
1110    check(get_meta(h, h1, key1), "Expected to get meta");
1111    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
1112
1113    // do a concurrent delete that changes the cas
1114    checkeq(ENGINE_SUCCESS, del(h, h1, key1, 0, 0), "Delete failed");
1115
1116    // attempt set_with_meta. should fail since cas is no longer valid.
1117    set_with_meta(h, h1, key1, keylen1, NULL, 0, 0, &last_meta, last_cas, true);
1118
1119    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, last_status.load(),
1120            (std::string{"Expected invalid cas error (KEY_EXISTS or"
1121                         " KEY_ENOENT), got: "} +
1122             std::to_string(last_status.load())).c_str());
1123
1124    // check the stat
1125    temp = get_int_stat(h, h1, "ep_num_ops_set_meta");
1126    check(temp == 0, "Expect zero op");
1127
1128    //
1129    // test race with a concurrent delete for a deleted key. should pass since
1130    // the delete will fail.
1131    //
1132
1133    // do get_meta for the deleted key
1134    wait_for_flusher_to_settle(h, h1);
1135    check(get_meta(h, h1, key1), "Expected to get meta");
1136    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
1137    check(last_deleted_flag, "Expected deleted flag to be set");
1138
1139    // do a concurrent delete. should fail.
1140    checkeq(ENGINE_KEY_ENOENT, del(h, h1, key1, 0, 0), "Delete failed");
1141
1142    // attempt set_with_meta. should pass since cas is still valid.
1143    set_with_meta(h, h1, key1, keylen1, NULL, 0, 0, &last_meta, last_cas, true);
1144    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
1145    // check the stat
1146    temp = get_int_stat(h, h1, "ep_num_ops_set_meta");
1147    check(temp == 1, "Expect some op");
1148
1149    //
1150    // test race with a concurrent delete for a nonexistent key. should pass
1151    // since the delete will fail.
1152    //
1153
1154    // do get_meta for a nonexisting key
1155    check(!get_meta(h, h1, key2), "Expected get meta to return false");
1156    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, last_status.load(), "Expected enoent");
1157
1158    // do a concurrent delete. should fail.
1159    checkeq(ENGINE_KEY_ENOENT, del(h, h1, key2, 0, 0), "Delete failed");
1160
1161    // attempt set_with_meta. should pass since cas is still valid.
1162    set_with_meta(h, h1, key2, keylen2, NULL, 0, 0, &last_meta, last_cas, true);
1163    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
1164    // check the stat
1165    temp = get_int_stat(h, h1, "ep_num_ops_set_meta");
1166    check(temp == 2, "Expect some ops");
1167
1168    h1->release(h, NULL, i);
1169    return SUCCESS;
1170}
1171
1172static enum test_result test_exp_persisted_set_del(ENGINE_HANDLE *h,
1173                                                   ENGINE_HANDLE_V1 *h1) {
1174    check(!get_meta(h, h1, "key3"), "Expected to get meta");
1175
1176    ItemMetaData itm_meta;
1177    itm_meta.revSeqno = 1;
1178    itm_meta.cas = 1;
1179    itm_meta.exptime = 0;
1180    itm_meta.flags = 0;
1181    set_with_meta(h, h1, "key3", 4, "val0", 4, 0, &itm_meta, last_meta.cas);
1182
1183    itm_meta.revSeqno = 2;
1184    itm_meta.cas = 2;
1185    set_with_meta(h, h1, "key3", 4, "val1", 4, 0, &itm_meta, last_meta.cas);
1186
1187    // MB-21725 Depending on how fast the flusher is, we may see 1 or 2.
1188    wait_for_stat_to_be_gte(h, h1, "ep_total_persisted", 1);
1189
1190    itm_meta.revSeqno = 3;
1191    itm_meta.cas = 3;
1192    itm_meta.exptime = 1735689600; // expires in 2025
1193    set_with_meta(h, h1, "key3", 4, "val1", 4, 0, &itm_meta, last_meta.cas);
1194
1195    testHarness.time_travel(500000000);
1196    // Wait for the item to be expired, either by the pager,
1197    // or by access (as part of persistence callback from a
1198    // previous set - slow disk), or the compactor (unlikely).
1199    wait_for_expired_items_to_be(h, h1, 1);
1200
1201    wait_for_flusher_to_settle(h, h1);
1202    wait_for_stat_to_be(h, h1, "curr_items", 0);
1203
1204    check(get_meta(h, h1, "key3"), "Expected to get meta");
1205    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
1206    check(last_meta.revSeqno == 4, "Expected seqno to match");
1207    check(last_meta.cas != 3, "Expected cas to be different");
1208    check(last_meta.flags == 0, "Expected flags to match");
1209
1210    return SUCCESS;
1211}
1212
1213static enum test_result test_temp_item_deletion(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)
1214{
1215    // Do get_meta for an existing key
1216    char const *k1 = "k1";
1217    item *i = NULL;
1218
1219    checkeq(ENGINE_SUCCESS,
1220            store(h, h1, NULL, OPERATION_SET, k1, "somevalue", &i),
1221            "Failed set.");
1222    wait_for_flusher_to_settle(h, h1);
1223
1224    checkeq(ENGINE_SUCCESS, del(h, h1, k1, 0, 0), "Delete failed");
1225    wait_for_flusher_to_settle(h, h1);
1226    wait_for_stat_to_be(h, h1, "curr_items", 0);
1227
1228    // Issue a get_meta for a deleted key. This will need to bring in a temp
1229    // item into the hashtable as a placeholder for the (deleted) metadata
1230    // which needs to be loaded from disk via BG fetch
1231    // We need to temporarily disable the reader threads as to prevent the
1232    // BGfetch from immediately running and removing our temp_item before
1233    // we've had chance to validate its existence.
1234    set_param(h, h1, protocol_binary_engine_param_flush,
1235              "max_num_readers", "0");
1236
1237    // Disable nonio so that we have better control of the expirypager
1238    set_param(h, h1, protocol_binary_engine_param_flush,
1239              "max_num_nonio", "0");
1240
1241    // Tell the harness not to handle EWOULDBLOCK for us - we want it to
1242    // be outstanding while we check the below stats.
1243    const void *cookie = testHarness.create_cookie();
1244    testHarness.set_ewouldblock_handling(cookie, false);
1245
1246    checkeq(false, get_meta(h, h1, k1, /*reqExtMeta*/false, cookie),
1247            "Expected get_meta to fail (EWOULDBLOCK)");
1248    checkeq(static_cast<protocol_binary_response_status>(ENGINE_EWOULDBLOCK),
1249            last_status.load(), "Expected EWOULDBLOCK");
1250
1251    checkeq(0, get_int_stat(h, h1, "curr_items"), "Expected zero curr_items");
1252    checkeq(1, get_int_stat(h, h1, "curr_temp_items"), "Expected single temp_items");
1253
1254    // Re-enable EWOULDBLOCK handling (and reader threads), and re-issue.
1255    testHarness.set_ewouldblock_handling(cookie, true);
1256    set_param(h, h1, protocol_binary_engine_param_flush,
1257              "max_num_readers", "1");
1258
1259    check(get_meta(h, h1, k1, /*reqExtMeta*/false, cookie),
1260          "Expected get_meta to succeed");
1261    check(last_deleted_flag, "Expected deleted flag to be set");
1262
1263    // Do get_meta for a non-existing key.
1264    char const *k2 = "k2";
1265    check(!get_meta(h, h1, k2), "Expected get meta to return false");
1266    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_ENOENT, last_status.load(), "Expected enoent");
1267
1268    // Ensure all bg fetches completed (two were requested)
1269    wait_for_stat_to_be(h, h1, "ep_bg_meta_fetched", 2);
1270
1271    // Trigger the expiry pager and verify that two temp items are deleted
1272    set_param(h, h1, protocol_binary_engine_param_flush,
1273              "max_num_nonio", "1");
1274
1275    wait_for_stat_to_be(h, h1, "ep_expired_pager", 1);
1276    checkeq(0, get_int_stat(h, h1, "curr_items"), "Expected zero curr_items");
1277    checkeq(0, get_int_stat(h, h1, "curr_temp_items"), "Expected zero temp_items");
1278
1279    h1->release(h, NULL, i);
1280    testHarness.destroy_cookie(cookie);
1281
1282    return SUCCESS;
1283}
1284
1285static enum test_result test_add_meta_conflict_resolution(ENGINE_HANDLE *h,
1286                                                          ENGINE_HANDLE_V1 *h1) {
1287    // put some random metadata
1288    ItemMetaData itemMeta;
1289    itemMeta.revSeqno = 10;
1290    itemMeta.cas = 0xdeadbeef;
1291    itemMeta.exptime = 0;
1292    itemMeta.flags = 0xdeadbeef;
1293
1294    add_with_meta(h, h1, "key", 3, NULL, 0, 0, &itemMeta);
1295    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
1296    checkeq(0, get_int_stat(h, h1, "ep_bg_meta_fetched"),
1297            "Expected no bg meta fetches, thanks to bloom filters");
1298
1299    checkeq(ENGINE_SUCCESS, del(h, h1, "key", 0, 0), "Delete failed");
1300    wait_for_flusher_to_settle(h, h1);
1301    wait_for_stat_to_be(h, h1, "curr_items", 0);
1302
1303    // Check all meta data is the same
1304    itemMeta.revSeqno++;
1305    itemMeta.cas++;
1306    add_with_meta(h, h1, "key", 3, NULL, 0, 0, &itemMeta);
1307    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, last_status.load(), "Expected exists");
1308    checkeq(1, get_int_stat(h, h1, "ep_bg_meta_fetched"),
1309          "Expected two be meta fetches");
1310    checkeq(1, get_int_stat(h, h1, "ep_num_ops_set_meta_res_fail"),
1311          "Expected set meta conflict resolution failure");
1312
1313    // Check has older flags fails
1314    itemMeta.flags = 0xdeadbeee;
1315    add_with_meta(h, h1, "key", 3, NULL, 0, 0, &itemMeta);
1316    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, last_status.load(), "Expected exists");
1317    checkeq(2, get_int_stat(h, h1, "ep_num_ops_set_meta_res_fail"),
1318          "Expected set meta conflict resolution failure");
1319
1320    // Check testing with old seqno
1321    itemMeta.revSeqno--;
1322    add_with_meta(h, h1, "key", 3, NULL, 0, 0, &itemMeta);
1323    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, last_status.load(), "Expected exists");
1324    checkeq(3, get_int_stat(h, h1, "ep_num_ops_set_meta_res_fail"),
1325          "Expected set meta conflict resolution failure");
1326
1327    itemMeta.revSeqno += 10;
1328    add_with_meta(h, h1, "key", 3, NULL, 0, 0, &itemMeta);
1329    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
1330    checkeq(3, get_int_stat(h, h1, "ep_num_ops_set_meta_res_fail"),
1331          "Expected set meta conflict resolution failure");
1332
1333    return SUCCESS;
1334}
1335
1336static enum test_result test_set_meta_conflict_resolution(ENGINE_HANDLE *h,
1337                                                          ENGINE_HANDLE_V1 *h1) {
1338    // put some random metadata
1339    ItemMetaData itemMeta;
1340    itemMeta.revSeqno = 10;
1341    itemMeta.cas = 0xdeadbeef;
1342    itemMeta.exptime = 0;
1343    itemMeta.flags = 0xdeadbeef;
1344
1345    checkeq(0, get_int_stat(h, h1, "ep_num_ops_set_meta"),
1346          "Expect zero setMeta ops");
1347
1348    set_with_meta(h, h1, "key", 3, NULL, 0, 0, &itemMeta, 0);
1349    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
1350    checkeq(0, get_int_stat(h, h1, "ep_bg_meta_fetched"),
1351            "Expected no bg meta fetches, thanks to bloom filters");
1352
1353    // Check all meta data is the same
1354    set_with_meta(h, h1, "key", 3, NULL, 0, 0, &itemMeta, 0);
1355    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, last_status.load(), "Expected exists");
1356    checkeq(1, get_int_stat(h, h1, "ep_num_ops_set_meta_res_fail"),
1357          "Expected set meta conflict resolution failure");
1358
1359    // Check has older flags fails
1360    itemMeta.flags = 0xdeadbeee;
1361    set_with_meta(h, h1, "key", 3, NULL, 0, 0, &itemMeta, 0);
1362    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, last_status.load(), "Expected exists");
1363    checkeq(2, get_int_stat(h, h1, "ep_num_ops_set_meta_res_fail"),
1364          "Expected set meta conflict resolution failure");
1365
1366    // Check has newer flags passes
1367    itemMeta.flags = 0xdeadbeff;
1368    set_with_meta(h, h1, "key", 3, NULL, 0, 0, &itemMeta, 0);
1369    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
1370
1371    // Check that newer exptime wins
1372    itemMeta.exptime = time(NULL) + 10;
1373    set_with_meta(h, h1, "key", 3, NULL, 0, 0, &itemMeta, 0);
1374    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
1375
1376    // Check that smaller exptime loses
1377    itemMeta.exptime = 0;
1378    set_with_meta(h, h1, "key", 3, NULL, 0, 0, &itemMeta, 0);
1379    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, last_status.load(), "Expected exists");
1380    checkeq(3, get_int_stat(h, h1, "ep_num_ops_set_meta_res_fail"),
1381          "Expected set meta conflict resolution failure");
1382
1383    // Check testing with old seqno
1384    itemMeta.revSeqno--;
1385    set_with_meta(h, h1, "key", 3, NULL, 0, 0, &itemMeta, 0);
1386    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, last_status.load(), "Expected exists");
1387    checkeq(4, get_int_stat(h, h1, "ep_num_ops_set_meta_res_fail"),
1388          "Expected set meta conflict resolution failure");
1389
1390    itemMeta.revSeqno += 10;
1391    set_with_meta(h, h1, "key", 3, NULL, 0, 0, &itemMeta, 0);
1392    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
1393    checkeq(4, get_int_stat(h, h1, "ep_num_ops_set_meta_res_fail"),
1394          "Expected set meta conflict resolution failure");
1395
1396    checkeq(0, get_int_stat(h, h1, "ep_bg_meta_fetched"),
1397            "Expect no bg meta fetches");
1398
1399    return SUCCESS;
1400}
1401
1402static enum test_result test_set_meta_lww_conflict_resolution(ENGINE_HANDLE *h,
1403                                                              ENGINE_HANDLE_V1 *h1) {
1404    // put some random metadata
1405    ItemMetaData itemMeta;
1406    itemMeta.revSeqno = 10;
1407    itemMeta.cas = 0xdeadbeef;
1408    itemMeta.exptime = 0;
1409    itemMeta.flags = 0xdeadbeef;
1410
1411    checkeq(0, get_int_stat(h, h1, "ep_num_ops_set_meta"),
1412          "Expect zero setMeta ops");
1413
1414    set_with_meta(h, h1, "key", 3, NULL, 0, 0, &itemMeta, 0,
1415                  FORCE_ACCEPT_WITH_META_OPS);
1416    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
1417    checkeq(0, get_int_stat(h, h1, "ep_bg_meta_fetched"),
1418            "Expected no bg meta fetchs, thanks to bloom filters");
1419
1420    // Check all meta data is the same
1421    set_with_meta(h, h1, "key", 3, NULL, 0, 0, &itemMeta, 0,
1422                  FORCE_ACCEPT_WITH_META_OPS);
1423    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, last_status.load(), "Expected exists");
1424    checkeq(1, get_int_stat(h, h1, "ep_num_ops_set_meta_res_fail"),
1425          "Expected set meta conflict resolution failure");
1426
1427    // Check that an older cas fails
1428    itemMeta.cas = 0xdeadbeee;
1429    set_with_meta(h, h1, "key", 3, NULL, 0, 0, &itemMeta, 0,
1430                  FORCE_ACCEPT_WITH_META_OPS);
1431    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, last_status.load(), "Expected exists");
1432    checkeq(2, get_int_stat(h, h1, "ep_num_ops_set_meta_res_fail"),
1433          "Expected set meta conflict resolution failure");
1434
1435    // Check that a higher cas passes
1436    itemMeta.cas = 0xdeadbeff;
1437    set_with_meta(h, h1, "key", 3, NULL, 0, 0, &itemMeta, 0,
1438                  FORCE_ACCEPT_WITH_META_OPS);
1439    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
1440
1441    // Check that we fail requests if the force flag is not set
1442    itemMeta.cas = 0xdeadbeff + 1;
1443    set_with_meta(h, h1, "key", 3, NULL, 0, 0, &itemMeta, 0,
1444                  0/*options*/);
1445    checkeq(PROTOCOL_BINARY_RESPONSE_EINVAL, last_status.load(), "Expected EINVAL");
1446
1447    return SUCCESS;
1448}
1449
1450static enum test_result test_del_meta_conflict_resolution(ENGINE_HANDLE *h,
1451                                                          ENGINE_HANDLE_V1 *h1) {
1452
1453    item *i = NULL;
1454    checkeq(ENGINE_SUCCESS,
1455            store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1456            "Failed set.");
1457    wait_for_flusher_to_settle(h, h1);
1458    h1->release(h, NULL, i);
1459
1460    // put some random metadata
1461    ItemMetaData itemMeta;
1462    itemMeta.revSeqno = 10;
1463    itemMeta.cas = 0xdeadbeef;
1464    itemMeta.exptime = 0;
1465    itemMeta.flags = 0xdeadbeef;
1466
1467    del_with_meta(h, h1, "key", 3, 0, &itemMeta);
1468    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
1469    wait_for_flusher_to_settle(h, h1);
1470    wait_for_stat_to_be(h, h1, "curr_items", 0);
1471
1472    // Check all meta data is the same
1473    del_with_meta(h, h1, "key", 3, 0, &itemMeta);
1474    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, last_status.load(), "Expected exists");
1475    checkeq(1, get_int_stat(h, h1, "ep_num_ops_del_meta_res_fail"),
1476          "Expected delete meta conflict resolution failure");
1477
1478    // Check has older flags fails
1479    itemMeta.flags = 0xdeadbeee;
1480    del_with_meta(h, h1, "key", 3, 0, &itemMeta);
1481    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, last_status.load(), "Expected exists");
1482    checkeq(2, get_int_stat(h, h1, "ep_num_ops_del_meta_res_fail"),
1483          "Expected delete meta conflict resolution failure");
1484
1485    // Check that smaller exptime loses
1486    itemMeta.exptime = 0;
1487    del_with_meta(h, h1, "key", 3, 0, &itemMeta);
1488    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, last_status.load(), "Expected exists");
1489    checkeq(3, get_int_stat(h, h1, "ep_num_ops_del_meta_res_fail"),
1490          "Expected delete meta conflict resolution failure");
1491
1492    // Check testing with old seqno
1493    itemMeta.revSeqno--;
1494    del_with_meta(h, h1, "key", 3, 0, &itemMeta);
1495    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, last_status.load(), "Expected exists");
1496    check(get_int_stat(h, h1, "ep_num_ops_del_meta_res_fail") == 4,
1497          "Expected delete meta conflict resolution failure");
1498
1499    itemMeta.revSeqno += 10;
1500    del_with_meta(h, h1, "key", 3, 0, &itemMeta);
1501    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
1502    check(get_int_stat(h, h1, "ep_num_ops_del_meta_res_fail") == 4,
1503          "Expected delete meta conflict resolution failure");
1504
1505    return SUCCESS;
1506}
1507
1508static enum test_result test_del_meta_lww_conflict_resolution(ENGINE_HANDLE *h,
1509                                                              ENGINE_HANDLE_V1 *h1) {
1510
1511    item *i = NULL;
1512    item_info info;
1513
1514    checkeq(ENGINE_SUCCESS,
1515            store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1516            "Failed set.");
1517
1518    info.nvalue = 1;
1519    h1->get_item_info(h, NULL, i, &info);
1520    wait_for_flusher_to_settle(h, h1);
1521    h1->release(h, NULL, i);
1522
1523    // put some random metadata
1524    ItemMetaData itemMeta;
1525    itemMeta.revSeqno = 10;
1526    itemMeta.cas = info.cas + 1;
1527    itemMeta.exptime = 0;
1528    itemMeta.flags = 0xdeadbeef;
1529
1530    // first check the command fails if no force is set
1531    del_with_meta(h, h1, "key", 3, 0, &itemMeta, 0, 0/*options*/);
1532    checkeq(PROTOCOL_BINARY_RESPONSE_EINVAL, last_status.load(), "Expected EINVAL");
1533
1534    del_with_meta(h, h1, "key", 3, 0, &itemMeta, 0, FORCE_ACCEPT_WITH_META_OPS);
1535    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
1536    wait_for_flusher_to_settle(h, h1);
1537    wait_for_stat_to_be(h, h1, "curr_items", 0);
1538
1539    // Check all meta data is the same
1540    del_with_meta(h, h1, "key", 3, 0, &itemMeta, 0, FORCE_ACCEPT_WITH_META_OPS);
1541    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, last_status.load(), "Expected exists");
1542    checkeq(1, get_int_stat(h, h1, "ep_num_ops_del_meta_res_fail"),
1543          "Expected delete meta conflict resolution failure");
1544
1545    // Check that higher rev seqno but lower cas fails
1546    itemMeta.cas = info.cas;
1547    itemMeta.revSeqno = 11;
1548    del_with_meta(h, h1, "key", 3, 0, &itemMeta, 0, FORCE_ACCEPT_WITH_META_OPS);
1549    checkeq(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, last_status.load(), "Expected exists");
1550    checkeq(2, get_int_stat(h, h1, "ep_num_ops_del_meta_res_fail"),
1551          "Expected delete meta conflict resolution failure");
1552
1553    // Check that a higher cas and lower rev seqno passes
1554    itemMeta.cas = info.cas + 2;
1555    itemMeta.revSeqno = 9;
1556    del_with_meta(h, h1, "key", 3, 0, &itemMeta, 0, FORCE_ACCEPT_WITH_META_OPS);
1557    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected sucess");
1558
1559    return SUCCESS;
1560}
1561
1562static enum test_result test_getMeta_with_item_eviction(ENGINE_HANDLE *h,
1563                                                        ENGINE_HANDLE_V1 *h1)
1564{
1565    char const *key = "test_get_meta";
1566    item *i = NULL;
1567    checkeq(ENGINE_SUCCESS,
1568            store(h, h1, NULL, OPERATION_SET, key, "somevalue", &i),
1569            "Failed set.");
1570    wait_for_flusher_to_settle(h, h1);
1571    evict_key(h, h1, key, 0, "Ejected.");
1572
1573    Item *it = reinterpret_cast<Item*>(i);
1574
1575    check(get_meta(h, h1, key), "Expected to get meta");
1576    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
1577    ItemMetaData metadata(it->getCas(), it->getRevSeqno(),
1578                          it->getFlags(), it->getExptime());
1579    verifyLastMetaData(metadata);
1580
1581    h1->release(h, NULL, i);
1582    return SUCCESS;
1583}
1584
1585static enum test_result test_set_with_meta_and_check_drift_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1586    // Activate n vbuckets (vb 0 is already)
1587    const int n_vbuckets = 10;
1588    for (int ii = 1; ii < n_vbuckets; ii++) {
1589        check(set_vbucket_state(h, h1, ii, vbucket_state_active),
1590              "Failed to set vbucket state.");
1591    }
1592
1593    // Let's make vbucket n/2 be the one who is ahead, n/3 is behind
1594    const int aheadVb = n_vbuckets/2;
1595    const int behindVb = n_vbuckets/3;
1596    checkne(aheadVb, behindVb, "Cannot have the same VB as ahead/behind");
1597
1598    HLC hlc(0/*init HLC*/,
1599            std::chrono::microseconds(0)/*ahead threshold*/,
1600            std::chrono::microseconds(0)/*behind threshold*/);
1601
1602    // grab the drift behind threshold
1603    uint64_t driftBehindThreshold = get_ull_stat(h, h1,
1604                                                 "ep_hlc_drift_ahead_threshold_us",
1605                                                 nullptr);
1606    // Create n keys
1607    const int n_keys = 5;
1608    for (int ii = 0 ; ii < n_vbuckets; ii++) {
1609        for (int k = 0; k < n_keys; k++) {
1610            std::string key = "key_" + std::to_string(k);
1611            ItemMetaData itm_meta;
1612            itm_meta.cas = hlc.nextHLC();
1613            if (ii == aheadVb) {
1614                // Push this guy *far* ahead (1 year)
1615                itm_meta.cas += 3154E10;
1616            } else if(ii == behindVb) {
1617                // just be sure it was already greater then 1 + driftthreshold
1618                checkge(itm_meta.cas, uint64_t(1) + driftBehindThreshold,
1619                        "HLC was already zero");
1620                // set to be way way behind...
1621                itm_meta.cas = 1;
1622            }
1623            set_with_meta(h, h1, key.data(), key.size(), NULL, 0, ii, &itm_meta,
1624                          0, FORCE_ACCEPT_WITH_META_OPS);
1625            checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1626                    "Expected success");
1627        }
1628    }
1629
1630    // Bucket stats should report drift
1631    checkge(get_ull_stat(h, h1, "ep_active_hlc_drift"), uint64_t(0),
1632            "Expected drift above zero");
1633    checkeq(uint64_t(n_keys*n_vbuckets), get_ull_stat(h, h1, "ep_active_hlc_drift_count"),
1634            "Expected ahead counter to match mutations");
1635
1636    // Victim VBs should have exceptions
1637    {
1638        std::string vbAheadName = "vb_" + std::to_string(aheadVb);
1639        std::string ahead_threshold_exceeded = vbAheadName + ":drift_ahead_threshold_exceeded";
1640        std::string behind_threshold_exceeded = vbAheadName + ":drift_behind_threshold_exceeded";
1641        std::string total_abs_drift = vbAheadName + ":total_abs_drift";
1642        std::string details = "vbucket-details " + std::to_string(aheadVb);
1643            checkeq(uint64_t(n_keys), get_ull_stat(h, h1, ahead_threshold_exceeded.data(), details.data()),
1644                "Expected ahead threshold to match mutations");
1645        checkeq(uint64_t(0), get_ull_stat(h, h1, behind_threshold_exceeded.data(), details.data()),
1646                "Expected no behind exceptions");
1647        checkge(get_ull_stat(h, h1, total_abs_drift.data(), details.data()), uint64_t(0),
1648                "Expected some drift");
1649    }
1650
1651    {
1652        std::string vbBehindName = "vb_" + std::to_string(behindVb);
1653        std::string ahead_threshold_exceeded = vbBehindName + ":drift_ahead_threshold_exceeded";
1654        std::string behind_threshold_exceeded = vbBehindName + ":drift_behind_threshold_exceeded";
1655        std::string total_abs_drift = vbBehindName + ":total_abs_drift";
1656        std::string details = "vbucket-details " + std::to_string(behindVb);
1657        checkeq(uint64_t(n_keys), get_ull_stat(h, h1, behind_threshold_exceeded.data(), details.data()),
1658                "Expected behind threshold to match mutations");
1659        checkeq(uint64_t(0), get_ull_stat(h, h1, ahead_threshold_exceeded.data(), details.data()),
1660                "Expected no ahead exceptions");
1661        checkge(get_ull_stat(h, h1, total_abs_drift.data(), details.data()), uint64_t(0),
1662                "Expected some drift");
1663    }
1664
1665
1666    return SUCCESS;
1667}
1668
1669static enum test_result test_del_with_meta_and_check_drift_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1670    // Activate n vbuckets (vb 0 is already)
1671    const int n_vbuckets = 10;
1672    for (int ii = 1; ii < n_vbuckets; ii++) {
1673        check(set_vbucket_state(h, h1, ii, vbucket_state_active),
1674              "Failed to set vbucket state.");
1675    }
1676
1677    // Let's make vbucket n/2 be the one who is ahead, n/3 is behind
1678    const int aheadVb = n_vbuckets/2;
1679    const int behindVb = n_vbuckets/3;
1680    checkne(aheadVb, behindVb, "Cannot have the same VB as ahead/behind");
1681
1682    HLC hlc(0/*init HLC*/,
1683            std::chrono::microseconds(0)/*ahead threshold*/,
1684            std::chrono::microseconds(0)/*behind threshold*/);
1685
1686    // grab the drift behind threshold
1687    uint64_t driftBehindThreshold = get_ull_stat(h, h1,
1688                                                 "ep_hlc_drift_ahead_threshold_us",
1689                                                 nullptr);
1690    // Create n keys * n_vbuckets
1691    const int n_keys = 5;
1692    for (int ii = 0 ; ii < n_vbuckets; ii++) {
1693        for (int k = 0; k < n_keys; k++) {
1694            std::string key = "key_" + std::to_string(k);
1695
1696            // In the del_with_meta test we want to pretend a del_wm came from
1697            // the past, so we want to ensure a delete doesn't get rejected
1698            // by LWW conflict resolution, thus write all documents that are
1699            // going to be deleted with set_with_meta, and write them way in the past.
1700            // This will trigger threshold and increment drift stats... so we
1701            // account for these later
1702            ItemMetaData itm_meta;
1703            itm_meta.cas = 1; // set to 1
1704            set_with_meta(h, h1, key.data(), key.size(), NULL, 0, ii, &itm_meta,
1705                          0, FORCE_ACCEPT_WITH_META_OPS);
1706            checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1707                    "Expected success");
1708        }
1709    }
1710
1711    checkeq(uint64_t(0), get_ull_stat(h, h1, "ep_active_ahead_exceptions"),
1712            "Expected ahead counter to match mutations");
1713    checkeq(uint64_t(n_keys*n_vbuckets), get_ull_stat(h, h1, "ep_active_behind_exceptions"),
1714            "Expected behind counter to match mutations");
1715
1716    // Del_with_meta n_keys to n_vbuckets
1717    for (int ii = 0 ; ii < n_vbuckets; ii++) {
1718        for (int k = 0; k < n_keys; k++) {
1719            std::string key = "key_" + std::to_string(k);
1720            ItemMetaData itm_meta;
1721            itm_meta.cas = hlc.nextHLC();
1722            if (ii == aheadVb) {
1723                // Push this guy *far* ahead (1 year)
1724                itm_meta.cas += 3154E10;
1725            } else if(ii == behindVb) {
1726                // just be sure it was already greater than 1 + driftthreshold
1727                checkge(itm_meta.cas, uint64_t(1) + driftBehindThreshold,
1728                        "HLC was already zero");
1729                // set to be way way behind, but ahead of the documents we have set
1730                itm_meta.cas = 2;
1731            }
1732            del_with_meta(h, h1, key.data(), key.size(), ii, &itm_meta,
1733                          1, FORCE_ACCEPT_WITH_META_OPS);
1734            checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1735                    "Expected success");
1736        }
1737    }
1738
1739    // Bucket stats should report drift
1740    checkge(get_ull_stat(h, h1, "ep_active_hlc_drift"), uint64_t(0),
1741            "Expected drift above zero");
1742    checkeq(2*uint64_t(n_keys*n_vbuckets), get_ull_stat(h, h1, "ep_active_hlc_drift_count"),
1743            "Expected ahead counter to match mutations");
1744
1745    // and should report total exception of all VBs
1746    checkeq(uint64_t(n_keys), get_ull_stat(h, h1, "ep_active_ahead_exceptions"),
1747            "Expected ahead counter to match mutations");
1748    checkeq(uint64_t(n_keys + (n_keys*n_vbuckets)), get_ull_stat(h, h1, "ep_active_behind_exceptions"),
1749            "Expected behind counter to match mutations");
1750
1751    // Victim VBs should have exceptions
1752    {
1753        std::string vbAheadName = "vb_" + std::to_string(aheadVb);
1754        std::string ahead_threshold_exceeded = vbAheadName + ":drift_ahead_threshold_exceeded";
1755        std::string behind_threshold_exceeded = vbAheadName + ":drift_behind_threshold_exceeded";
1756        std::string total_abs_drift = vbAheadName + ":total_abs_drift";
1757        std::string details = "vbucket-details " + std::to_string(aheadVb);
1758
1759        checkeq(uint64_t(n_keys),
1760                get_ull_stat(h, h1, ahead_threshold_exceeded.data(), details.data()),
1761                "Expected ahead threshold to match mutations");
1762        checkge(get_ull_stat(h, h1, total_abs_drift.data(), details.data()), uint64_t(0),
1763                "Expected some drift");
1764    }
1765
1766    {
1767        std::string vbBehindName = "vb_" + std::to_string(behindVb);
1768        std::string ahead_threshold_exceeded = vbBehindName + ":drift_ahead_threshold_exceeded";
1769        std::string behind_threshold_exceeded = vbBehindName + ":drift_behind_threshold_exceeded";
1770        std::string total_abs_drift = vbBehindName + ":total_abs_drift";
1771        std::string details = "vbucket-details " + std::to_string(behindVb);
1772
1773        // *2 behind due to the initial set_with_meta
1774        checkeq(uint64_t(n_keys*2), get_ull_stat(h, h1, behind_threshold_exceeded.data(), details.data()),
1775                "Expected behind threshold to match mutations");
1776        checkeq(uint64_t(0), get_ull_stat(h, h1, ahead_threshold_exceeded.data(), details.data()),
1777                "Expected no ahead exceptions");
1778        checkge(get_ull_stat(h, h1, total_abs_drift.data(), details.data()), uint64_t(0),
1779                "Expected some drift");
1780    }
1781
1782
1783    return SUCCESS;
1784}
1785
1786static enum test_result test_setting_drift_threshold(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1787
1788    std::vector<std::tuple<std::string, std::string, std::string> > configData =
1789        {std::make_tuple("ep_hlc_drift_ahead_threshold_us",
1790                         "hlc_drift_ahead_threshold_us",
1791                         "vb_0:drift_ahead_threshold"),
1792         std::make_tuple("ep_hlc_drift_behind_threshold_us",
1793                         "hlc_drift_behind_threshold_us",
1794                         "vb_0:drift_behind_threshold")};
1795
1796    std::vector<std::pair<std::string, std::chrono::microseconds> > values = {
1797        {"0", std::chrono::microseconds(0)},
1798        {"1", std::chrono::microseconds(1)},
1799        {"-1", std::chrono::microseconds(-1)},
1800        {"-0", std::chrono::microseconds(0)},
1801        {"18446744073709551615",
1802         std::chrono::microseconds(18446744073709551615ull)}};
1803
1804    for (auto data : values) {
1805        for (auto conf : configData) {
1806            check(set_param(h, h1, protocol_binary_engine_param_vbucket,
1807                     std::get<1>(conf).c_str(), data.first.data()),
1808                "Expected set_param success");
1809
1810            checkeq(data.second.count(),
1811                    int64_t(get_ull_stat(h, h1, std::get<0>(conf).c_str(), nullptr)),
1812                    "Expected the stat to change to the new value");
1813
1814            // The VB stat values are in nanoseconds
1815            checkeq(std::chrono::nanoseconds(data.second).count(),
1816                    int64_t(get_ull_stat(h, h1, std::get<2>(conf).c_str(), "vbucket-details 0")),
1817                    "Expected the VB stats to change to the new value");
1818        }
1819    }
1820    return SUCCESS;
1821}
1822
1823/*
1824 * Perform set_with_meta and check CAS regeneration is ok.
1825 */
1826static enum test_result test_cas_regeneration(ENGINE_HANDLE *h,
1827                                              ENGINE_HANDLE_V1 *h1) {
1828
1829    // First store a key from the past (small CAS).
1830    ItemMetaData itemMeta;
1831    itemMeta.revSeqno = 10;
1832    itemMeta.cas = 0x1;
1833    itemMeta.exptime = 0;
1834    itemMeta.flags = 0xdeadbeef;
1835    int force = 0;
1836
1837    if (strstr(testHarness.get_current_testcase()->cfg,
1838               "conflict_resolution_type=lww") != nullptr) {
1839        force = FORCE_ACCEPT_WITH_META_OPS;
1840    }
1841
1842    // Set the key with a low CAS value
1843    set_with_meta(h, h1, "key", 3, nullptr, 0, 0, &itemMeta, 0, force);
1844    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(), "Expected success");
1845
1846    check(get_meta(h, h1, "key"), "Failed to get_meta");
1847
1848    // CAS must be what we set.
1849    checkeq(itemMeta.cas, last_meta.cas, "CAS is not the value we stored");
1850
1851    itemMeta.cas++;
1852
1853    // Check that the code requires skip
1854    set_with_meta(h, h1, "key", 3, nullptr, 0, 0, &itemMeta, 0,
1855                  REGENERATE_CAS/*but no skip*/);
1856    checkeq(PROTOCOL_BINARY_RESPONSE_EINVAL, last_status.load(),
1857            "Expected EINVAL");
1858
1859    set_with_meta(h, h1, "key", 3, nullptr, 0, 0, &itemMeta, 0,
1860                  REGENERATE_CAS|SKIP_CONFLICT_RESOLUTION_FLAG);
1861
1862    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1863            "Expected success");
1864
1865    check(get_meta(h, h1, "key"), "Failed to get_meta");
1866
1867    uint64_t cas = last_meta.cas;
1868    // Check item has a new CAS
1869    checkne(itemMeta.cas, cas, "CAS was not regenerated");
1870
1871    itemMeta.cas++;
1872    // All flags set should still regen the cas (lww and seqno)
1873    set_with_meta(h, h1, "key", 3, nullptr, 0, 0, &itemMeta, 0,
1874                  REGENERATE_CAS|SKIP_CONFLICT_RESOLUTION_FLAG|force);
1875
1876    checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1877            "Expected success");
1878
1879    check(get_meta(h, h1, "key"), "Failed to get_meta");
1880    // Check item has a new CAS
1881    checkne(itemMeta.cas, last_meta.cas, "CAS was not regenerated");
1882    checkne(cas, last_meta.cas, "CAS was not regenerated");
1883    return SUCCESS;
1884}
1885
1886/*
1887 * Test that we can send options and nmeta
1888 * The nmeta is just going to be ignored though, but should not fail
1889 */
1890static enum test_result test_cas_options_and_nmeta(ENGINE_HANDLE *h,
1891                                                   ENGINE_HANDLE_V1 *h1) {
1892    ItemMetaData itemMeta;
1893    itemMeta.revSeqno = 10;
1894    itemMeta.cas = 0x1;
1895    itemMeta.exptime = 0;
1896    itemMeta.flags = 0xdeadbeef;
1897
1898    // Watson (4.6) accepts valid encodings, but ignores them
1899    std::vector<char> junkMeta = {-2,-1,2,3};
1900
1901    int force = 0;
1902
1903    if (strstr(testHarness.get_current_testcase()->cfg,
1904               "conflict_resolution_type=lww") != nullptr) {
1905        force = FORCE_ACCEPT_WITH_META_OPS;
1906    }
1907
1908    // Set the key and junk nmeta
1909    set_with_meta(h, h1, "key", 3, NULL, 0, 0, &itemMeta, 0,
1910                  force, PROTOCOL_BINARY_RAW_BYTES,
1911                  nullptr, junkMeta);
1912    checkeq(PROTOCOL_BINARY_RESPONSE_EINVAL, last_status.load(), "Expected EINVAL");
1913
1914    // Set the key and junk nmeta that's quite large
1915    junkMeta.resize(std::numeric_limits<uint16_t>::max());
1916    set_with_meta(h, h1, "key", 3, NULL, 0, 0, &itemMeta, 0,
1917                  force, PROTOCOL_BINARY_RAW_BYTES,
1918                  nullptr, junkMeta);
1919    checkeq(PROTOCOL_BINARY_RESPONSE_EINVAL, last_status.load(), "Expected EINVAL");
1920
1921    // Test that valid meta can be sent. It should be ignored and success
1922    // returned
1923    // Encodings which should not fail, see ext_meta_parser.cc
1924#pragma pack(1)
1925    struct adjusted_time_metadata {
1926        uint8_t type;
1927        uint16_t length;
1928        int64_t value;
1929    };
1930    struct conf_res_metadata {
1931        uint8_t type;
1932        uint16_t length;
1933        uint8_t value;
1934    };
1935    struct with_cas_metadata1 {
1936        uint8_t version;
1937        adjusted_time_metadata adjusted_time;
1938    };
1939    struct with_cas_metadata2 {
1940        uint8_t version;
1941        conf_res_metadata conf_res;
1942    };
1943    struct with_cas_metadata3 {
1944        uint8_t version;
1945        conf_res_metadata conf_res;
1946        adjusted_time_metadata adjusted_time;
1947    };
1948    struct with_cas_metadata4 {
1949        uint8_t version;
1950        adjusted_time_metadata adjusted_time;
1951        conf_res_metadata conf_res;
1952    };
1953#pragma pack()
1954
1955    {
1956        with_cas_metadata1 validMetaData = {META_EXT_VERSION_ONE,
1957                                            {CMD_META_ADJUSTED_TIME,
1958                                             htons(sizeof(int64_t)), -1}};
1959        std::vector<char> validMetaVector(reinterpret_cast<char*>(&validMetaData),
1960                                          reinterpret_cast<char*>(&validMetaData) +
1961                                          sizeof(validMetaData));
1962
1963        // Set the key with a low CAS value and real nmeta
1964        set_with_meta(h, h1, "key1", 4, nullptr, 0, 0, &itemMeta, 0,
1965                      force, PROTOCOL_BINARY_RAW_BYTES,
1966                      nullptr, validMetaVector);
1967        checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1968                "Expected success");
1969
1970        itemMeta.cas++;
1971        del_with_meta(h, h1, "key1", 4, 0, &itemMeta, 0,
1972                      force, nullptr, validMetaVector);
1973        checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1974                "Expected success");
1975    }
1976
1977    {
1978        with_cas_metadata2 validMetaData = {META_EXT_VERSION_ONE,
1979                                            {CMD_META_CONFLICT_RES_MODE,
1980                                             htons(sizeof(uint8_t)), 0xff}};
1981        std::vector<char> validMetaVector(reinterpret_cast<char*>(&validMetaData),
1982                                          reinterpret_cast<char*>(&validMetaData) +
1983                                          sizeof(validMetaData));
1984
1985        // Set the key with a low CAS value and real nmeta
1986        set_with_meta(h, h1, "key2", 4, nullptr, 0, 0, &itemMeta, 0,
1987                      force, PROTOCOL_BINARY_RAW_BYTES,
1988                      nullptr, validMetaVector);
1989        checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1990                "Expected success");
1991
1992        itemMeta.cas++;
1993        del_with_meta(h, h1, "key2", 4, 0, &itemMeta, 0,
1994                      force, nullptr, validMetaVector);
1995        checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1996                "Expected success");
1997    }
1998
1999    {
2000        with_cas_metadata3 validMetaData = {META_EXT_VERSION_ONE,
2001                                            {CMD_META_CONFLICT_RES_MODE,
2002                                             htons(sizeof(uint8_t)), 0xff},
2003                                            {CMD_META_ADJUSTED_TIME,
2004                                             htons(sizeof(int64_t)), -1}};
2005        std::vector<char> validMetaVector(reinterpret_cast<char*>(&validMetaData),
2006                                          reinterpret_cast<char*>(&validMetaData) +
2007                                          sizeof(validMetaData));
2008
2009        // Set the key with a low CAS value and real nmeta
2010        set_with_meta(h, h1, "key3", 4, nullptr, 0, 0, &itemMeta, 0,
2011                      force, PROTOCOL_BINARY_RAW_BYTES,
2012                      nullptr, validMetaVector);
2013        checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
2014                "Expected success");
2015
2016        itemMeta.cas++;
2017        del_with_meta(h, h1, "key3", 4, 0, &itemMeta, 0,
2018                      force, nullptr, validMetaVector);
2019        checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
2020                "Expected success");
2021    }
2022
2023    {
2024        with_cas_metadata4 validMetaData = {META_EXT_VERSION_ONE,
2025                                            {CMD_META_ADJUSTED_TIME,
2026                                             htons(sizeof(int64_t)), -1},
2027                                            {CMD_META_CONFLICT_RES_MODE,
2028                                             htons(sizeof(uint8_t)), 0xff}};
2029        std::vector<char> validMetaVector(reinterpret_cast<char*>(&validMetaData),
2030                                          reinterpret_cast<char*>(&validMetaData) +
2031                                          sizeof(validMetaData));
2032
2033        // Set the key with a low CAS value and real nmeta
2034        set_with_meta(h, h1, "key4", 4, NULL, 0, 0, &itemMeta, 0,
2035                      force, PROTOCOL_BINARY_RAW_BYTES,
2036                      nullptr, validMetaVector);
2037        checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
2038                "Expected success");
2039
2040        itemMeta.cas++;
2041        del_with_meta(h, h1, "key4", 4, 0, &itemMeta, 0,
2042                      force, nullptr, validMetaVector);
2043        checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
2044                "Expected success");
2045    }
2046
2047    return SUCCESS;
2048}
2049
2050// Test manifest //////////////////////////////////////////////////////////////
2051
2052const char *default_dbname = "./ep_testsuite_xdcr";
2053
2054BaseTestCase testsuite_testcases[] = {
2055
2056        // XDCR unit tests
2057        TestCase("get meta", test_get_meta, test_setup,
2058                 teardown, NULL, prepare, cleanup),
2059        TestCase("get meta with extras", test_get_meta_with_extras,
2060                 test_setup, teardown, NULL, prepare, cleanup),
2061        TestCase("get meta deleted", test_get_meta_deleted,
2062                 test_setup, teardown, NULL, prepare, cleanup),
2063        TestCase("get meta nonexistent", test_get_meta_nonexistent,
2064                 test_setup, teardown, NULL, prepare, cleanup),
2065        TestCase("get meta followed by get", test_get_meta_with_get,
2066                 test_setup, teardown, NULL, prepare, cleanup),
2067        TestCase("get meta followed by set", test_get_meta_with_set,
2068                 test_setup, teardown, NULL, prepare, cleanup),
2069        TestCase("get meta followed by delete", test_get_meta_with_delete,
2070                 test_setup, teardown, NULL, prepare, cleanup),
2071        TestCase("add with meta", test_add_with_meta, test_setup,
2072                 teardown, NULL, prepare, cleanup),
2073        TestCase("delete with meta", test_delete_with_meta,
2074                 test_setup, teardown, NULL, prepare, cleanup),
2075        TestCase("delete with meta deleted", test_delete_with_meta_deleted,
2076                 test_setup, teardown, NULL, prepare, cleanup),
2077        TestCase("delete with meta nonexistent",
2078                 test_delete_with_meta_nonexistent, test_setup,
2079                 teardown, NULL, prepare, cleanup),
2080        TestCase("delete with meta nonexistent no temp",
2081                 test_delete_with_meta_nonexistent_no_temp, test_setup,
2082                 teardown, NULL, prepare, cleanup),
2083        TestCase("delete_with_meta race with concurrent delete",
2084                 test_delete_with_meta_race_with_delete, test_setup,
2085                 teardown, NULL, prepare, cleanup),
2086        TestCase("delete_with_meta race with concurrent delete",
2087                 test_delete_with_meta_race_with_delete, test_setup,
2088                 teardown, "item_eviction_policy=full_eviction",
2089                 prepare, cleanup),
2090        TestCase("delete_with_meta race with concurrent set",
2091                 test_delete_with_meta_race_with_set, test_setup,
2092                 teardown, NULL, prepare, cleanup),
2093        TestCase("set with meta", test_set_with_meta, test_setup,
2094                 teardown, NULL, prepare, cleanup),
2095        TestCase("set with meta by force", test_set_with_meta_by_force,
2096                 test_setup, teardown, NULL, prepare, cleanup),
2097        TestCase("set with meta deleted", test_set_with_meta_deleted,
2098                 test_setup, teardown, NULL, prepare, cleanup),
2099        TestCase("set with meta nonexistent", test_set_with_meta_nonexistent,
2100                 test_setup, teardown, NULL, prepare, cleanup),
2101        TestCase("set_with_meta race with concurrent set",
2102                 test_set_with_meta_race_with_set, test_setup,
2103                 teardown, NULL, prepare, cleanup),
2104        TestCase("set_with_meta race with concurrent delete",
2105                 test_set_with_meta_race_with_delete, test_setup,
2106                 teardown, NULL, prepare, cleanup),
2107        TestCase("test set_with_meta exp persisted", test_exp_persisted_set_del,
2108                 test_setup, teardown, "exp_pager_stime=3", prepare, cleanup),
2109        TestCase("test del meta conflict resolution",
2110                 test_del_meta_conflict_resolution, test_setup, teardown, NULL,
2111                 prepare, cleanup),
2112        TestCase("test add meta conflict resolution",
2113                 test_add_meta_conflict_resolution, test_setup, teardown, NULL,
2114                 prepare, cleanup),
2115        TestCase("test set meta conflict resolution",
2116                 test_set_meta_conflict_resolution, test_setup, teardown, NULL,
2117                 prepare, cleanup),
2118        TestCase("test del meta lww conflict resolution",
2119                 test_del_meta_lww_conflict_resolution, test_setup, teardown,
2120                 "conflict_resolution_type=lww",prepare, cleanup),
2121        TestCase("test set meta lww conflict resolution",
2122                 test_set_meta_lww_conflict_resolution, test_setup, teardown,
2123                 "conflict_resolution_type=lww",prepare, cleanup),
2124        TestCase("temp item deletion", test_temp_item_deletion,
2125                 test_setup, teardown,
2126                 "exp_pager_stime=1", prepare, cleanup),
2127        TestCase("test get_meta with item_eviction",
2128                 test_getMeta_with_item_eviction, test_setup, teardown,
2129                 "item_eviction_policy=full_eviction", prepare, cleanup),
2130
2131        TestCase("test set_with_meta and drift stats",
2132                 test_set_with_meta_and_check_drift_stats, test_setup,
2133                 teardown, "hlc_drift_ahead_threshold_us=5000000;"
2134                 "hlc_drift_behind_threshold_us=0;conflict_resolution_type=lww",
2135                 prepare, cleanup),
2136        TestCase("test del_with_meta and drift stats",
2137                 test_del_with_meta_and_check_drift_stats, test_setup,
2138                 teardown, "hlc_drift_ahead_threshold_us=0;"
2139                 "hlc_drift_behind_threshold_us=5000000;conflict_resolution_type=lww",
2140                 prepare, cleanup),
2141        TestCase("test setting drift threshold",
2142                 test_setting_drift_threshold, test_setup,
2143                 teardown, nullptr,
2144                 prepare, cleanup),
2145        TestCase("test CAS regeneration lww",
2146                 test_cas_regeneration, test_setup, teardown,
2147                 "conflict_resolution_type=lww",
2148                 prepare, cleanup),
2149        TestCase("test CAS regeneration seqno",
2150                 test_cas_regeneration, test_setup, teardown,
2151                 "conflict_resolution_type=seqno",
2152                 prepare, cleanup),
2153        TestCase("test CAS options and nmeta (lww)",
2154                 test_cas_options_and_nmeta, test_setup, teardown,
2155                 "conflict_resolution_type=lww",
2156                 prepare, cleanup),
2157        TestCase("test CAS options and nmeta (seqno)",
2158                 test_cas_options_and_nmeta, test_setup, teardown,
2159                 "conflict_resolution_type=seqno",
2160                 prepare, cleanup),
2161
2162        TestCase(NULL, NULL, NULL, NULL, NULL, prepare, cleanup)
2163};
2164