1 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3  *     Copyright 2012 Couchbase, Inc
4  *
5  *   Licensed under the Apache License, Version 2.0 (the "License");
6  *   you may not use this file except in compliance with the License.
7  *   You may obtain a copy of the License at
8  *
9  *       http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *   Unless required by applicable law or agreed to in writing, software
12  *   distributed under the License is distributed on an "AS IS" BASIS,
13  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *   See the License for the specific language governing permissions and
15  *   limitations under the License.
16  */
17 
18 #include "config.h"
19 #include "ep_test_apis.h"
20 #include "ep_testsuite_common.h"
21 
22 #include <memcached/util.h>
23 #include <platform/cb_malloc.h>
24 #include <platform/platform.h>
25 #include <stdlib.h>
26 #include <string.h>
27 
28 #include <algorithm>
29 #include <iostream>
30 #include <list>
31 #include <mutex>
32 #include <sstream>
33 #include <thread>
34 
35 #include "mock/mock_dcp.h"
36 
37 template<typename T> class HistogramStats;
38 
39 // Due to the limitations of the add_stats callback (essentially we cannot pass
40 // a context into it) we instead have a single, global `vals` map. The
41 // vals_mutex is to ensure serialised modifications to this data structure.
42 std::mutex vals_mutex;
43 statistic_map vals;
44 
45 // get_stat and get_histo_stat can only be called one at a time as they use
46 // the three global variables (requested_stat_name, actual_stat_value and
47 // histogram_stat_int_value).  Therefore the two functions need to acquire a
48 // lock and keep it for the whole function duration.
49 
50 // The requested_stat_name and actual_stat_value are used in an optimized
51 // add_stats callback (add_individual_stat) which checks for one stat
52 // (and hence doesn't have to keep a map of all of them).
53 struct {
54     std::mutex mutex;
55     std::string requested_stat_name;
56     std::string actual_stat_value;
57     /* HistogramStats<T>* is supported C++14 onwards.
58      * Until then use a separate ptr for each type.
59      */
60     HistogramStats<uint64_t>* histogram_stat_int_value;
61 } get_stat_context;
62 
63 bool dump_stats = false;
64 std::atomic<protocol_binary_response_status> last_status(
65     static_cast<protocol_binary_response_status>(0));
66 std::string last_key;
67 std::string last_body;
68 std::string last_ext;
69 bool last_deleted_flag(false);
70 std::atomic<uint8_t> last_conflict_resolution_mode(static_cast<uint8_t>(-1));
71 std::atomic<uint64_t> last_cas(0);
72 std::atomic<uint8_t> last_datatype(0x00);
73 ItemMetaData last_meta;
74 std::atomic<uint64_t> last_uuid(0);
75 std::atomic<uint64_t> last_seqno(0);
76 
77 /* HistogramBinStats is used to hold a histogram bin object a histogram stat.
78    This is a class used to hold already computed stats. Hence we do not expect
79    any change once a bin object is created */
80 template<typename T>
81 class HistogramBinStats {
82 public:
HistogramBinStats(const T& s, const T& e, uint64_t count)83     HistogramBinStats(const T& s, const T& e, uint64_t count)
84         : start_(s), end_(e), count_(count) { }
85 
start() const86     T start() const {
87         return start_;
88     }
89 
end() const90     T end() const {
91         return end_;
92     }
93 
count() const94     uint64_t count() const {
95         return count_;
96     }
97 
98 private:
99     T start_;
100     T end_;
101     uint64_t count_;
102 };
103 
104 
105 /* HistogramStats is used to hold necessary info from a histogram stat.
106    Since this class used to hold already computed stats, only write apis to add
107    new bins is implemented */
108 template<typename T>
109 class HistogramStats {
110 public:
HistogramStats()111     HistogramStats() : total_count(0) {}
112 
113     /* Add a new bin */
add_bin(const T& start, const T& end, uint64_t count)114     void add_bin(const T& start, const T& end, uint64_t count) {
115         bins.push_back(HistogramBinStats<T>(start, end, count));
116         total_count += count;
117     }
118 
119     /* Num of bins in the histogram */
num_bins() const120     size_t num_bins() const {
121         return bins.size();
122     }
123 
total() const124     uint64_t total() const {
125         return total_count;
126     }
127 
128     /* Add a bin iterator when needed */
129 private:
130     /* List of all the bins in the histogram stats */
131     std::list<HistogramBinStats<T>> bins;
132     /* Total number of samples across all histogram bins */
133     uint64_t total_count;
134 };
135 
136 static void get_histo_stat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
137                                   const char *statname, const char *statkey);
138 
139 void encodeExt(char *buffer, uint32_t val);
140 void encodeWithMetaExt(char *buffer, ItemMetaData *meta);
141 
decayingSleep(useconds_t *sleepTime)142 void decayingSleep(useconds_t *sleepTime) {
143     static const useconds_t maxSleepTime = 500000;
144     usleep(*sleepTime);
145     *sleepTime = std::min(*sleepTime << 1, maxSleepTime);
146 }
147 
vb_map_response(const void *cookie, const void *map, size_t mapsize)148 ENGINE_ERROR_CODE vb_map_response(const void *cookie,
149                                   const void *map,
150                                   size_t mapsize) {
151     (void)cookie;
152     last_body.assign(static_cast<const char*>(map), mapsize);
153     return ENGINE_SUCCESS;
154 }
155 
add_response(const void *key, uint16_t keylen, const void *ext, uint8_t extlen, const void *body, uint32_t bodylen, uint8_t datatype, uint16_t status, uint64_t cas, const void *cookie)156 bool add_response(const void *key, uint16_t keylen, const void *ext,
157                   uint8_t extlen, const void *body, uint32_t bodylen,
158                   uint8_t datatype, uint16_t status, uint64_t cas,
159                   const void *cookie) {
160     (void)cookie;
161     static std::mutex m;
162     std::lock_guard<std::mutex> lg(m);
163     last_status.store(static_cast<protocol_binary_response_status>(status));
164     last_body.assign(static_cast<const char*>(body), bodylen);
165     last_ext.assign(static_cast<const char*>(ext), extlen);
166     last_key.assign(static_cast<const char*>(key), keylen);
167     last_cas.store(cas);
168     last_datatype.store(datatype);
169     return true;
170 }
171 
add_response_set_del_meta(const void *key, uint16_t keylen, const void *ext, uint8_t extlen, const void *body, uint32_t bodylen, uint8_t datatype, uint16_t status, uint64_t cas, const void *cookie)172 bool add_response_set_del_meta(const void *key, uint16_t keylen, const void *ext,
173                                uint8_t extlen, const void *body, uint32_t bodylen,
174                                uint8_t datatype, uint16_t status, uint64_t cas,
175                                const void *cookie) {
176     (void)cookie;
177     const uint8_t* ext_bytes = reinterpret_cast<const uint8_t*> (ext);
178     if (ext && extlen > 0) {
179         uint64_t vb_uuid;
180         uint64_t seqno;
181         memcpy(&vb_uuid, ext_bytes, 8);
182         memcpy(&seqno, ext_bytes + 8, 8);
183         last_uuid.store(ntohll(vb_uuid));
184         last_seqno.store(ntohll(seqno));
185     }
186 
187     return add_response(key, keylen, ext, extlen, body, bodylen, datatype,
188                         status, cas, cookie);
189 }
190 
add_response_ret_meta(const void *key, uint16_t keylen, const void *ext, uint8_t extlen, const void *body, uint32_t bodylen, uint8_t datatype, uint16_t status, uint64_t cas, const void *cookie)191 bool add_response_ret_meta(const void *key, uint16_t keylen, const void *ext,
192                            uint8_t extlen, const void *body, uint32_t bodylen,
193                            uint8_t datatype, uint16_t status, uint64_t cas,
194                            const void *cookie) {
195     (void)cookie;
196     const uint8_t* ext_bytes = reinterpret_cast<const uint8_t*> (ext);
197     if (ext && extlen == 16) {
198         memcpy(&last_meta.flags, ext_bytes, 4);
199         memcpy(&last_meta.exptime, ext_bytes + 4, 4);
200         last_meta.exptime = ntohl(last_meta.exptime);
201         uint64_t revId = 0;
202         memcpy(&revId, ext_bytes + 8, 8);
203         last_meta.revSeqno = ntohll(revId);
204         last_meta.cas = cas;
205     }
206     return add_response(key, keylen, ext, extlen, body, bodylen, datatype,
207                         status, cas, cookie);
208 }
209 
add_stats(const char* key, const uint16_t klen, const char* val, const uint32_t vlen, gsl::not_null<const void*>)210 void add_stats(const char* key,
211                const uint16_t klen,
212                const char* val,
213                const uint32_t vlen,
214                gsl::not_null<const void*>) {
215     std::string k(key, klen);
216     std::string v(val, vlen);
217 
218     if (dump_stats) {
219         std::cout << "stat[" << k << "] = " << v << std::endl;
220     }
221 
222     std::lock_guard<std::mutex> lh(vals_mutex);
223     vals[k] = v;
224 }
225 
226 /* Callback passed to engine interface `get_stats`, used by get_int_stat and
227  * friends to lookup a specific stat. If `key` matches the requested key name,
228  * then record its value in actual_stat_value.
229  */
add_individual_stat(const char* key, const uint16_t klen, const char* val, const uint32_t vlen, gsl::not_null<const void*>)230 void add_individual_stat(const char* key,
231                          const uint16_t klen,
232                          const char* val,
233                          const uint32_t vlen,
234                          gsl::not_null<const void*>) {
235     if (get_stat_context.actual_stat_value.empty() &&
236             get_stat_context.requested_stat_name.compare(
237                     0, get_stat_context.requested_stat_name.size(),
238                     key, klen) == 0) {
239         get_stat_context.actual_stat_value = std::string(val, vlen);
240     }
241 }
242 
add_individual_histo_stat(const char* key, const uint16_t klen, const char* val, const uint32_t vlen, gsl::not_null<const void*> cookie)243 void add_individual_histo_stat(const char* key,
244                                const uint16_t klen,
245                                const char* val,
246                                const uint32_t vlen,
247                                gsl::not_null<const void*> cookie) {
248     /* Convert key to string */
249     std::string key_str(key, klen);
250     size_t pos1 = key_str.find(get_stat_context.requested_stat_name);
251     if (pos1 != std::string::npos)
252     {
253         get_stat_context.actual_stat_value.append(val, vlen);
254         /* Parse start and end from the key.
255            Key is in the format task_name_START,END (backfill_tasks_20,100) */
256         pos1 += get_stat_context.requested_stat_name.length();
257         /* Find ',' to move to end of bin_start */
258         size_t pos2 = key_str.find(',', pos1);
259         if ((std::string::npos == pos2) || (pos1 >= pos2)) {
260             throw std::invalid_argument("Malformed histogram stat: " + key_str);
261         }
262         auto start = std::stoull(std::string(key_str, pos1, pos2));
263 
264         /* Move next to ',' for starting character of bin_end */
265         pos1 = pos2 + 1;
266         /* key_str ends with bin_end */
267         pos2 = key_str.length();
268         if (pos1 >= pos2) {
269             throw std::invalid_argument("Malformed histogram stat: " + key_str);
270         }
271         auto end = std::stoull(std::string(key_str, pos1, pos2));
272         get_stat_context.histogram_stat_int_value->add_bin(start, end,
273                                                            std::stoull(val));
274     }
275 }
276 
277 
encodeExt(char *buffer, uint32_t val)278 void encodeExt(char *buffer, uint32_t val) {
279     val = htonl(val);
280     memcpy(buffer, (char*)&val, sizeof(val));
281 }
282 
encodeWithMetaExt(char* buffer, uint64_t cas, uint64_t revSeqno, uint32_t flags, uint32_t exp)283 void encodeWithMetaExt(char* buffer,
284                        uint64_t cas,
285                        uint64_t revSeqno,
286                        uint32_t flags,
287                        uint32_t exp) {
288     memcpy(buffer, (char*)&flags, sizeof(flags));
289     memcpy(buffer + 4, (char*)&exp, sizeof(exp));
290     memcpy(buffer + 8, (char*)&revSeqno, sizeof(revSeqno));
291     memcpy(buffer + 16, (char*)&cas, sizeof(cas));
292 }
293 
encodeWithMetaExt(char* buffer, RawItemMetaData* meta)294 void encodeWithMetaExt(char* buffer, RawItemMetaData* meta) {
295     uint32_t flags = meta->flags;
296     uint32_t exp = htonl(meta->exptime);
297     uint64_t seqno = htonll(meta->revSeqno);
298     uint64_t cas = htonll(meta->cas);
299     encodeWithMetaExt(buffer, cas, seqno, flags, exp);
300 }
301 
encodeWithMetaExt(char* buffer, ItemMetaData* meta)302 void encodeWithMetaExt(char* buffer, ItemMetaData* meta) {
303     uint32_t flags = meta->flags;
304     uint32_t exp = htonl(meta->exptime);
305     uint64_t seqno = htonll(meta->revSeqno);
306     uint64_t cas = htonll(meta->cas);
307     encodeWithMetaExt(buffer, cas, seqno, flags, exp);
308 }
309 
createPacket(uint8_t opcode, uint16_t vbid, uint64_t cas, const char *ext, uint8_t extlen, const char *key, uint32_t keylen, const char *val, uint32_t vallen, uint8_t datatype, const char *meta, uint16_t nmeta)310 protocol_binary_request_header* createPacket(uint8_t opcode,
311                                              uint16_t vbid,
312                                              uint64_t cas,
313                                              const char *ext,
314                                              uint8_t extlen,
315                                              const char *key,
316                                              uint32_t keylen,
317                                              const char *val,
318                                              uint32_t vallen,
319                                              uint8_t datatype,
320                                              const char *meta,
321                                              uint16_t nmeta) {
322     char *pkt_raw;
323     uint32_t headerlen = sizeof(protocol_binary_request_header);
324     pkt_raw = static_cast<char*>(cb_calloc(1, headerlen + extlen + keylen + vallen + nmeta));
325     cb_assert(pkt_raw);
326     protocol_binary_request_header *req =
327         (protocol_binary_request_header*)pkt_raw;
328     req->request.opcode = opcode;
329     req->request.keylen = htons(keylen);
330     req->request.extlen = extlen;
331     req->request.vbucket = htons(vbid);
332     req->request.bodylen = htonl(keylen + vallen + extlen + nmeta);
333     req->request.cas = htonll(cas);
334     req->request.datatype = datatype;
335 
336     if (extlen > 0) {
337         memcpy(pkt_raw + headerlen, ext, extlen);
338     }
339 
340     if (keylen > 0) {
341         memcpy(pkt_raw + headerlen + extlen, key, keylen);
342     }
343 
344     if (vallen > 0) {
345         memcpy(pkt_raw + headerlen + extlen + keylen, val, vallen);
346     }
347 
348     // Extended meta: To be used for set_with_meta/del_with_meta/add_with_meta
349     if (meta && nmeta > 0) {
350         memcpy(pkt_raw + headerlen + extlen + keylen + vallen,
351                meta, nmeta);
352     }
353 
354     return req;
355 }
356 
createCheckpoint(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)357 void createCheckpoint(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
358     protocol_binary_request_header *request = createPacket(PROTOCOL_BINARY_CMD_CREATE_CHECKPOINT);
359     check(h1->unknown_command(h, NULL, request, add_response, testHarness.doc_namespace) == ENGINE_SUCCESS,
360           "Failed to create a new checkpoint.");
361     cb_free(request);
362 }
363 
del(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *key, uint64_t cas, uint16_t vbucket, const void* cookie)364 ENGINE_ERROR_CODE del(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *key,
365                       uint64_t cas, uint16_t vbucket, const void* cookie) {
366     mutation_descr_t mut_info{};
367     return del(h, h1, key, &cas, vbucket, cookie, &mut_info);
368 }
369 
del(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *key, uint64_t* cas, uint16_t vbucket, const void* cookie, mutation_descr_t* mut_info)370 ENGINE_ERROR_CODE del(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *key,
371                       uint64_t* cas, uint16_t vbucket, const void* cookie,
372                       mutation_descr_t* mut_info) {
373     bool create_cookie = false;
374     if (cookie == nullptr) {
375         cookie = testHarness.create_cookie();
376         create_cookie = true;
377     }
378 
379     auto ret = h1->remove(h,
380                           cookie,
381                           DocKey(key, testHarness.doc_namespace),
382                           *cas,
383                           vbucket,
384                           *mut_info);
385     if (create_cookie) {
386         testHarness.destroy_cookie(cookie);
387     }
388 
389     return ret;
390 }
391 
392 /** Simplified version of store for handling the common case of performing
393  * a delete with a value.
394  */
delete_with_value(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1, const void* cookie, uint64_t cas, const char* key, cb::const_char_buffer value, cb::mcbp::Datatype datatype)395 ENGINE_ERROR_CODE delete_with_value(ENGINE_HANDLE* h,
396                                     ENGINE_HANDLE_V1* h1,
397                                     const void* cookie,
398                                     uint64_t cas,
399                                     const char* key,
400                                     cb::const_char_buffer value,
401                                     cb::mcbp::Datatype datatype) {
402     auto ret = storeCasVb11(h,
403                             h1,
404                             cookie,
405                             OPERATION_SET,
406                             key,
407                             value.data(),
408                             value.size(),
409                             9258,
410                             cas,
411                             /*vb*/ 0,
412                             /*exp*/ 0,
413                             uint8_t(datatype),
414                             DocumentState::Deleted);
415     wait_for_flusher_to_settle(h, h1);
416 
417     return ENGINE_ERROR_CODE(ret.first);
418 }
419 
del_with_meta(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1, const char* key, const size_t keylen, const uint32_t vb, ItemMetaData* itemMeta, uint64_t cas_for_delete, uint32_t options, const void* cookie, const std::vector<char>& nmeta, protocol_binary_datatype_t datatype, const std::vector<char>& value)420 void del_with_meta(ENGINE_HANDLE* h,
421                    ENGINE_HANDLE_V1* h1,
422                    const char* key,
423                    const size_t keylen,
424                    const uint32_t vb,
425                    ItemMetaData* itemMeta,
426                    uint64_t cas_for_delete,
427                    uint32_t options,
428                    const void* cookie,
429                    const std::vector<char>& nmeta,
430                    protocol_binary_datatype_t datatype,
431                    const std::vector<char>& value) {
432     RawItemMetaData meta{itemMeta->cas,
433                          itemMeta->revSeqno,
434                          itemMeta->flags,
435                          itemMeta->exptime};
436     del_with_meta(h,
437                   h1,
438                   key,
439                   keylen,
440                   vb,
441                   &meta,
442                   cas_for_delete,
443                   options,
444                   cookie,
445                   nmeta,
446                   datatype,
447                   value);
448 }
449 
del_with_meta(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1, const char* key, const size_t keylen, const uint32_t vb, RawItemMetaData* itemMeta, uint64_t cas_for_delete, uint32_t options, const void* cookie, const std::vector<char>& nmeta, protocol_binary_datatype_t datatype, const std::vector<char>& value)450 void del_with_meta(ENGINE_HANDLE* h,
451                    ENGINE_HANDLE_V1* h1,
452                    const char* key,
453                    const size_t keylen,
454                    const uint32_t vb,
455                    RawItemMetaData* itemMeta,
456                    uint64_t cas_for_delete,
457                    uint32_t options,
458                    const void* cookie,
459                    const std::vector<char>& nmeta,
460                    protocol_binary_datatype_t datatype,
461                    const std::vector<char>& value) {
462     int blen = 24;
463     std::unique_ptr<char[]> ext(new char[30]);
464     std::unique_ptr<ExtendedMetaData> emd;
465 
466     encodeWithMetaExt(ext.get(), itemMeta);
467 
468     if (options) {
469         uint32_t optionsSwapped = htonl(options);
470         memcpy(ext.get() + blen, (char*)&optionsSwapped, sizeof(optionsSwapped));
471         blen += sizeof(uint32_t);
472     }
473 
474     if (nmeta.size() > 0) {
475         uint16_t nmetaSize = htons(nmeta.size());
476         memcpy(ext.get() + blen, (char*)&nmetaSize, sizeof(nmetaSize));
477         blen += sizeof(uint16_t);
478     }
479 
480     protocol_binary_request_header *pkt;
481     pkt = createPacket(PROTOCOL_BINARY_CMD_DEL_WITH_META,
482                        vb,
483                        cas_for_delete,
484                        ext.get(),
485                        blen,
486                        key,
487                        keylen,
488                        value.data(),
489                        value.size(),
490                        datatype,
491                        nmeta.data(),
492                        nmeta.size());
493 
494     check(h1->unknown_command(h, cookie, pkt, add_response_set_del_meta, testHarness.doc_namespace) == ENGINE_SUCCESS,
495           "Expected to be able to delete with meta");
496     cb_free(pkt);
497 }
498 
evict_key(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *key, uint16_t vbucketId, const char *msg, bool expectError)499 void evict_key(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *key,
500                uint16_t vbucketId, const char *msg, bool expectError) {
501     int nonResidentItems = get_int_stat(h, h1, "ep_num_non_resident");
502     int numEjectedItems = get_int_stat(h, h1, "ep_num_value_ejects");
503     protocol_binary_request_header *pkt = createPacket(PROTOCOL_BINARY_CMD_EVICT_KEY, 0, 0,
504                                                        NULL, 0, key, strlen(key));
505     pkt->request.vbucket = htons(vbucketId);
506 
507     checkeq(ENGINE_SUCCESS,
508             h1->unknown_command(h, NULL, pkt, add_response,
509                                 testHarness.doc_namespace),
510           "Failed to perform CMD_EVICT_KEY.");
511 
512     cb_free(pkt);
513     if (expectError) {
514         checkeq(PROTOCOL_BINARY_RESPONSE_KEY_EEXISTS, last_status.load(),
515                 "evict_key: expected KEY_EEXISTS when evicting key");
516     } else {
517         if (last_body != "Already ejected.") {
518             nonResidentItems++;
519             numEjectedItems++;
520         }
521         checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
522                 "evict_key: expected SUCCESS when evicting key.");
523     }
524 
525     checkeq(nonResidentItems, get_int_stat(h, h1, "ep_num_non_resident"),
526           "Incorrect number of non-resident items");
527     checkeq(numEjectedItems, get_int_stat(h, h1, "ep_num_value_ejects"),
528           "Incorrect number of ejected items");
529 
530     if (msg != NULL && last_body != msg) {
531         fprintf(stderr, "Expected evict to return ``%s'', but it returned ``%s''\n",
532                 msg, last_body.c_str());
533         abort();
534     }
535 }
536 
checkpointPersistence(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, uint64_t checkpoint_id, uint16_t vb)537 ENGINE_ERROR_CODE checkpointPersistence(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
538                                         uint64_t checkpoint_id, uint16_t vb) {
539     checkpoint_id = htonll(checkpoint_id);
540     protocol_binary_request_header *request;
541     request = createPacket(PROTOCOL_BINARY_CMD_CHECKPOINT_PERSISTENCE, vb, 0, NULL, 0, NULL, 0,
542                            (const char *)&checkpoint_id, sizeof(uint64_t));
543     ENGINE_ERROR_CODE rv = h1->unknown_command(h, NULL, request, add_response, testHarness.doc_namespace);
544     cb_free(request);
545     return rv;
546 }
547 
seqnoPersistence(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1, const void* cookie, uint16_t vbucket, uint64_t seqno)548 ENGINE_ERROR_CODE seqnoPersistence(ENGINE_HANDLE* h,
549                                    ENGINE_HANDLE_V1* h1,
550                                    const void* cookie,
551                                    uint16_t vbucket,
552                                    uint64_t seqno) {
553     seqno = htonll(seqno);
554     char buffer[8];
555     memcpy(buffer, &seqno, sizeof(uint64_t));
556     protocol_binary_request_header* request =
557         createPacket(PROTOCOL_BINARY_CMD_SEQNO_PERSISTENCE, vbucket, 0, buffer, 8);
558 
559     ENGINE_ERROR_CODE rv = h1->unknown_command(
560             h, cookie, request, add_response, testHarness.doc_namespace);
561     cb_free(request);
562     return rv;
563 }
564 
gat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char* key, uint16_t vb, uint32_t exp)565 cb::EngineErrorItemPair gat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
566                             const char* key, uint16_t vb, uint32_t exp) {
567     const auto* cookie = testHarness.create_cookie();
568     auto ret = h1->get_and_touch(
569             h, cookie, DocKey(key, testHarness.doc_namespace), vb, exp);
570     testHarness.destroy_cookie(cookie);
571 
572     if (ret.first == cb::engine_errc::success) {
573         item_info info;
574         check(h1->get_item_info(h, ret.second.get(), &info),
575               "gat Failed to get item info");
576 
577         last_body.assign((const char*)info.value[0].iov_base,
578                          info.value[0].iov_len);
579     }
580     return ret;
581 }
582 
get_item_info(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, item_info *info, const char* key, uint16_t vb)583 bool get_item_info(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, item_info *info,
584                    const char* key, uint16_t vb) {
585     auto ret = get(h, h1, NULL, key, vb);
586     if (ret.first != cb::engine_errc::success) {
587         return false;
588     }
589     if (!h1->get_item_info(h, ret.second.get(), info)) {
590         fprintf(stderr, "get_item_info failed\n");
591         return false;
592     }
593 
594     return true;
595 }
596 
get_key(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, item *i, std::string &key)597 bool get_key(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, item *i,
598              std::string &key) {
599 
600     item_info info;
601     if (!h1->get_item_info(h, i, &info)) {
602         fprintf(stderr, "get_item_info failed\n");
603         return false;
604     }
605 
606     key.assign((const char*)info.key, info.nkey);
607     return true;
608 }
609 
getl(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1, const void* cookie, const char* key, uint16_t vb, uint32_t lock_timeout)610 cb::EngineErrorItemPair getl(ENGINE_HANDLE* h,
611                              ENGINE_HANDLE_V1* h1,
612                              const void* cookie,
613                              const char* key,
614                              uint16_t vb,
615                              uint32_t lock_timeout) {
616     bool create_cookie = false;
617     if (cookie == nullptr) {
618         cookie = testHarness.create_cookie();
619         create_cookie = true;
620     }
621     auto ret = h1->get_locked(h,
622                               cookie,
623                               DocKey(key, testHarness.doc_namespace),
624                               vb,
625                               lock_timeout);
626     if (create_cookie) {
627         testHarness.destroy_cookie(cookie);
628     }
629 
630     return ret;
631 }
632 
get_meta(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1, const char* key, const void* cookie)633 bool get_meta(ENGINE_HANDLE* h,
634               ENGINE_HANDLE_V1* h1,
635               const char* key,
636               const void* cookie) {
637     cb::EngineErrorMetadataPair out;
638 
639     return get_meta(h, h1, key, out, cookie);
640 }
641 
get_meta(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1, const char* key, cb::EngineErrorMetadataPair& out, const void* cookie)642 bool get_meta(ENGINE_HANDLE* h,
643               ENGINE_HANDLE_V1* h1,
644               const char* key,
645               cb::EngineErrorMetadataPair& out,
646               const void* cookie) {
647     DocKey docKey(key, testHarness.doc_namespace);
648     bool cookie_create = false;
649     if (cookie == nullptr) {
650         cookie = testHarness.create_cookie();
651         cookie_create = true;
652     }
653 
654     out = h1->get_meta(h, cookie, docKey, /*vb*/ 0);
655 
656     if (cookie_create) {
657         testHarness.destroy_cookie(cookie);
658     }
659 
660     return out.first == cb::engine_errc::success;
661 }
662 
observe(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1, std::map<std::string, uint16_t> obskeys)663 ENGINE_ERROR_CODE observe(ENGINE_HANDLE* h,
664                           ENGINE_HANDLE_V1* h1,
665                           std::map<std::string, uint16_t> obskeys) {
666     std::stringstream value;
667     std::map<std::string, uint16_t>::iterator it;
668     for (it = obskeys.begin(); it != obskeys.end(); ++it) {
669         uint16_t vb = htons(it->second);
670         uint16_t keylen = htons(it->first.length());
671         value.write((char*) &vb, sizeof(uint16_t));
672         value.write((char*) &keylen, sizeof(uint16_t));
673         value.write(it->first.c_str(), it->first.length());
674     }
675 
676     protocol_binary_request_header *request;
677     request = createPacket(PROTOCOL_BINARY_CMD_OBSERVE, 0, 0, NULL, 0, NULL, 0,
678                            value.str().data(), value.str().length());
679 
680     auto ret = h1->unknown_command(
681             h, nullptr, request, add_response, testHarness.doc_namespace);
682     cb_free(request);
683     return ret;
684 }
685 
observe_seqno(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1, uint16_t vb_id, uint64_t uuid)686 ENGINE_ERROR_CODE observe_seqno(ENGINE_HANDLE* h,
687                                 ENGINE_HANDLE_V1* h1,
688                                 uint16_t vb_id,
689                                 uint64_t uuid) {
690     protocol_binary_request_header *request;
691     uint64_t vb_uuid = htonll(uuid);
692     std::stringstream data;
693     data.write((char *) &vb_uuid, sizeof(uint64_t));
694 
695     request = createPacket(PROTOCOL_BINARY_CMD_OBSERVE_SEQNO, vb_id, 0, NULL, 0,
696                            NULL, 0, data.str().data(), data.str().length());
697     auto ret = h1->unknown_command(
698             h, NULL, request, add_response, testHarness.doc_namespace);
699     cb_free(request);
700     return ret;
701 }
702 
get_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char* key, uint16_t vbid)703 void get_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char* key,
704                  uint16_t vbid) {
705     protocol_binary_request_header *pkt;
706     pkt = createPacket(PROTOCOL_BINARY_CMD_GET_REPLICA, vbid, 0, NULL, 0, key, strlen(key));
707     check(h1->unknown_command(h, NULL, pkt, add_response, testHarness.doc_namespace) == ENGINE_SUCCESS,
708                               "Get Replica Failed");
709     cb_free(pkt);
710 }
711 
prepare_get_replica(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, vbucket_state_t state, bool makeinvalidkey)712 protocol_binary_request_header* prepare_get_replica(ENGINE_HANDLE *h,
713                                                     ENGINE_HANDLE_V1 *h1,
714                                                     vbucket_state_t state,
715                                                     bool makeinvalidkey) {
716     uint16_t id = 0;
717     const char *key = "k0";
718     protocol_binary_request_header *pkt;
719     pkt = createPacket(PROTOCOL_BINARY_CMD_GET_REPLICA, id, 0, NULL, 0, key, strlen(key));
720 
721     if (!makeinvalidkey) {
722         check(store(h,
723                     h1,
724                     NULL,
725                     OPERATION_SET,
726                     key,
727                     "replicadata",
728                     nullptr,
729                     0,
730                     id) == ENGINE_SUCCESS,
731               "Get Replica Failed");
732 
733         check(set_vbucket_state(h, h1, id, state),
734               "Failed to set vbucket active state, Get Replica Failed");
735     }
736 
737     return pkt;
738 }
739 
set_param(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, protocol_binary_engine_param_t paramtype, const char *param, const char *val, uint16_t vb)740 bool set_param(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, protocol_binary_engine_param_t paramtype,
741                const char *param, const char *val, uint16_t vb) {
742     char ext[4];
743     protocol_binary_request_header *pkt;
744     encodeExt(ext, static_cast<uint32_t>(paramtype));
745     pkt = createPacket(PROTOCOL_BINARY_CMD_SET_PARAM, vb, 0, ext, sizeof(protocol_binary_engine_param_t), param,
746                        strlen(param), val, strlen(val));
747 
748     if (h1->unknown_command(h, NULL, pkt, add_response, testHarness.doc_namespace) != ENGINE_SUCCESS) {
749         cb_free(pkt);
750         return false;
751     }
752 
753     cb_free(pkt);
754     return last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS;
755 }
756 
set_vbucket_state(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, uint16_t vb, vbucket_state_t state)757 bool set_vbucket_state(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
758                        uint16_t vb, vbucket_state_t state) {
759 
760     char ext[4];
761     protocol_binary_request_header *pkt;
762     encodeExt(ext, static_cast<uint32_t>(state));
763     pkt = createPacket(PROTOCOL_BINARY_CMD_SET_VBUCKET, vb, 0, ext, 4);
764 
765     if (h1->unknown_command(h, NULL, pkt, add_response, testHarness.doc_namespace) != ENGINE_SUCCESS) {
766         return false;
767     }
768 
769     cb_free(pkt);
770     return last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS;
771 }
772 
get_all_vb_seqnos(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, vbucket_state_t state, const void *cookie)773 bool get_all_vb_seqnos(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
774                        vbucket_state_t state, const void *cookie) {
775     protocol_binary_request_header *pkt;
776     if (state) {
777         char ext[sizeof(vbucket_state_t)];
778         encodeExt(ext, static_cast<uint32_t>(state));
779         pkt = createPacket(PROTOCOL_BINARY_CMD_GET_ALL_VB_SEQNOS, 0, 0, ext,
780                            sizeof(vbucket_state_t));
781     } else {
782         pkt = createPacket(PROTOCOL_BINARY_CMD_GET_ALL_VB_SEQNOS);
783     }
784 
785     check(h1->unknown_command(h, cookie, pkt, add_response, testHarness.doc_namespace) ==
786           ENGINE_SUCCESS, "Error in getting all vb info");
787 
788     cb_free(pkt);
789     return last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS;
790 }
791 
verify_all_vb_seqnos(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, int vb_start, int vb_end)792 void verify_all_vb_seqnos(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
793                           int vb_start, int vb_end) {
794     const int per_vb_resp_size = sizeof(uint16_t) + sizeof(uint64_t);
795     const int high_seqno_offset = sizeof(uint16_t);
796 
797     /* Check if the total response length is as expected. We expect 10 bytes
798      (2 for vb_id + 8 for seqno) */
799     checkeq((vb_end - vb_start + 1) * per_vb_resp_size,
800             static_cast<int>(last_body.size()),
801             "Failed to get all vb info.");
802     /* Check if the contents are correct */
803     for (int i = 0; i < (vb_end - vb_start + 1); i++) {
804         /* Check for correct vb_id */
805         checkeq(static_cast<const uint16_t>(vb_start + i),
806                 ntohs(*(reinterpret_cast<const uint16_t*>(last_body.data() +
807                                                           per_vb_resp_size*i))),
808               "vb_id mismatch");
809         /* Check for correct high_seqno */
810         std::string vb_stat_seqno("vb_" + std::to_string(vb_start + i) +
811                                   ":high_seqno");
812         uint64_t high_seqno_vb =
813         get_ull_stat(h, h1, vb_stat_seqno.c_str(), "vbucket-seqno");
814         checkeq(high_seqno_vb,
815                 ntohll(*(reinterpret_cast<const uint64_t*>(last_body.data() +
816                                                            per_vb_resp_size*i +
817                                                            high_seqno_offset))),
818                 "high_seqno mismatch");
819     }
820 }
821 
store_with_meta(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, protocol_binary_command cmd, const char *key, const size_t keylen, const char *val, const size_t vallen, const uint32_t vb, ItemMetaData *itemMeta, uint64_t cas_for_store, uint32_t options, uint8_t datatype, const void *cookie, const std::vector<char>& nmeta)822 static void store_with_meta(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
823                      protocol_binary_command cmd, const char *key,
824                      const size_t keylen, const char *val, const size_t vallen,
825                      const uint32_t vb, ItemMetaData *itemMeta,
826                      uint64_t cas_for_store, uint32_t options,
827                      uint8_t datatype, const void *cookie,
828                      const std::vector<char>& nmeta) {
829     int blen = 24;
830     std::unique_ptr<char[]> ext(new char[30]);
831     std::unique_ptr<ExtendedMetaData> emd;
832 
833     encodeWithMetaExt(ext.get(), itemMeta);
834 
835     if (options) {
836         uint32_t optionsSwapped = htonl(options);
837         memcpy(ext.get() + blen, (char*)&optionsSwapped, sizeof(optionsSwapped));
838         blen += sizeof(uint32_t);
839     }
840 
841     if (nmeta.size() > 0) {
842         uint16_t nmetaSize = htons(nmeta.size());
843         memcpy(ext.get() + blen, (char*)&nmetaSize, sizeof(nmetaSize));
844         blen += sizeof(uint16_t);
845     }
846 
847     protocol_binary_request_header *pkt;
848     pkt = createPacket(cmd, vb, cas_for_store, ext.get(), blen, key, keylen,
849                        val, vallen, datatype, nmeta.data(), nmeta.size());
850 
851     check(h1->unknown_command(h, cookie, pkt, add_response_set_del_meta, testHarness.doc_namespace) == ENGINE_SUCCESS,
852           "Expected to be able to store with meta");
853     cb_free(pkt);
854 }
855 
set_with_meta(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *key, const size_t keylen, const char *val, const size_t vallen, const uint32_t vb, ItemMetaData *itemMeta, uint64_t cas_for_set, uint32_t options, uint8_t datatype, const void *cookie, const std::vector<char>& nmeta)856 void set_with_meta(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *key,
857                    const size_t keylen, const char *val, const size_t vallen,
858                    const uint32_t vb, ItemMetaData *itemMeta,
859                    uint64_t cas_for_set, uint32_t options, uint8_t datatype,
860                    const void *cookie, const std::vector<char>& nmeta) {
861     store_with_meta(h, h1, PROTOCOL_BINARY_CMD_SET_WITH_META, key, keylen, val,
862                     vallen, vb, itemMeta, cas_for_set, options, datatype,
863                     cookie, nmeta);
864 }
865 
add_with_meta(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *key, const size_t keylen, const char *val, const size_t vallen, const uint32_t vb, ItemMetaData *itemMeta, uint64_t cas_for_add, uint32_t options, uint8_t datatype, const void *cookie, const std::vector<char>& nmeta)866 void add_with_meta(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *key,
867                    const size_t keylen, const char *val, const size_t vallen,
868                    const uint32_t vb, ItemMetaData *itemMeta,
869                    uint64_t cas_for_add, uint32_t options, uint8_t datatype,
870                    const void *cookie, const std::vector<char>& nmeta) {
871     store_with_meta(h, h1, PROTOCOL_BINARY_CMD_ADD_WITH_META, key, keylen, val,
872                     vallen, vb, itemMeta, cas_for_add, options, datatype,
873                     cookie, nmeta);
874 }
875 
return_meta(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1, const char* key, const size_t keylen, const char* val, const size_t vallen, const uint32_t vb, const uint64_t cas, const uint32_t flags, const uint32_t exp, const uint32_t type, uint8_t datatype, const void* cookie)876 static ENGINE_ERROR_CODE return_meta(ENGINE_HANDLE* h,
877                                      ENGINE_HANDLE_V1* h1,
878                                      const char* key,
879                                      const size_t keylen,
880                                      const char* val,
881                                      const size_t vallen,
882                                      const uint32_t vb,
883                                      const uint64_t cas,
884                                      const uint32_t flags,
885                                      const uint32_t exp,
886                                      const uint32_t type,
887                                      uint8_t datatype,
888                                      const void* cookie) {
889     char ext[12];
890     encodeExt(ext, type);
891     encodeExt(ext + 4, flags);
892     encodeExt(ext + 8, exp);
893     protocol_binary_request_header *pkt;
894     pkt = createPacket(PROTOCOL_BINARY_CMD_RETURN_META, vb, cas, ext, 12, key, keylen, val,
895                        vallen, datatype);
896     auto ret = h1->unknown_command(
897             h, cookie, pkt, add_response_ret_meta, testHarness.doc_namespace);
898     cb_free(pkt);
899 
900     return ret;
901 }
902 
set_ret_meta(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1, const char* key, const size_t keylen, const char* val, const size_t vallen, const uint32_t vb, const uint64_t cas, const uint32_t flags, const uint32_t exp, uint8_t datatype, const void* cookie)903 ENGINE_ERROR_CODE set_ret_meta(ENGINE_HANDLE* h,
904                                ENGINE_HANDLE_V1* h1,
905                                const char* key,
906                                const size_t keylen,
907                                const char* val,
908                                const size_t vallen,
909                                const uint32_t vb,
910                                const uint64_t cas,
911                                const uint32_t flags,
912                                const uint32_t exp,
913                                uint8_t datatype,
914                                const void* cookie) {
915     return return_meta(h,
916                        h1,
917                        key,
918                        keylen,
919                        val,
920                        vallen,
921                        vb,
922                        cas,
923                        flags,
924                        exp,
925                        SET_RET_META,
926                        datatype,
927                        cookie);
928 }
929 
add_ret_meta(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1, const char* key, const size_t keylen, const char* val, const size_t vallen, const uint32_t vb, const uint64_t cas, const uint32_t flags, const uint32_t exp, uint8_t datatype, const void* cookie)930 ENGINE_ERROR_CODE add_ret_meta(ENGINE_HANDLE* h,
931                                ENGINE_HANDLE_V1* h1,
932                                const char* key,
933                                const size_t keylen,
934                                const char* val,
935                                const size_t vallen,
936                                const uint32_t vb,
937                                const uint64_t cas,
938                                const uint32_t flags,
939                                const uint32_t exp,
940                                uint8_t datatype,
941                                const void* cookie) {
942     return return_meta(h,
943                        h1,
944                        key,
945                        keylen,
946                        val,
947                        vallen,
948                        vb,
949                        cas,
950                        flags,
951                        exp,
952                        ADD_RET_META,
953                        datatype,
954                        cookie);
955 }
956 
del_ret_meta(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1, const char* key, const size_t keylen, const uint32_t vb, const uint64_t cas, const void* cookie)957 ENGINE_ERROR_CODE del_ret_meta(ENGINE_HANDLE* h,
958                                ENGINE_HANDLE_V1* h1,
959                                const char* key,
960                                const size_t keylen,
961                                const uint32_t vb,
962                                const uint64_t cas,
963                                const void* cookie) {
964     return return_meta(h,
965                        h1,
966                        key,
967                        keylen,
968                        NULL,
969                        0,
970                        vb,
971                        cas,
972                        0,
973                        0,
974                        DEL_RET_META,
975                        0x00,
976                        cookie);
977 }
978 
disable_traffic(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)979 void disable_traffic(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
980     protocol_binary_request_header *pkt = createPacket(PROTOCOL_BINARY_CMD_DISABLE_TRAFFIC);
981     check(h1->unknown_command(h, NULL, pkt, add_response, testHarness.doc_namespace) == ENGINE_SUCCESS,
982           "Failed to send data traffic command to the server");
983     check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
984           "Failed to disable data traffic");
985     cb_free(pkt);
986 }
987 
enable_traffic(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)988 void enable_traffic(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
989     protocol_binary_request_header *pkt = createPacket(PROTOCOL_BINARY_CMD_ENABLE_TRAFFIC);
990     check(h1->unknown_command(h, NULL, pkt, add_response, testHarness.doc_namespace) == ENGINE_SUCCESS,
991           "Failed to send data traffic command to the server");
992     check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
993           "Failed to enable data traffic");
994     cb_free(pkt);
995 }
996 
start_persistence(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)997 void start_persistence(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
998     if (!isPersistentBucket(h, h1)) {
999         // Nothing to do for non-persistent buckets
1000         return;
1001     }
1002 
1003     protocol_binary_request_header *pkt = createPacket(PROTOCOL_BINARY_CMD_START_PERSISTENCE);
1004     check(h1->unknown_command(h, NULL, pkt, add_response, testHarness.doc_namespace) == ENGINE_SUCCESS,
1005           "Failed to stop persistence.");
1006     check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
1007           "Error starting persistence.");
1008     cb_free(pkt);
1009 }
1010 
stop_persistence(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1011 void stop_persistence(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1012     if (!isPersistentBucket(h, h1)) {
1013         // Nothing to do for non-persistent buckets
1014         return;
1015     }
1016 
1017     useconds_t sleepTime = 128;
1018     while (true) {
1019         if (get_str_stat(h, h1, "ep_flusher_state", 0) == "running") {
1020             break;
1021         }
1022         decayingSleep(&sleepTime);
1023     }
1024 
1025     protocol_binary_request_header *pkt = createPacket(PROTOCOL_BINARY_CMD_STOP_PERSISTENCE);
1026     check(h1->unknown_command(h, NULL, pkt, add_response, testHarness.doc_namespace) == ENGINE_SUCCESS,
1027           "Failed to stop persistence.");
1028     check(last_status == PROTOCOL_BINARY_RESPONSE_SUCCESS,
1029           "Error stopping persistence.");
1030     cb_free(pkt);
1031 }
1032 
store(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const void *cookie, ENGINE_STORE_OPERATION op, const char *key, const char *value, item **outitem, uint64_t casIn, uint16_t vb, uint32_t exp, uint8_t datatype, DocumentState docState)1033 ENGINE_ERROR_CODE store(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1034                         const void *cookie, ENGINE_STORE_OPERATION op,
1035                         const char *key, const char *value, item **outitem,
1036                         uint64_t casIn, uint16_t vb, uint32_t exp,
1037                         uint8_t datatype, DocumentState docState) {
1038     auto ret = storeCasVb11(h, h1, cookie, op, key, value, strlen(value),
1039                         9258, casIn, vb, exp, datatype, docState);
1040     if (outitem) {
1041         *outitem = ret.second.release();
1042     }
1043     return ENGINE_ERROR_CODE(ret.first);
1044 }
1045 
storeCasOut(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const void *cookie, const uint16_t vb, const std::string& key, const std::string& value, const protocol_binary_datatype_t datatype, item*& out_item, uint64_t& out_cas, DocumentState docState)1046 ENGINE_ERROR_CODE storeCasOut(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1047                               const void *cookie, const uint16_t vb,
1048                               const std::string& key, const std::string& value,
1049                               const protocol_binary_datatype_t datatype,
1050                               item*& out_item, uint64_t& out_cas,
1051                               DocumentState docState) {
1052     bool create_cookie = false;
1053     if (cookie == nullptr) {
1054         cookie = testHarness.create_cookie();
1055         create_cookie = true;
1056     }
1057 
1058     auto ret = allocate(h, h1, cookie, key, value.size(), 0, 0, datatype, vb);
1059     checkeq(cb::engine_errc::success, ret.first, "Allocation failed.");
1060     item_info info;
1061     check(h1->get_item_info(h, ret.second.get(), &info),
1062           "Unable to get item_info");
1063     memcpy(info.value[0].iov_base, value.data(), value.size());
1064     ENGINE_ERROR_CODE res = h1->store(
1065             h, cookie, ret.second.get(), out_cas, OPERATION_SET, docState);
1066 
1067     if (create_cookie) {
1068         testHarness.destroy_cookie(cookie);
1069     }
1070 
1071     return res;
1072 }
1073 
storeCasVb11(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const void *cookie, ENGINE_STORE_OPERATION op, const char *key, const char *value, size_t vlen, uint32_t flags, uint64_t casIn, uint16_t vb, uint32_t exp, uint8_t datatype, DocumentState docState)1074 cb::EngineErrorItemPair storeCasVb11(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1075                                const void *cookie, ENGINE_STORE_OPERATION op,
1076                                const char *key, const char *value, size_t vlen,
1077                                uint32_t flags, uint64_t casIn,
1078                                uint16_t vb, uint32_t exp, uint8_t datatype,
1079                                DocumentState docState) {
1080     uint64_t cas = 0;
1081 
1082     auto rv = allocate(h, h1, cookie, key, vlen, flags, exp, datatype, vb);
1083     if (rv.first != cb::engine_errc::success) {
1084         return rv;
1085     }
1086     item_info info;
1087     if (!h1->get_item_info(h, rv.second.get(), &info)) {
1088         abort();
1089     }
1090 
1091     cb_assert(info.value[0].iov_len == vlen);
1092     std::copy(value, value + vlen, reinterpret_cast<char*>(info.value[0].iov_base));
1093     h1->item_set_cas(h, rv.second.get(), casIn);
1094 
1095     bool create_cookie = false;
1096     if (cookie == nullptr) {
1097         cookie = testHarness.create_cookie();
1098         create_cookie = true;
1099     }
1100 
1101     auto storeRet = h1->store(h, cookie, rv.second.get(), cas, op, docState);
1102 
1103     if (create_cookie) {
1104         testHarness.destroy_cookie(cookie);
1105     }
1106 
1107     return {cb::engine_errc(storeRet), std::move(rv.second)};
1108 }
1109 
touch(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char* key, uint16_t vb, uint32_t exp)1110 ENGINE_ERROR_CODE touch(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char* key,
1111            uint16_t vb, uint32_t exp) {
1112     const auto* cookie = testHarness.create_cookie();
1113     auto result = h1->get_and_touch(
1114             h, cookie, DocKey(key, testHarness.doc_namespace), vb, exp);
1115     testHarness.destroy_cookie(cookie);
1116 
1117     // Update the global cas value (used by some tests)
1118     if (result.first == cb::engine_errc::success) {
1119         item_info info{};
1120         check(h1->get_item_info(h, result.second.get(), &info),
1121               "Failed to get item info");
1122         last_cas.store(info.cas);
1123     }
1124 
1125     return ENGINE_ERROR_CODE(result.first);
1126 }
1127 
unl(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const void* cookie, const char* key, uint16_t vb, uint64_t cas)1128 ENGINE_ERROR_CODE unl(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1129                       const void* cookie, const char* key,
1130                       uint16_t vb, uint64_t cas) {
1131     bool create_cookie = false;
1132     if (cookie == nullptr) {
1133         cookie = testHarness.create_cookie();
1134         create_cookie = true;
1135     }
1136     auto ret = h1->unlock(
1137             h, cookie, DocKey(key, testHarness.doc_namespace), vb, cas);
1138 
1139     if (create_cookie) {
1140         testHarness.destroy_cookie(cookie);
1141     }
1142     return ret;
1143 }
1144 
compact_db(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const uint16_t vbucket_id, const uint16_t db_file_id, const uint64_t purge_before_ts, const uint64_t purge_before_seq, const uint8_t drop_deletes)1145 void compact_db(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1146                      const uint16_t vbucket_id,
1147                      const uint16_t db_file_id,
1148                      const uint64_t purge_before_ts,
1149                      const uint64_t purge_before_seq,
1150                      const uint8_t  drop_deletes) {
1151     protocol_binary_request_compact_db req;
1152     memset(&req, 0, sizeof(req));
1153     req.message.body.purge_before_ts  = htonll(purge_before_ts);
1154     req.message.body.purge_before_seq = htonll(purge_before_seq);
1155     req.message.body.drop_deletes     = drop_deletes;
1156 
1157     std::string backend = get_str_stat(h, h1, "ep_backend");
1158     const char *args = (const char *)&(req.message.body);
1159     uint32_t argslen = 24;
1160 
1161     protocol_binary_request_header* pkt =
1162             createPacket(PROTOCOL_BINARY_CMD_COMPACT_DB,
1163                          vbucket_id,
1164                          0,
1165                          args,
1166                          argslen,
1167                          NULL,
1168                          0,
1169                          NULL,
1170                          0);
1171     check(h1->unknown_command(h, NULL, pkt, add_response, testHarness.doc_namespace) == ENGINE_SUCCESS,
1172           "Failed to request compact vbucket");
1173     cb_free(pkt);
1174 }
1175 
vbucketDelete(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1, uint16_t vb, const char* args)1176 ENGINE_ERROR_CODE vbucketDelete(ENGINE_HANDLE* h,
1177                                 ENGINE_HANDLE_V1* h1,
1178                                 uint16_t vb,
1179                                 const char* args) {
1180     uint32_t argslen = args ? strlen(args) : 0;
1181     protocol_binary_request_header *pkt =
1182         createPacket(PROTOCOL_BINARY_CMD_DEL_VBUCKET, vb, 0, NULL, 0, NULL, 0,
1183                      args, argslen);
1184 
1185     auto ret = h1->unknown_command(
1186             h, NULL, pkt, add_response, testHarness.doc_namespace);
1187     cb_free(pkt);
1188 
1189     return ret;
1190 }
1191 
verify_key(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char* key, uint16_t vbucket)1192 ENGINE_ERROR_CODE verify_key(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1193                              const char* key, uint16_t vbucket) {
1194     auto rv = get(h, h1, NULL, key, vbucket);
1195     return ENGINE_ERROR_CODE(rv.first);
1196 }
1197 
get_value(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1, const void* cookie, const char* key, uint16_t vbucket, DocStateFilter state)1198 std::pair<ENGINE_ERROR_CODE, std::string> get_value(ENGINE_HANDLE* h,
1199                                                     ENGINE_HANDLE_V1* h1,
1200                                                     const void* cookie,
1201                                                     const char* key,
1202                                                     uint16_t vbucket,
1203                                                     DocStateFilter state) {
1204     auto rv = get(h, h1, cookie, key, vbucket, state);
1205     if (rv.first != cb::engine_errc::success) {
1206         return {ENGINE_ERROR_CODE(rv.first), ""};
1207     }
1208     item_info info;
1209     if (!h1->get_item_info(h, rv.second.get(), &info)) {
1210         return {ENGINE_FAILED, ""};
1211     }
1212     auto value = std::string(reinterpret_cast<char*>(info.value[0].iov_base),
1213                              info.value[0].iov_len);
1214     return make_pair(ENGINE_ERROR_CODE(rv.first), value);
1215 }
1216 
verify_vbucket_missing(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, uint16_t vb)1217 bool verify_vbucket_missing(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1218                             uint16_t vb) {
1219     const auto vbstr = "vb_" + std::to_string(vb);
1220 
1221     // Try up to three times to verify the bucket is missing.  Bucket
1222     // state changes are async.
1223     {
1224         std::lock_guard<std::mutex> lh(vals_mutex);
1225         vals.clear();
1226     }
1227 
1228     const auto* cookie = testHarness.create_cookie();
1229     check(h1->get_stats(h, cookie, {}, add_stats) == ENGINE_SUCCESS,
1230           "Failed to get stats.");
1231     testHarness.destroy_cookie(cookie);
1232 
1233     {
1234         std::lock_guard<std::mutex> lh(vals_mutex);
1235         if (vals.find(vbstr) == vals.end()) {
1236             return true;
1237         }
1238 
1239         std::cerr << "Expected bucket missing, got " <<
1240                 vals[vbstr] << std::endl;
1241     }
1242     return false;
1243 }
1244 
verify_vbucket_state(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, uint16_t vb, vbucket_state_t expected, bool mute)1245 bool verify_vbucket_state(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, uint16_t vb,
1246                           vbucket_state_t expected, bool mute) {
1247     protocol_binary_request_header *pkt;
1248     pkt = createPacket(PROTOCOL_BINARY_CMD_GET_VBUCKET, vb, 0);
1249 
1250     ENGINE_ERROR_CODE errcode = h1->unknown_command(h, NULL, pkt, add_response, testHarness.doc_namespace);
1251     cb_free(pkt);
1252     if (errcode != ENGINE_SUCCESS) {
1253         if (!mute) {
1254             fprintf(stderr, "Error code when getting vbucket %d\n", errcode);
1255         }
1256         return false;
1257     }
1258 
1259     if (last_status != PROTOCOL_BINARY_RESPONSE_SUCCESS) {
1260         if (!mute) {
1261             fprintf(stderr, "Last protocol status was %d (%s)\n",
1262                     last_status.load(),
1263                     last_body.size() > 0 ? last_body.c_str() : "unknown");
1264         }
1265         return false;
1266     }
1267 
1268     vbucket_state_t state;
1269     memcpy(&state, last_body.data(), sizeof(state));
1270     state = static_cast<vbucket_state_t>(ntohl(state));
1271     return state == expected;
1272 }
1273 
sendDcpAck(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const void* cookie, protocol_binary_command opcode, protocol_binary_response_status status, uint32_t opaque)1274 void sendDcpAck(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1275                 const void* cookie, protocol_binary_command opcode,
1276                 protocol_binary_response_status status, uint32_t opaque) {
1277     protocol_binary_response_header pkt;
1278     pkt.response.magic = PROTOCOL_BINARY_RES;
1279     pkt.response.opcode = opcode;
1280     pkt.response.status = htons(status);
1281     pkt.response.opaque = opaque;
1282 
1283     check(h1->dcp.response_handler(h, cookie, &pkt) == ENGINE_SUCCESS,
1284           "Expected success");
1285 }
1286 
1287 class engine_error : public std::exception {
1288 public:
engine_error(ENGINE_ERROR_CODE code_)1289     engine_error(ENGINE_ERROR_CODE code_)
1290         : code(code_) {}
1291 
1292     virtual const char* what() const NOEXCEPT {
1293         return "engine_error";
1294     }
1295 
1296     ENGINE_ERROR_CODE code;
1297 };
1298 
1299 /* The following set of functions get a given stat as the specified type
1300  * (int, float, unsigned long, string, bool, etc).
1301  * If the engine->get_stats() call fails, throws a engine_error exception.
1302  * If the given statname doesn't exist under the given statname, throws a
1303  * std::out_of_range exception.
1304  */
1305 template<>
get_stat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *statname, const char *statkey)1306 int get_stat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1307              const char *statname, const char *statkey) {
1308     return std::stoi(get_str_stat(h, h1, statname, statkey));
1309 }
1310 template<>
get_stat(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1, const char* statname, const char* statkey)1311 uint64_t get_stat(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1,
1312                   const char* statname, const char* statkey) {
1313     return std::stoull(get_str_stat(h, h1, statname, statkey));
1314 }
1315 
1316 template<>
get_stat(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1, const char* statname, const char* statkey)1317 bool get_stat(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1,
1318               const char* statname, const char* statkey) {
1319     return get_str_stat(h, h1, statname, statkey) == "true";
1320 }
1321 
1322 template<>
get_stat(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1, const char* statname, const char* statkey)1323 std::string get_stat(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1,
1324                      const char* statname, const char* statkey) {
1325     std::lock_guard<std::mutex> lh(get_stat_context.mutex);
1326 
1327     get_stat_context.requested_stat_name = statname;
1328     get_stat_context.actual_stat_value.clear();
1329 
1330     const auto* cookie = testHarness.create_cookie();
1331     ENGINE_ERROR_CODE err =
1332             h1->get_stats(h,
1333                           cookie,
1334                           {statkey, statkey == NULL ? 0 : strlen(statkey)},
1335                           add_individual_stat);
1336     testHarness.destroy_cookie(cookie);
1337 
1338     if (err != ENGINE_SUCCESS) {
1339         throw engine_error(err);
1340     }
1341 
1342     if (get_stat_context.actual_stat_value.empty()) {
1343         throw std::out_of_range(std::string("Failed to find requested statname '") +
1344                                 statname + "'");
1345     }
1346 
1347     // Here we are explictly forcing a copy of the object to work
1348     // around std::string copy-on-write data-race issues seen on some
1349     // versions of libstdc++ - see MB-18510 / MB-19688.
1350     return std::string(get_stat_context.actual_stat_value.begin(),
1351                        get_stat_context.actual_stat_value.end());
1352 }
1353 
1354 /// Backward-compatible functions (encode type name in function name).
get_int_stat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *statname, const char *statkey)1355 int get_int_stat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *statname,
1356              const char *statkey) {
1357     return get_stat<int>(h, h1, statname, statkey);
1358 }
1359 
get_float_stat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *statname, const char *statkey)1360 float get_float_stat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *statname,
1361                      const char *statkey) {
1362     return std::stof(get_str_stat(h, h1, statname, statkey));
1363 }
1364 
get_ul_stat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *statname, const char *statkey)1365 uint32_t get_ul_stat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *statname,
1366                       const char *statkey) {
1367     return std::stoul(get_str_stat(h, h1, statname, statkey));
1368 }
1369 
get_ull_stat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *statname, const char *statkey)1370 uint64_t get_ull_stat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *statname,
1371                       const char *statkey) {
1372     return get_stat<uint64_t>(h, h1, statname, statkey);
1373 }
1374 
get_str_stat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *statname, const char *statkey)1375 std::string get_str_stat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1376                          const char *statname, const char *statkey) {
1377     return get_stat<std::string>(h, h1, statname, statkey);
1378 }
1379 
get_bool_stat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *statname, const char *statkey)1380 bool get_bool_stat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *statname,
1381                    const char *statkey) {
1382     const auto s = get_str_stat(h, h1, statname, statkey);
1383 
1384     if (s == "true") {
1385         return true;
1386     } else if (s == "false") {
1387         return false;
1388     } else {
1389         throw std::invalid_argument("Unable to convert string '" + s + "' to type bool");
1390     }
1391 }
1392 
1393 /* Fetches the value for a given statname in the given statkey set.
1394  * @return te value of statname, or default_value if that statname was not
1395  * found.
1396  */
get_int_stat_or_default(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, int default_value, const char *statname, const char *statkey)1397 int get_int_stat_or_default(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1398                             int default_value, const char *statname,
1399                             const char *statkey) {
1400     try {
1401         return get_int_stat(h, h1, statname, statkey);
1402     } catch (std::out_of_range&) {
1403         return default_value;
1404     }
1405 }
1406 
get_histo_stat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *statname, const char *statkey, const Histo_stat_info histo_info)1407 uint64_t get_histo_stat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1408                         const char *statname, const char *statkey,
1409                         const Histo_stat_info histo_info)
1410 {
1411     std::lock_guard<std::mutex> lh(get_stat_context.mutex);
1412 
1413     get_stat_context.histogram_stat_int_value = new HistogramStats<uint64_t>();
1414     get_histo_stat(h, h1, statname, statkey);
1415 
1416     /* Get the necessary info from the histogram */
1417     uint64_t ret_val = 0;
1418     switch (histo_info) {
1419         case Histo_stat_info::TOTAL_COUNT:
1420             ret_val = get_stat_context.histogram_stat_int_value->total();
1421             break;
1422         case Histo_stat_info::NUM_BINS:
1423             ret_val =
1424                     static_cast<uint64_t>(get_stat_context.
1425                                           histogram_stat_int_value->num_bins());
1426             break;
1427     }
1428 
1429     delete get_stat_context.histogram_stat_int_value;
1430     return ret_val;
1431 }
1432 
get_histo_stat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *statname, const char *statkey)1433 static void get_histo_stat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1434                                   const char *statname, const char *statkey)
1435 {
1436     get_stat_context.requested_stat_name = statname;
1437     /* Histo stats for tasks are append as task_name_START,END.
1438        Hence append _ */
1439     get_stat_context.requested_stat_name.append("_");
1440 
1441     const auto* cookie = testHarness.create_cookie();
1442     ENGINE_ERROR_CODE err =
1443             h1->get_stats(h,
1444                           cookie,
1445                           {statkey, statkey == NULL ? 0 : strlen(statkey)},
1446                           add_individual_histo_stat);
1447     testHarness.destroy_cookie(cookie);
1448 
1449     if (err != ENGINE_SUCCESS) {
1450         throw engine_error(err);
1451     }
1452 
1453     return;
1454 }
1455 
get_all_stats(ENGINE_HANDLE *h,ENGINE_HANDLE_V1 *h1, const char *statset)1456 statistic_map get_all_stats(ENGINE_HANDLE *h,ENGINE_HANDLE_V1 *h1,
1457                             const char *statset) {
1458     {
1459         std::lock_guard<std::mutex> lh(vals_mutex);
1460         vals.clear();
1461     }
1462     const auto* cookie = testHarness.create_cookie();
1463     ENGINE_ERROR_CODE err =
1464             h1->get_stats(h,
1465                           cookie,
1466                           {statset, statset == NULL ? 0 : strlen(statset)},
1467                           add_stats);
1468     testHarness.destroy_cookie(cookie);
1469 
1470     if (err != ENGINE_SUCCESS) {
1471         throw engine_error(err);
1472     }
1473 
1474     std::lock_guard<std::mutex> lh(vals_mutex);
1475     return vals;
1476 }
1477 
verify_curr_items(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, int exp, const char *msg)1478 void verify_curr_items(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, int exp,
1479                        const char *msg) {
1480     int curr_items = get_int_stat(h, h1, "curr_items");
1481     if (curr_items != exp) {
1482         std::cerr << "Expected "<< exp << " curr_items after " << msg
1483                   << ", got " << curr_items << std::endl;
1484         abort();
1485     }
1486 }
1487 
wait_for_stat_to_be_gte(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *stat, int final, const char* stat_key, const time_t max_wait_time_in_secs)1488 void wait_for_stat_to_be_gte(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1489                              const char *stat, int final,
1490                              const char* stat_key,
1491                              const time_t max_wait_time_in_secs) {
1492     useconds_t sleepTime = 128;
1493     WaitTimeAccumulator<int> accumulator("to be greater or equal than", stat,
1494                                          stat_key, final,
1495                                          max_wait_time_in_secs);
1496     for (;;) {
1497         auto current = get_int_stat(h, h1, stat, stat_key);
1498         if (current >= final) {
1499             break;
1500         }
1501         accumulator.incrementAndAbortIfLimitReached(current, sleepTime);
1502         decayingSleep(&sleepTime);
1503     }
1504 }
1505 
wait_for_stat_to_be_lte(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *stat, int final, const char* stat_key, const time_t max_wait_time_in_secs)1506 void wait_for_stat_to_be_lte(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1507                              const char *stat, int final,
1508                              const char* stat_key,
1509                              const time_t max_wait_time_in_secs) {
1510     useconds_t sleepTime = 128;
1511     WaitTimeAccumulator<int> accumulator("to be less than or equal to", stat,
1512                                          stat_key, final,
1513                                          max_wait_time_in_secs);
1514     for (;;) {
1515         auto current = get_int_stat(h, h1, stat, stat_key);
1516         if (current <= final) {
1517             break;
1518         }
1519         accumulator.incrementAndAbortIfLimitReached(current, sleepTime);
1520         decayingSleep(&sleepTime);
1521     }
1522 }
1523 
wait_for_expired_items_to_be(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, int final, const time_t max_wait_time_in_secs)1524 void wait_for_expired_items_to_be(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1525                                   int final,
1526                                   const time_t max_wait_time_in_secs) {
1527     useconds_t sleepTime = 128;
1528     WaitTimeAccumulator<int> accumulator("to be", "expired items",
1529                                          NULL, final,
1530                                          max_wait_time_in_secs);
1531     for (;;) {
1532         auto current = get_int_stat(h, h1, "ep_expired_access") +
1533                        get_int_stat(h, h1, "ep_expired_compactor") +
1534                        get_int_stat(h, h1, "ep_expired_pager");
1535         if (current == final) {
1536             break;
1537         }
1538         accumulator.incrementAndAbortIfLimitReached(current, sleepTime);
1539         decayingSleep(&sleepTime);
1540     }
1541 }
1542 
wait_for_memory_usage_below(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, int mem_threshold, const time_t max_wait_time_in_secs)1543 void wait_for_memory_usage_below(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1544                                  int mem_threshold,
1545                                  const time_t max_wait_time_in_secs) {
1546     useconds_t sleepTime = 128;
1547     WaitTimeAccumulator<int> accumulator("to be below", "mem_used", NULL,
1548                                          mem_threshold,
1549                                          max_wait_time_in_secs);
1550     for (;;) {
1551         auto current = get_int_stat(h, h1, "mem_used");
1552         if (current <= mem_threshold) {
1553             break;
1554         }
1555         accumulator.incrementAndAbortIfLimitReached(current, sleepTime);
1556         decayingSleep(&sleepTime);
1557     }
1558 }
1559 
wait_for_warmup_complete(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1560 bool wait_for_warmup_complete(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1561     if (!isWarmupEnabled(h, h1)) {
1562         return true;
1563     }
1564 
1565     useconds_t sleepTime = 128;
1566     do {
1567         try {
1568             if (get_str_stat(h, h1, "ep_warmup_thread", "warmup") == "complete") {
1569                 return true;
1570             }
1571         } catch (engine_error&) {
1572             // If the stat call failed then the warmup stats group no longer
1573             // exists and hence warmup is complete.
1574             return true;
1575         }
1576         decayingSleep(&sleepTime);
1577     } while(true);
1578 }
1579 
wait_for_flusher_to_settle(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1580 void wait_for_flusher_to_settle(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1581     wait_for_stat_to_be(h, h1, "ep_queue_size", 0);
1582 
1583     /* check that vb backfill queue is empty as well */
1584     checkeq(0,
1585             get_int_stat(h, h1, "ep_vb_backfill_queue_size", 0),
1586             "even though disk queue is empty, vb backfill queue is not!!");
1587 
1588     if (!isPersistentBucket(h, h1)) {
1589         // We don't run flusher in non-persistent buckets
1590         return;
1591     }
1592     // We also need to to wait for any outstanding flushes to disk to
1593     // complete - specifically so when in full eviction mode we have
1594     // waited for the item counts in each vBucket to be synced with
1595     // the number of items on disk. See
1596     // EPBucket::commit().
1597     wait_for_stat_to_be(h, h1, "ep_flusher_todo", 0);
1598 }
1599 
wait_for_item_compressor_to_settle(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1)1600 void wait_for_item_compressor_to_settle(ENGINE_HANDLE* h,
1601                                         ENGINE_HANDLE_V1* h1) {
1602     int visited_count = get_int_stat(h, h1, "ep_item_compressor_num_visited");
1603 
1604     // We need to wait for at least one more run of the item compressor
1605     wait_for_stat_to_be(
1606             h, h1, "ep_item_compressor_num_visited", visited_count + 1);
1607 }
1608 
wait_for_rollback_to_finish(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)1609 void wait_for_rollback_to_finish(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1610     useconds_t sleepTime = 128;
1611     while (get_int_stat(h, h1, "ep_rollback_count") == 0) {
1612         decayingSleep(&sleepTime);
1613     }
1614 }
1615 
wait_for_persisted_value(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *key, const char *val, uint16_t vbucketId)1616 void wait_for_persisted_value(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1617                               const char *key, const char *val,
1618                               uint16_t vbucketId) {
1619 
1620     int commitNum = 0;
1621     if (isPersistentBucket(h, h1)) {
1622          commitNum = get_int_stat(h, h1, "ep_commit_num");
1623     }
1624     check(ENGINE_SUCCESS == store(h,
1625                                   h1,
1626                                   NULL,
1627                                   OPERATION_SET,
1628                                   key,
1629                                   val,
1630                                   nullptr,
1631                                   0,
1632                                   vbucketId),
1633           "Failed to store an item.");
1634 
1635     if (isPersistentBucket(h, h1)) {
1636         // Wait for persistence...
1637         wait_for_flusher_to_settle(h, h1);
1638         wait_for_stat_change(h, h1, "ep_commit_num", commitNum);
1639     }
1640 }
1641 
set_degraded_mode(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const void* cookie, bool enable)1642 void set_degraded_mode(ENGINE_HANDLE *h,
1643                        ENGINE_HANDLE_V1 *h1,
1644                        const void* cookie,
1645                        bool enable)
1646 {
1647     protocol_binary_request_header *pkt;
1648     if (enable) {
1649         pkt = createPacket(PROTOCOL_BINARY_CMD_DISABLE_TRAFFIC, 0, 0);
1650     } else {
1651         pkt = createPacket(PROTOCOL_BINARY_CMD_ENABLE_TRAFFIC, 0, 0);
1652     }
1653 
1654     ENGINE_ERROR_CODE errcode = h1->unknown_command(h, NULL, pkt, add_response, testHarness.doc_namespace);
1655     cb_free(pkt);
1656     if (errcode != ENGINE_SUCCESS) {
1657         std::cerr << "Failed to set degraded mode to " << enable
1658                   << ". api call return engine code: " << errcode << std::endl;
1659         cb_assert(false);
1660     }
1661 
1662     if (last_status != PROTOCOL_BINARY_RESPONSE_SUCCESS) {
1663         std::cerr << "Failed to set degraded mode to " << enable
1664                   << ". protocol code: " << last_status << std::endl;
1665         if (last_body.size() > 0) {
1666             std::cerr << "\tBody: [" << last_body << "]" << std::endl;
1667         }
1668 
1669         cb_assert(false);
1670     }
1671 }
1672 
abort_msg(const char *expr, const char *msg, const char *file, int line)1673 bool abort_msg(const char *expr, const char *msg, const char *file, int line) {
1674     fprintf(stderr, "%s:%d Test failed: `%s' (%s)\n",
1675             file, line, msg, expr);
1676     abort();
1677 }
1678 
1679 /* Helper function to validate the return from store() */
validate_store_resp(ENGINE_ERROR_CODE ret, int& num_items)1680 void validate_store_resp(ENGINE_ERROR_CODE ret, int& num_items)
1681 {
1682     switch (ret) {
1683         case ENGINE_SUCCESS:
1684             num_items++;
1685             break;
1686         case ENGINE_TMPFAIL:
1687             /* TMPFAIL means we are hitting high memory usage; retry */
1688             break;
1689         default:
1690             check(false,
1691                   ("write_items_upto_mem_perc: Unexpected response from "
1692                    "store(): " + std::to_string(ret)).c_str());
1693             break;
1694     }
1695 }
1696 
write_items(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1, int num_items, int start_seqno, const char* key_prefix, const char* value, uint32_t expiry, uint16_t vb, DocumentState docState)1697 void write_items(ENGINE_HANDLE* h,
1698                  ENGINE_HANDLE_V1* h1,
1699                  int num_items,
1700                  int start_seqno,
1701                  const char* key_prefix,
1702                  const char* value,
1703                  uint32_t expiry,
1704                  uint16_t vb,
1705                  DocumentState docState) {
1706     int j = 0;
1707     while (1) {
1708         if (j == num_items) {
1709             break;
1710         }
1711         std::string key(key_prefix + std::to_string(j + start_seqno));
1712         ENGINE_ERROR_CODE ret = store(h,
1713                                       h1,
1714                                       nullptr,
1715                                       OPERATION_SET,
1716                                       key.c_str(),
1717                                       value,
1718                                       nullptr,
1719                                       /*cas*/ 0,
1720                                       vb,
1721                                       expiry,
1722                                       0,
1723                                       docState);
1724         validate_store_resp(ret, j);
1725     }
1726 }
1727 
1728 /* Helper function to write unique items starting from keyXX until memory usage
1729    hits "mem_thresh_perc" (XX is start_seqno) */
write_items_upto_mem_perc(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, int mem_thresh_perc, int start_seqno, const char *key_prefix, const char *value)1730 int write_items_upto_mem_perc(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1731                               int mem_thresh_perc, int start_seqno,
1732                               const char *key_prefix, const char *value)
1733 {
1734     float maxSize = static_cast<float>(get_int_stat(h, h1, "ep_max_size",
1735                                                     "memory"));
1736     float mem_thresh = static_cast<float>(mem_thresh_perc) / (100.0);
1737     int num_items = 0;
1738     while (1) {
1739         /* Load items into server until mem_thresh_perc of the mem quota
1740          is used. Getting stats is expensive, only check every 100
1741          iterations. */
1742         if ((num_items % 100) == 0) {
1743             float memUsed = float(get_int_stat(h, h1, "mem_used", "memory"));
1744             if (memUsed > (maxSize * mem_thresh)) {
1745                 /* Persist all items written so far. */
1746                 break;
1747             }
1748         }
1749         std::string key("key" + std::to_string(num_items + start_seqno));
1750         ENGINE_ERROR_CODE ret =
1751                 store(h, h1, nullptr, OPERATION_SET, key.c_str(), "somevalue");
1752         validate_store_resp(ret, num_items);
1753     }
1754     return num_items;
1755 }
1756 
1757 
get_CAS(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const std::string& key)1758 uint64_t get_CAS(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
1759                  const std::string& key) {
1760     auto ret = get(h, h1, nullptr, key, 0);
1761     checkeq(cb::engine_errc::success, ret.first, "get_CAS: Failed to get key");
1762 
1763     item_info info;
1764     check(h1->get_item_info(h, ret.second.get(), &info),
1765           "get_CAS: Failed to get item info for key");
1766 
1767     return info.cas;
1768 }
1769 
allocate(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1, const void* cookie, const std::string& key, size_t nbytes, int flags, rel_time_t exptime, uint8_t datatype, uint16_t vb)1770 cb::EngineErrorItemPair allocate(ENGINE_HANDLE* h,
1771                                  ENGINE_HANDLE_V1* h1,
1772                                  const void* cookie,
1773                                  const std::string& key,
1774                                  size_t nbytes,
1775                                  int flags,
1776                                  rel_time_t exptime,
1777                                  uint8_t datatype,
1778                                  uint16_t vb) {
1779     bool cookie_created = false;
1780     if (cookie == nullptr) {
1781         cookie = testHarness.create_cookie();
1782         cookie_created = true;
1783     }
1784     auto ret = h1->allocate(h,
1785                             cookie,
1786                             DocKey(key, testHarness.doc_namespace),
1787                             nbytes,
1788                             flags,
1789                             exptime,
1790                             datatype,
1791                             vb);
1792     if (cookie_created) {
1793         testHarness.destroy_cookie(cookie);
1794     }
1795 
1796     return ret;
1797 }
1798 
get(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1, const void* cookie, const std::string& key, uint16_t vb, DocStateFilter documentStateFilter)1799 cb::EngineErrorItemPair get(ENGINE_HANDLE* h,
1800                             ENGINE_HANDLE_V1* h1,
1801                             const void* cookie,
1802                             const std::string& key,
1803                             uint16_t vb,
1804                             DocStateFilter documentStateFilter) {
1805     bool create_cookie = false;
1806     if (cookie == nullptr) {
1807         cookie = testHarness.create_cookie();
1808         create_cookie = true;
1809     }
1810 
1811     auto ret = h1->get(h,
1812                        cookie,
1813                        DocKey(key, testHarness.doc_namespace),
1814                        vb,
1815                        documentStateFilter);
1816 
1817     if (create_cookie) {
1818         testHarness.destroy_cookie(cookie);
1819     }
1820     return ret;
1821 }
1822 
repeat_till_true(std::function<bool()> functor, uint16_t max_repeat, std::chrono::microseconds sleepTime)1823 bool repeat_till_true(std::function<bool()> functor,
1824                       uint16_t max_repeat,
1825                       std::chrono::microseconds sleepTime) {
1826     bool fSuccess = false;
1827     do {
1828         fSuccess = functor();
1829         if (!fSuccess) {
1830             std::this_thread::sleep_for(sleepTime);
1831             max_repeat--;
1832         }
1833     } while (!fSuccess && max_repeat > 0);
1834     return fSuccess;
1835 }
1836 
reset_stats(gsl::not_null<ENGINE_HANDLE*> h)1837 void reset_stats(gsl::not_null<ENGINE_HANDLE*> h) {
1838     auto* h1 = reinterpret_cast<ENGINE_HANDLE_V1*>(h.get());
1839     const auto* cookie = testHarness.create_cookie();
1840     h1->reset_stats(h, cookie);
1841     testHarness.destroy_cookie(cookie);
1842 }
1843 
get_stats(gsl::not_null<ENGINE_HANDLE*> h, cb::const_char_buffer key, ADD_STAT callback)1844 ENGINE_ERROR_CODE get_stats(gsl::not_null<ENGINE_HANDLE*> h,
1845                             cb::const_char_buffer key,
1846                             ADD_STAT callback) {
1847     auto* h1 = reinterpret_cast<ENGINE_HANDLE_V1*>(h.get());
1848     const auto* cookie = testHarness.create_cookie();
1849     auto ret = h1->get_stats(h, cookie, key, callback);
1850     testHarness.destroy_cookie(cookie);
1851     return ret;
1852 }
1853