xref: /5.5.2/kv_engine/include/memcached/engine.h (revision 1ed4a1e8)
1 /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 #pragma once
3 #define MEMCACHED_ENGINE_H
4 
5 #include <sys/types.h>
6 #include <cstdint>
7 #include <cstdio>
8 #include <cstring>
9 #include <functional>
10 #include <memory>
11 #include <stdexcept>
12 #include <string>
13 #include <utility>
14 
15 #include <boost/optional/optional.hpp>
16 #include <gsl/gsl>
17 
18 #include "memcached/allocator_hooks.h"
19 #include "memcached/callback.h"
20 #include "memcached/collections.h"
21 #include "memcached/config_parser.h"
22 #include "memcached/dcp.h"
23 #include "memcached/dockey.h"
24 #include "memcached/engine_common.h"
25 #include "memcached/extension.h"
26 #include "memcached/protocol_binary.h"
27 #include "memcached/server_api.h"
28 #include "memcached/types.h"
29 #include "memcached/vbucket.h"
30 
31 /*! \mainpage memcached public API
32  *
33  * \section intro_sec Introduction
34  *
35  * The memcached project provides an API for providing engines as well
36  * as data definitions for those implementing the protocol in C.  This
37  * documentation will explain both to you.
38  *
39  * \section docs_sec API Documentation
40  *
41  * Jump right into <a href="modules.html">the modules docs</a> to get started.
42  *
43  * \example default_engine.cc
44  */
45 
46 /**
47  * \defgroup Engine Storage Engine API
48  * \defgroup Protex Protocol Extension API
49  * \defgroup Protocol Binary Protocol Structures
50  *
51  * \addtogroup Engine
52  * @{
53  *
54  * Most interesting here is to implement engine_interface_v1 for your
55  * engine.
56  */
57 
58 #define ENGINE_INTERFACE_VERSION 1
59 
60 
61 /**
62  * Abstract interface to an engine.
63  */
64 #ifdef WIN32
65 #undef interface
66 #endif
67 
68 /* This is typedefed in types.h */
69 struct server_handle_v1_t {
70     uint64_t interface; /**< The version number on the server structure */
71     SERVER_CORE_API* core;
72     SERVER_STAT_API* stat;
73     SERVER_CALLBACK_API* callback;
74     SERVER_LOG_API* log;
75     SERVER_COOKIE_API* cookie;
76     ALLOCATOR_HOOKS_API* alloc_hooks;
77     SERVER_DOCUMENT_API* document;
78 };
79 
80 /**
81  * The signature for the "create_instance" function exported from the module.
82  *
83  * This function should fill out an engine inteface structure according to
84  * the interface parameter (Note: it is possible to return a lower version
85  * number).
86  *
87  * @param interface The highest interface level the server supports
88  * @param get_server_api function to get the server API from
89  * @param Where to store the interface handle
90  * @return See description of ENGINE_ERROR_CODE
91  */
92 typedef ENGINE_ERROR_CODE (* CREATE_INSTANCE)(uint64_t interface,
93                                               GET_SERVER_API get_server_api,
94                                               ENGINE_HANDLE** handle);
95 
96 /**
97  * The signature for the "destroy_engine" function exported from the module.
98  *
99  * This function is called prior to closing of the module. This function should
100  * free any globally allocated resources.
101  *
102  */
103 typedef void (* DESTROY_ENGINE)(void);
104 
105 /**
106  * A unique_ptr to use with items returned from the engine interface.
107  */
108 namespace cb {
109 class ItemDeleter;
110 typedef std::unique_ptr<item, ItemDeleter> unique_item_ptr;
111 
112 using EngineErrorItemPair = std::pair<cb::engine_errc, cb::unique_item_ptr>;
113 
114 using EngineErrorMetadataPair = std::pair<engine_errc, item_info>;
115 
116 enum class StoreIfStatus {
117     Continue,
118     Fail,
119     GetItemInfo // please get me the item_info
120 };
121 
122 using StoreIfPredicate = std::function<StoreIfStatus(
123         const boost::optional<item_info>&, cb::vbucket_info)>;
124 
125 struct EngineErrorCasPair {
126     engine_errc status;
127     uint64_t cas;
128 };
129 }
130 
131 /**
132  * The different compression modes that a bucket supports
133  */
134 enum class BucketCompressionMode : uint8_t {
135     Off,     //Data will be stored as uncompressed
136     Passive, //Data will be stored as provided by the client
137     Active   //Bucket will actively try to compress stored
138              //data
139 };
140 
141 /* The default minimum compression ratio */
142 static const float default_min_compression_ratio = 1.2f;
143 
144 /* The default maximum size for a value */
145 static const size_t default_max_item_size = 20 * 1024 * 1024;
146 
147 /**
148  * Definition of the first version of the engine interface
149  */
150 typedef struct engine_interface_v1 {
151     /**
152      * Engine info.
153      */
154     struct engine_interface interface;
155 
156     /**
157      * Initialize an engine instance.
158      * This is called *after* creation, but before the engine may be used.
159      *
160      * @param handle the engine handle
161      * @param config_str configuration this engine needs to initialize itself.
162      */
163     ENGINE_ERROR_CODE(*initialize)
164     (gsl::not_null<ENGINE_HANDLE*> handle, const char* config_str);
165 
166     /**
167      * Tear down this engine.
168      *
169      * @param handle the engine handle
170      * @param force the flag indicating the force shutdown or not.
171      */
172     void (*destroy)(gsl::not_null<ENGINE_HANDLE*> handle, const bool force);
173 
174     /*
175      * Item operations.
176      */
177 
178     /**
179      * Allocate an item.
180      *
181      * @param handle the engine handle
182      * @param cookie The cookie provided by the frontend
183      * @param output variable that will receive the item
184      * @param key the item's key
185      * @param nbytes the number of bytes that will make up the
186      *        value of this item.
187      * @param flags the item's flags
188      * @param exptime the maximum lifetime of this item
189      * @param vbucket virtual bucket to request allocation from
190      *
191      * @return {cb::engine_errc::success, unique_item_ptr} if all goes well
192      */
193     cb::EngineErrorItemPair (*allocate)(gsl::not_null<ENGINE_HANDLE*> handle,
194                                         gsl::not_null<const void*> cookie,
195                                         const DocKey& key,
196                                         const size_t nbytes,
197                                         const int flags,
198                                         const rel_time_t exptime,
199                                         uint8_t datatype,
200                                         uint16_t vbucket);
201 
202     /**
203      * Allocate an item.
204      *
205      * @param handle the engine handle
206      * @param cookie The cookie provided by the frontend
207      * @param key the item's key
208      * @param nbytes the number of bytes that will make up the
209      *               value of this item.
210      * @param priv_nbytes The number of bytes in nbytes containing
211      *                    system data (and may exceed the item limit).
212      * @param flags the item's flags
213      * @param exptime the maximum lifetime of this item
214      * @param vbucket virtual bucket to request allocation from
215      * @return pair containing the item and the items information
216      * @thows cb::engine_error with:
217      *
218      *   * `cb::engine_errc::no_bucket` The client is bound to the dummy
219      *                                  `no bucket` which don't allow
220      *                                  allocations.
221      *
222      *   * `cb::engine_errc::no_memory` The bucket is full
223      *
224      *   * `cb::engine_errc::too_big` The requested memory exceeds the
225      *                                limit set for items in the bucket.
226      *
227      *   * `cb::engine_errc::disconnect` The client should be disconnected
228      *
229      *   * `cb::engine_errc::not_my_vbucket` The requested vbucket belongs
230      *                                       to someone else
231      *
232      *   * `cb::engine_errc::temporary_failure` Temporary failure, the
233      *                                          _client_ should try again
234      *
235      *   * `cb::engine_errc::too_busy` Too busy to serve the request,
236      *                                 back off and try again.
237      */
238     std::pair<cb::unique_item_ptr, item_info> (*allocate_ex)(
239             gsl::not_null<ENGINE_HANDLE*> handle,
240             gsl::not_null<const void*> cookie,
241             const DocKey& key,
242             size_t nbytes,
243             size_t priv_nbytes,
244             int flags,
245             rel_time_t exptime,
246             uint8_t datatype,
247             uint16_t vbucket);
248 
249     /**
250      * Remove an item.
251      *
252      * @param handle the engine handle
253      * @param cookie The cookie provided by the frontend
254      * @param key the key identifying the item to be removed
255      * @param vbucket the virtual bucket id
256      * @param mut_info On a successful remove write the mutation details to
257      *                 this address.
258      *
259      * @return ENGINE_SUCCESS if all goes well
260      */
261     ENGINE_ERROR_CODE(*remove)
262     (gsl::not_null<ENGINE_HANDLE*> handle,
263      gsl::not_null<const void*> cookie,
264      const DocKey& key,
265      uint64_t& cas,
266      uint16_t vbucket,
267      mutation_descr_t& mut_info);
268 
269     /**
270      * Indicate that a caller who received an item no longer needs
271      * it.
272      *
273      * @param handle the engine handle
274      * @param item the item to be released
275      */
276     void (*release)(gsl::not_null<ENGINE_HANDLE*> handle,
277                     gsl::not_null<item*> item);
278 
279     /**
280      * Retrieve an item.
281      *
282      * @param handle the engine handle
283      * @param cookie The cookie provided by the frontend
284      * @param item output variable that will receive the located item
285      * @param key the key to look up
286      * @param vbucket the virtual bucket id
287      * @param documentStateFilter The document to return must be in any of
288      *                            of these states. (If `Alive` is set, return
289      *                            KEY_ENOENT if the document in the engine
290      *                            is in another state)
291      *
292      * @return ENGINE_SUCCESS if all goes well
293      */
294     cb::EngineErrorItemPair (*get)(gsl::not_null<ENGINE_HANDLE*> handle,
295                                    gsl::not_null<const void*> cookie,
296                                    const DocKey& key,
297                                    uint16_t vbucket,
298                                    DocStateFilter documentStateFilter);
299 
300     /**
301      * Retrieve metadata for a given item.
302      *
303      * @param handle the engine handle
304      * @param cookie The cookie provided by the frontend
305      * @param key the key to look up
306      * @param vbucket the virtual bucket id
307      *
308      * @return  Pair (ENGINE_SUCCESS, Metadata) if all goes well
309      */
310     cb::EngineErrorMetadataPair (*get_meta)(
311             gsl::not_null<ENGINE_HANDLE*> handle,
312             gsl::not_null<const void*> cookie,
313             const DocKey& key,
314             uint16_t vbucket);
315 
316     /**
317      * Optionally retrieve an item. Only non-deleted items may be fetched
318      * through this interface (Documents in deleted state may be evicted
319      * from memory and we don't want to go to disk in order to fetch these)
320      *
321      * @param handle the engine handle
322      * @param cookie The cookie provided by the frontend
323      * @param key the key to look up
324      * @param vbucket the virtual bucket id
325      * @param filter callback filter to see if the item should be returned
326      *               or not. If filter returns false the item should be
327      *               skipped.
328      *               Note: the filter is applied only to the *metadata* of the
329      *               item_info - i.e. the `value` should not be expected to be
330      *               present when filter is invoked.
331      * @return A pair of the error code and (optionally) the item
332      */
333     cb::EngineErrorItemPair (*get_if)(
334             gsl::not_null<ENGINE_HANDLE*> handle,
335             gsl::not_null<const void*> cookie,
336             const DocKey& key,
337             uint16_t vbucket,
338             std::function<bool(const item_info&)> filter);
339 
340     /**
341      * Lock and Retrieve an item.
342      *
343      * @param handle the engine handle
344      * @param cookie The cookie provided by the frontend
345      * @param item output variable that will receive the located item
346      * @param key the key to look up
347      * @param vbucket the virtual bucket id
348      * @param lock_timeout the number of seconds to hold the lock
349      *                     (0 == use the engines default lock time)
350      *
351      * @return ENGINE_SUCCESS if all goes well
352      */
353     cb::EngineErrorItemPair (*get_locked)(gsl::not_null<ENGINE_HANDLE*> handle,
354                                           gsl::not_null<const void*> cookie,
355                                           const DocKey& key,
356                                           uint16_t vbucket,
357                                           uint32_t lock_timeout);
358 
359     /**
360      * Get and update the expiry time for the document
361      *
362      * @param handle the engine handle
363      * @param cookie The cookie provided by the frontend
364      * @param key the key to look up
365      * @param vbucket the virtual bucket id
366      * @param expirytime the new expiry time for the object
367      * @return A pair of the error code and (optionally) the item
368      */
369     cb::EngineErrorItemPair (*get_and_touch)(
370             gsl::not_null<ENGINE_HANDLE*> handle,
371             gsl::not_null<const void*> cookie,
372             const DocKey& key,
373             uint16_t vbucket,
374             uint32_t expirytime);
375 
376     /**
377      * Unlock an item.
378      *
379      * @param handle the engine handle
380      * @param cookie The cookie provided by the frontend
381      * @param key the key to look up
382      * @param vbucket the virtual bucket id
383      * @param cas the cas value for the locked item
384      *
385      * @return ENGINE_SUCCESS if all goes well
386      */
387     ENGINE_ERROR_CODE(*unlock)
388     (gsl::not_null<ENGINE_HANDLE*> handle,
389      gsl::not_null<const void*> cookie,
390      const DocKey& key,
391      uint16_t vbucket,
392      uint64_t cas);
393 
394     /**
395      * Store an item into the underlying engine with the given
396      * state. If the DocumentState is set to DocumentState::Deleted
397      * the document shall not be returned unless explicitly asked for
398      * documents in that state, and the underlying engine may choose to
399      * purge it whenever it please.
400      *
401      * @param handle the engine handle
402      * @param cookie The cookie provided by the frontend
403      * @param item the item to store
404      * @param cas the CAS value for conditional sets
405      * @param operation the type of store operation to perform.
406      * @param document_state The state the document should have after
407      *                       the update
408      *
409      * @return ENGINE_SUCCESS if all goes well
410      */
411     ENGINE_ERROR_CODE(*store)
412     (gsl::not_null<ENGINE_HANDLE*> handle,
413      gsl::not_null<const void*> cookie,
414      gsl::not_null<item*> item,
415      uint64_t& cas,
416      ENGINE_STORE_OPERATION operation,
417      DocumentState document_state);
418 
419     /**
420      * Store an item into the underlying engine with the given
421      * state only if the predicate argument returns true when called against an
422      * existing item.
423      *
424      * @param handle the engine handle
425      * @param cookie The cookie provided by the frontend
426      * @param item the item to store
427      * @param cas the CAS value for conditional sets
428      * @param operation the type of store operation to perform.
429      * @param predicate a function that will be called from the engine the
430      *                  result of which determines how the store behaves.
431      *                  The function is given any existing item's item_info (as
432      *                  a boost::optional) and a cb::vbucket_info object. In the
433      *                  case that the optional item_info is not initialised the
434      *                  function can return cb::StoreIfStatus::GetInfo to
435      *                  request that the engine tries to get the item_info, the
436      *                  engine may ignore this return code if it knows better
437      *                  i.e. a memory only engine and no item_info can be
438      *                  fetched. The function can also return ::Fail if it
439      *                  wishes to fail the store_if (returning predicate_failed)
440      *                  or the predicate can return ::Continue and the rest of
441      *                  the store_if will execute (and possibly fail for other
442      *                  reasons).
443      * @param document_state The state the document should have after
444      *                       the update
445      *
446      * @return a std::pair containing the engine_error code and new CAS
447      */
448     cb::EngineErrorCasPair (*store_if)(gsl::not_null<ENGINE_HANDLE*> handle,
449                                        gsl::not_null<const void*> cookie,
450                                        gsl::not_null<item*> item,
451                                        uint64_t cas,
452                                        ENGINE_STORE_OPERATION operation,
453                                        cb::StoreIfPredicate predicate,
454                                        DocumentState document_state);
455 
456     /**
457      * Flush the cache.
458      *
459      * @param handle the engine handle
460      * @param cookie The cookie provided by the frontend
461      *
462      * @return ENGINE_SUCCESS if all goes well
463      */
464     ENGINE_ERROR_CODE(*flush)
465     (gsl::not_null<ENGINE_HANDLE*> handle, gsl::not_null<const void*> cookie);
466 
467     /*
468      * Statistics
469      */
470 
471     /**
472      * Get statistics from the engine.
473      *
474      * @param handle the engine handle
475      * @param cookie The cookie provided by the frontend
476      * @param key optional argument to stats
477      * @param add_stat callback to feed results to the output
478      *
479      * @return ENGINE_SUCCESS if all goes well
480      */
481     ENGINE_ERROR_CODE(*get_stats)
482     (gsl::not_null<ENGINE_HANDLE*> handle,
483      gsl::not_null<const void*> cookie,
484      cb::const_char_buffer key,
485      ADD_STAT add_stat);
486 
487     /**
488      * Reset the stats.
489      *
490      * @param handle the engine handle
491      * @param cookie The cookie provided by the frontend
492      */
493     void (*reset_stats)(gsl::not_null<ENGINE_HANDLE*> handle,
494                         gsl::not_null<const void*> cookie);
495 
496     /**
497      * Any unknown command will be considered engine specific.
498      *
499      * @param handle the engine handle
500      * @param cookie The cookie provided by the frontend
501      * @param request pointer to request header to be filled in
502      * @param response function to transmit data
503      * @param doc_namespace namespace the command applies to
504      *
505      * @return ENGINE_SUCCESS if all goes well
506      */
507     ENGINE_ERROR_CODE(*unknown_command)
508     (gsl::not_null<ENGINE_HANDLE*> handle,
509      const void* cookie,
510      gsl::not_null<protocol_binary_request_header*> request,
511      ADD_RESPONSE response,
512      DocNamespace doc_namespace);
513 
514     /**
515      * Set the CAS id on an item.
516      */
517     void (*item_set_cas)(gsl::not_null<ENGINE_HANDLE*> handle,
518                          gsl::not_null<item*> item,
519                          uint64_t cas);
520 
521     /**
522      * Set the data type on an item.
523      */
524     void (*item_set_datatype)(gsl::not_null<ENGINE_HANDLE*> handle,
525                               gsl::not_null<item*> item,
526                               protocol_binary_datatype_t datatype);
527 
528     /**
529      * Get information about an item.
530      *
531      * The loader of the module may need the pointers to the actual data within
532      * an item. Instead of having to create multiple functions to get each
533      * individual item, this function will get all of them.
534      *
535      * @param handle the engine that owns the object
536      * @param cookie connection cookie for this item
537      * @param item the item to request information about
538      * @param item_info
539      * @return true if successful
540      */
541     bool (*get_item_info)(gsl::not_null<ENGINE_HANDLE*> handle,
542                           gsl::not_null<const item*> item,
543                           gsl::not_null<item_info*> item_info);
544 
545     /**
546      * Set information of an item.
547      *
548      * Set updated item information.
549      *
550      * @param handle the engine that owns the object
551      * @param cookie connection cookie for this item
552      * @param item the item who's information is to be updated
553      * @param item_info
554      * @return true if successful
555      */
556     bool (*set_item_info)(gsl::not_null<ENGINE_HANDLE*> handle,
557                           gsl::not_null<item*> item,
558                           gsl::not_null<const item_info*> itm_info);
559 
560     struct dcp_interface dcp;
561 
562     /**
563      * Set the current log level
564      *
565      * @param handle the engine handle
566      * @param level the current log level
567      */
568     void (*set_log_level)(gsl::not_null<ENGINE_HANDLE*> handle,
569                           EXTENSION_LOG_LEVEL level);
570 
571     collections_interface collections;
572 
573     /**
574      * @param handle the engine handle
575      * @returns if XATTRs are enabled for this bucket
576      */
577     bool (*isXattrEnabled)(gsl::not_null<ENGINE_HANDLE*> handle);
578 
579     /**
580      * @param handle the engine handle
581      * @returns the compression mode of the bucket
582      */
583     BucketCompressionMode (*getCompressionMode)(gsl::not_null<ENGINE_HANDLE*> handle);
584 
585     /**
586      * @param handle the engine handle
587      * @returns the maximum item size supported by the bucket
588      */
589     size_t (*getMaxItemSize)(gsl::not_null<ENGINE_HANDLE*> handle);
590 
591     /**
592      * @param handle the engine handle
593      * @returns the minimum compression ratio defined in the bucket
594      */
595     float (*getMinCompressionRatio)(gsl::not_null<ENGINE_HANDLE*> handle);
596 
597 } ENGINE_HANDLE_V1;
598 
599 namespace cb {
600 class ItemDeleter {
601 public:
ItemDeleter()602     ItemDeleter() : handle(nullptr) {}
603 
604     /**
605      * Create a new instance of the item deleter.
606      *
607      * @param handle_ the handle to the the engine who owns the item
608      */
ItemDeleter(ENGINE_HANDLE * handle_)609     ItemDeleter(ENGINE_HANDLE* handle_)
610         : handle(handle_) {
611         if (handle == nullptr) {
612             throw std::invalid_argument(
613                 "cb::ItemDeleter: engine handle cannot be nil");
614         }
615     }
616 
617     /**
618      * Create a copy constructor to allow us to use std::move of the item
619      */
ItemDeleter(const ItemDeleter & other)620     ItemDeleter(const ItemDeleter& other) : handle(other.handle) {
621     }
622 
operator()623     void operator()(item* item) {
624         if (handle) {
625             auto* v1 = reinterpret_cast<ENGINE_HANDLE_V1*>(handle);
626             v1->release(handle, item);
627         } else {
628             throw std::invalid_argument("cb::ItemDeleter: item attempted to be "
629                                         "freed by null engine handle");
630         }
631     }
632 
633 private:
634     ENGINE_HANDLE* handle;
635 };
636 
makeEngineErrorItemPair(cb::engine_errc err)637 inline EngineErrorItemPair makeEngineErrorItemPair(cb::engine_errc err) {
638     return {err, unique_item_ptr{nullptr, ItemDeleter{}}};
639 }
640 
makeEngineErrorItemPair(cb::engine_errc err,item * it,ENGINE_HANDLE * handle)641 inline EngineErrorItemPair makeEngineErrorItemPair(cb::engine_errc err,
642                                                    item* it,
643                                                    ENGINE_HANDLE* handle) {
644     return {err, unique_item_ptr{it, ItemDeleter{handle}}};
645 }
646 }
647 
648 /**
649  * @}
650  */
651