1/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/*
3 *     Copyright 2010 Couchbase, Inc
4 *
5 *   Licensed under the Apache License, Version 2.0 (the "License");
6 *   you may not use this file except in compliance with the License.
7 *   You may obtain a copy of the License at
8 *
9 *       http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *   Unless required by applicable law or agreed to in writing, software
12 *   distributed under the License is distributed on an "AS IS" BASIS,
13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *   See the License for the specific language governing permissions and
15 *   limitations under the License.
16 */
17
18#ifndef _FDB_TYPES_H
19#define _FDB_TYPES_H
20
21#include <stdint.h>
22#include <stddef.h>
23#ifndef _MSC_VER
24#include <stdbool.h>
25#else
26#ifndef __cplusplus
27#pragma once
28#define false (0)
29#define true (1)
30#define bool int
31#endif
32#endif
33
34/**
35 * Maximum key length supported.
36 */
37#define FDB_MAX_KEYLEN (65408) // 2^16 - 64*2 (64: max chunk size)
38/**
39 * Maximum metadata length supported.
40 */
41#define FDB_MAX_METALEN (65535UL) // 2^16 - 1
42/**
43 * Maximum value length supported.
44 */
45#define FDB_MAX_BODYLEN (4294967295UL) // 2^32 - 1
46
47// Number of latency stat types
48#define FDB_LATENCY_NUM_STATS 5
49
50#ifdef __cplusplus
51extern "C" {
52#endif
53
54/**
55 * Flags to be passed to fdb_open() API
56 */
57typedef uint32_t fdb_open_flags;
58enum {
59    /**
60     * Open a ForestDB file with read-write mode and
61     * create a new empty ForestDB file if it doesn't exist.
62     */
63    FDB_OPEN_FLAG_CREATE = 1,
64    /**
65     * Open a ForestDB file in read only mode, but
66     * return an error if a file doesn't exist.
67     */
68    FDB_OPEN_FLAG_RDONLY = 2,
69
70    /**
71     * Open a ForestDB file with legacy CRC.
72     *
73     * This flag is intended to be used by upgrade tests.
74     *
75     * This flag is only valid if the file to be opened is a new file or an
76     * existing file with legacy CRC.
77     *
78     * Opening existing files which use CRC32C with this flag results
79     * in FDB_RESULT_INVALID_ARGS.
80     */
81    FDB_OPEN_WITH_LEGACY_CRC = 4
82};
83
84/**
85 * Options to be passed to fdb_commit() API.
86 * Combinational options can be possible.
87 */
88typedef uint8_t fdb_commit_opt_t;
89enum {
90    /**
91     * Perform commit without any options.
92     */
93    FDB_COMMIT_NORMAL = 0x00,
94    /**
95     * Manually flush WAL entries even though it doesn't
96     * reach the configured threshold
97     */
98    FDB_COMMIT_MANUAL_WAL_FLUSH = 0x01
99};
100
101/**
102 * Flag to enable / disable a sequence btree.
103 */
104typedef uint8_t fdb_seqtree_opt_t;
105enum {
106    FDB_SEQTREE_NOT_USE = 0,
107    FDB_SEQTREE_USE = 1
108};
109
110/**
111 * Durability options for ForestDB.
112 */
113typedef uint8_t fdb_durability_opt_t;
114enum {
115    /**
116     * Synchronous commit through OS page cache.
117     */
118    FDB_DRB_NONE = 0x0,
119    /**
120     * Synchronous commit through the direct IO option to bypass
121     * the OS page cache.
122     */
123    FDB_DRB_ODIRECT = 0x1,
124    /**
125     * Asynchronous commit through OS page cache.
126     */
127    FDB_DRB_ASYNC = 0x2,
128    /**
129     * Asynchronous commit through the direct IO option to bypass
130     * the OS page cache.
131     */
132    FDB_DRB_ODIRECT_ASYNC = 0x3
133};
134
135/**
136 * Options for compaction mode.
137 */
138typedef uint8_t fdb_compaction_mode_t;
139enum {
140    FDB_COMPACTION_MANUAL = 0,
141    FDB_COMPACTION_AUTO = 1
142};
143
144/**
145 * Transaction isolation level.
146 * Note that both serializable and repeatable-read isolation levels are not
147 * supported at this moment. We plan to support them in the future releases.
148 */
149typedef uint8_t fdb_isolation_level_t;
150enum {
151    // FDB_ISOLATION_SERIALIZABLE = 0,
152    // FDB_ISOLATION_REPEATABLE_READ = 1,
153    /**
154     * Prevent a transaction from reading uncommitted data from other
155     * transactions.
156     */
157    FDB_ISOLATION_READ_COMMITTED = 2,
158    /**
159     * Allow a transaction to see uncommitted data from other transaction.
160     */
161    FDB_ISOLATION_READ_UNCOMMITTED = 3
162};
163
164/**
165 * Pointer type definition of a customized compare function for fixed size key.
166 */
167typedef int (*fdb_custom_cmp_fixed)(void *a, void *b);
168
169/**
170 * Pointer type definition of a customized compare function for variable length key.
171 */
172typedef int (*fdb_custom_cmp_variable)(void *a, size_t len_a,
173                                       void *b, size_t len_b);
174
175typedef uint64_t fdb_seqnum_t;
176#define FDB_SNAPSHOT_INMEM ((fdb_seqnum_t)(-1))
177
178/**
179 * ForestDB doc structure definition
180 */
181typedef struct fdb_doc_struct {
182    /**
183     * key length.
184     */
185    size_t keylen;
186    /**
187     * metadata length.
188     */
189    size_t metalen;
190    /**
191     * doc body length.
192     */
193    size_t bodylen;
194    /**
195     * actual doc size written on disk.
196     */
197    size_t size_ondisk;
198    /**
199     * Pointer to doc's key.
200     */
201    void *key;
202    /**
203     * Sequence number assigned to a doc.
204     */
205    fdb_seqnum_t seqnum;
206    /**
207     * Offset to the doc (header + key + metadata + body) on disk.
208     */
209    uint64_t offset;
210    /**
211     * Pointer to doc's metadata.
212     */
213    void *meta;
214    /**
215     * Pointer to doc's body.
216     */
217    void *body;
218    /**
219     * Is a doc deleted?
220     */
221    bool deleted;
222    /**
223     * Flags for miscellaneous doc properties.
224     */
225     uint32_t flags;
226    /**
227     * Use the seqnum set by user instead of auto-generating.
228     */
229#define FDB_CUSTOM_SEQNUM 0x01
230} fdb_doc;
231
232/**
233 * Opaque reference to a ForestDB file handle, which is exposed in public APIs.
234 */
235typedef struct _fdb_file_handle fdb_file_handle;
236
237/**
238 * Opaque reference to a ForestDB KV store handle, which is exposed in public APIs.
239 */
240typedef struct _fdb_kvs_handle fdb_kvs_handle;
241
242/**
243 * Compaction status for callback function.
244 */
245typedef uint32_t fdb_compaction_status;
246enum {
247    FDB_CS_BEGIN = 0x1,
248    FDB_CS_MOVE_DOC = 0x2,
249    FDB_CS_BATCH_MOVE = 0x4,
250    FDB_CS_FLUSH_WAL = 0x8,
251    FDB_CS_END = 0x10, // invoked at the end of every phase of compaction
252    FDB_CS_COMPLETE = 0x20 // invoked on completion of compaction
253};
254
255/**
256 * Compaction decision returned if FDB_CS_MOVE_DOC callback option is used.
257 * If this compaction callback option is used then it is upto its corresponding
258 * callback function to specify, using the given return values below, if a
259 * given document should be retained in the newly compacted file or dropped.
260 */
261typedef int fdb_compact_decision;
262enum {
263    FDB_CS_KEEP_DOC = 0x0,
264    FDB_CS_DROP_DOC = 0x1
265};
266
267/**
268 * Pointer type definition of a callback function for compaction.
269 */
270typedef fdb_compact_decision (*fdb_compaction_callback)(
271                               fdb_file_handle *fhandle,
272                               fdb_compaction_status status,
273                               const char *kv_store_name,
274                               fdb_doc *doc,
275                               uint64_t last_oldfile_offset,
276                               uint64_t last_newfile_offset,
277                               void *ctx);
278
279/**
280  * Encryption algorithms known to ForestDB.
281  */
282typedef int fdb_encryption_algorithm_t;
283enum {
284    FDB_ENCRYPTION_NONE = 0,    /**< No encryption (default) */
285    FDB_ENCRYPTION_AES256 = 1   /**< AES with 256-bit key */
286};
287
288/**
289  * File encryption key.
290  */
291typedef struct {
292    fdb_encryption_algorithm_t algorithm;
293    uint8_t bytes[32];
294} fdb_encryption_key;
295
296/**
297 * ForestDB config options that are passed to fdb_open API.
298 */
299typedef struct {
300    /**
301     * Chunk size (bytes) that is used to build B+-tree at each level.
302     * It is set to 8 bytes by default and has a min value of 4 bytes
303     * and a max value of 64 bytes.
304     * This is a local config to each ForestDB file.
305     */
306    uint16_t chunksize;
307    /**
308     * Size of block that is a unit of IO operations.
309     * It is set to 4KB by default and has a min value of 1KB and a max value of
310     * 128KB. This is a global config that is used across all ForestDB files.
311     */
312    uint32_t blocksize;
313    /**
314     * Buffer cache size in bytes. If the size is set to zero, then the buffer
315     * cache is disabled. This is a global config that is used across all
316     * ForestDB files.
317     */
318    uint64_t buffercache_size;
319    /**
320     * WAL index size threshold in memory (4096 entries by default).
321     * This is a local config to each ForestDB file.
322     */
323    uint64_t wal_threshold;
324    /**
325     * Flag to enable flushing the WAL whenever it reaches its threshold size.
326     * This reduces memory usage when a lot of data is written before a commit.
327     */
328    bool wal_flush_before_commit;
329    /**
330     * Flag to enable automatic commit.
331     * This is a local config to each ForestDB file.
332     */
333    bool auto_commit;
334    /**
335     * Interval for purging logically deleted documents in the unit of second.
336     * It is set to 0 second (purge during next compaction) by default.
337     * This is a local config to each ForestDB file.
338     */
339    uint32_t purging_interval;
340    /**
341     * Flag to enable or disable a sequence B+-Tree.
342     * This is a local config to each ForestDB file.
343     */
344    fdb_seqtree_opt_t seqtree_opt;
345    /**
346     * Flag to enable synchronous or asynchronous commit options.
347     * This is a local config to each ForestDB file.
348     */
349    fdb_durability_opt_t durability_opt;
350    /**
351     * Flags for fdb_open API. It can be used for specifying read-only mode.
352     * This is a local config to each ForestDB file.
353     */
354    fdb_open_flags flags;
355    /**
356     * Maximum size (bytes) of temporary buffer for compaction (4MB by default).
357     * This is a local config to each ForestDB file.
358     */
359    uint32_t compaction_buf_maxsize;
360    /**
361     * Destroy all the cached blocks in the global buffer cache when a ForestDB
362     * file is closed. It is set to true by default. This is a global config
363     * that is used across all ForestDB files.
364     */
365    bool cleanup_cache_onclose;
366    /**
367     * Compress the body of document when it is written on disk. The compression
368     * is disabled by default. This is a global config that is used across all
369     * ForestDB files.
370     */
371    bool compress_document_body;
372    /**
373     * Flag to enable auto compaction for the file. The auto compaction is disabled
374     * by default.
375     * This is a local config to each ForestDB file.
376     */
377    fdb_compaction_mode_t compaction_mode;
378    /**
379     * Compaction threshold in the unit of percentage (%). It can be calculated
380     * as '(stale data size)/(total file size)'. The compaction daemon triggers
381     * compaction if this threshold is satisfied.
382     * Compaction will not be performed when this value is set to zero or 100.
383     * This is a local config to each ForestDB file.
384     */
385    uint8_t compaction_threshold;
386    /**
387     * The minimum filesize to perform compaction.
388     * This is a local config to each ForestDB file.
389     */
390    uint64_t compaction_minimum_filesize;
391    /**
392     * Duration that the compaction daemon periodically wakes up, in the unit of
393     * second. This is a global config that is used across all ForestDB files.
394     */
395    uint64_t compactor_sleep_duration;
396    /**
397     * Flag to enable supporting multiple KV instances in a DB instance.
398     * This is a global config that is used across all ForestDB files.
399     */
400    bool multi_kv_instances;
401    /**
402     * Duration that prefetching of DB file will be performed when the file
403     * is opened, in the unit of second. If the duration is set to zero,
404     * prefetching is disabled. This is a local config to each ForestDB file.
405     */
406    uint64_t prefetch_duration;
407    /**
408     * Number of in-memory WAL index partitions for a DB file.
409     * This is a local config to each ForestDB file.
410     */
411    uint16_t num_wal_partitions;
412    /**
413     * Number of buffer cache partitions for each DB file.
414     * This is a local config to each ForestDB file.
415     */
416    uint16_t num_bcache_partitions;
417    /**
418     * Callback function for compaction.
419     * This is a local config to each ForestDB file.
420     */
421    fdb_compaction_callback compaction_cb;
422    /**
423     * Mask to select when to invoke callback function during compaction.
424     * Note that mask value is a combination of flags defined in
425     * fdb_compaction_status.
426     * This is a local config to each ForestDB file.
427     */
428    uint32_t compaction_cb_mask;
429    /**
430     * Auxiliary data for compaction callback function.
431     * This is a local config to each ForestDB file.
432     */
433    void *compaction_cb_ctx;
434    /**
435     * Maximum probability (range: 20% ~ 100%) for the compactor to grab
436     * the writer's lock during each batch write in case the writer's throughput
437     * is faster than the compactor, to make sure that the compactor can keep
438     * pace with the writer and eventually complete the compaction.
439     * Note that we plan to reduce the compaction overhead significantly soon
440     * and deprecate this parameter when it is not needed anymore.
441     * This is a local config to each ForestDB file.
442     */
443    size_t max_writer_lock_prob;
444    /**
445     * Number of daemon compactor threads. It is set to 4 threads by default.
446     * If many files are opened and accessed concurrently, then it is
447     * recommended to increase this value if the host machine has enough cores
448     * and disk I/O bandwidth.
449     * This is a global config that is configured across all ForestDB files.
450     */
451    size_t num_compactor_threads;
452    /**
453     * Number of background flusher threads. It is set to 4 threads by default.
454     * For write intensive workloads with large commit intervals and many files
455     * it is recommended to increase this value if the host machine has enough
456     * cores and disk I/O bandwidth.
457     * This is a global config that is configured across all ForestDB files.
458     */
459    size_t num_bgflusher_threads;
460    /**
461     * Encryption key for the database. Default value has algorithm = FDB_ENCRYPTION_NONE,
462     * i.e. no encryption. When a database file is being created, its contents will be
463     * encrypted with the given key. When a database is re-opened, the same key
464     * must be given, otherwise fdb_open will fail with error FDB_RESULT_NO_DB_HEADERS.
465     */
466    fdb_encryption_key encryption_key;
467
468} fdb_config;
469
470typedef struct {
471    /**
472     * Flag to create a new empty KV store instance in a DB instance,
473     * if it doesn't exist.
474     */
475    bool create_if_missing;
476    /**
477     * Customized compare function for an KV store instance.
478     */
479    fdb_custom_cmp_variable custom_cmp;
480} fdb_kvs_config;
481
482/**
483 * Pointer type definition of an error logging callback function.
484 */
485typedef void (*fdb_log_callback)(int err_code, const char *err_msg, void *ctx_data);
486
487/**
488 * Function pointer definition of the fatal error callback function.
489 */
490typedef void (*fdb_fatal_error_callback)(void);
491
492/**
493 * ForestDB iterator options.Combinational options can be passed to the iterator.
494 * For example, FDB_ITR_SKIP_MIN_KEY | FDB_ITR_SKIP_MAX_KEY means
495 * "The smallest and largest keys in the iteration ragne won't be returned by the
496 * iterator".
497 */
498typedef uint16_t fdb_iterator_opt_t;
499enum {
500    /**
501     * Return both key and value through iterator.
502     */
503    FDB_ITR_NONE = 0x00,
504    /**
505     * Return only non-deleted items through iterator.
506     */
507    FDB_ITR_NO_DELETES = 0x02,
508    /**
509     * The lowest key specified will not be returned by the iterator.
510     */
511    FDB_ITR_SKIP_MIN_KEY = 0x04,
512    /**
513     * The highest key specified will not be returned by the iterator.
514     */
515    FDB_ITR_SKIP_MAX_KEY = 0x08
516};
517
518/**
519 * ForestDB iterator seek options.
520 */
521typedef uint8_t fdb_iterator_seek_opt_t;
522enum {
523    /**
524     * If seek_key does not exist return the next sorted key higher than it.
525     */
526    FDB_ITR_SEEK_HIGHER = 0x00,
527    /**
528     * If seek_key does not exist return the previous sorted key lower than it.
529     */
530    FDB_ITR_SEEK_LOWER = 0x01
531};
532
533/**
534 * Opaque reference to ForestDB iterator structure definition, which is exposed
535 * in public APIs.
536 */
537typedef struct _fdb_iterator fdb_iterator;
538
539/**
540 * Using off_t turned out to be a real challenge. On "unix-like" systems
541 * its size is set by a combination of #defines like: _LARGE_FILE,
542 * _FILE_OFFSET_BITS and/or _LARGEFILE_SOURCE etc. The interesting
543 * part is however Windows.
544 *
545 * Windows follows the LLP64 data model:
546 * http://en.wikipedia.org/wiki/LLP64#64-bit_data_models
547 *
548 * This means both the int and long int types have a size of 32 bits
549 * regardless if it's a 32 or 64 bits Windows system.
550 *
551 * And Windows defines the type off_t as being a signed long integer:
552 * http://msdn.microsoft.com/en-us/library/323b6b3k.aspx
553 *
554 * This means we can't use off_t on Windows if we deal with files
555 * that can have a size of 2Gb or more.
556 */
557typedef int64_t cs_off_t;
558
559/**
560 * Information about a ForestDB file
561 */
562typedef struct {
563    /**
564     * A file name.
565     */
566    const char* filename;
567    /**
568     * A new file name that is used after compaction.
569     */
570    const char* new_filename;
571    /**
572     * Total number of non-deleted documents aggregated across all KV stores.
573     */
574    uint64_t doc_count;
575    /**
576     * Total number of deleted documents aggregated across all KV stores.
577     */
578    uint64_t deleted_count;
579    /**
580     * Disk space actively used by the file.
581     */
582    uint64_t space_used;
583    /**
584     * Total disk space used by the file, including stale btree nodes and docs.
585     */
586    uint64_t file_size;
587    /**
588     * Number of KV store instances in a ForestDB file
589     */
590    size_t num_kv_stores;
591} fdb_file_info;
592
593/**
594 * Information about a ForestDB KV store
595 */
596typedef struct {
597    /**
598     * A KV store name.
599     */
600    const char* name;
601    /**
602     * Last sequence number assigned.
603     */
604    fdb_seqnum_t last_seqnum;
605    /**
606     * Total number of non-deleted documents in a KV store.
607     */
608    uint64_t doc_count;
609    /**
610     * Total number of deleted documents in a KV store.
611     */
612    uint64_t deleted_count;
613    /**
614     * Disk space actively used by the KV store.
615     */
616    uint64_t space_used;
617    /**
618     * File handle that owns the KV store.
619     */
620    fdb_file_handle* file;
621} fdb_kvs_info;
622
623/**
624 * Information about a ForestDB KV store's operational counters
625 */
626typedef struct {
627    /**
628     * Number of fdb_set operations.
629     */
630    uint64_t num_sets;
631    /**
632     * Number of fdb_del operations.
633     */
634    uint64_t num_dels;
635    /**
636     * Number of fdb_commit operations.
637     */
638    uint64_t num_commits;
639    /**
640     * Number of fdb_compact operations on underlying file.
641     */
642    uint64_t num_compacts;
643    /**
644     * Number of fdb_get* (includes metaonly, byseq etc) operations.
645     */
646    uint64_t num_gets;
647    /**
648     * Number of fdb_iterator_get* (includes meta_only) operations.
649     */
650    uint64_t num_iterator_gets;
651    /**
652     * Number of fdb_iterator_moves (includes next,prev,seek) operations.
653     */
654    uint64_t num_iterator_moves;
655} fdb_kvs_ops_info;
656
657/**
658 * Latency stat type for each public API
659 */
660typedef uint8_t fdb_latency_stat_type;
661enum {
662    FDB_LATENCY_SETS      = 0, // fdb_set API
663    FDB_LATENCY_GETS      = 1, // fdb_get API
664    FDB_LATENCY_COMMITS   = 2, // fdb_commit API
665    FDB_LATENCY_SNAPSHOTS = 3, // fdb_snapshot_open API
666    FDB_LATENCY_COMPACTS  = 4  // fdb_compact API
667};
668
669/**
670 * Latency statistics of a specific ForestDB api call
671 */
672typedef struct {
673    /**
674     * Total number this call was invoked.
675     */
676    uint64_t lat_count;
677    /**
678     * The fastest call took this amount of time in micro seconds.
679     */
680    uint32_t lat_min;
681    /**
682     * The slowest call took this amount of time in micro seconds.
683     */
684    uint32_t lat_max;
685    /**
686     * The average time taken by this call in micro seconds.
687     */
688    uint32_t lat_avg;
689} fdb_latency_stat;
690
691/**
692 * List of ForestDB KV store names
693 */
694typedef struct {
695    /**
696     * Number of KV store names listed in kvs_names.
697     */
698    size_t num_kvs_names;
699    /**
700     * Pointer to array of KV store names.
701     */
702    char **kvs_names;
703} fdb_kvs_name_list;
704
705/**
706 * Persisted Snapshot Marker in file (Sequence number + KV Store name)
707 */
708typedef struct {
709    /**
710     * NULL-terminated KV Store name.
711     */
712    char *kv_store_name;
713    /**
714     * A Sequence number of the above KV store, which results from an
715     * fdb_commit operation.
716     */
717    fdb_seqnum_t seqnum;
718} fdb_kvs_commit_marker_t;
719
720/**
721 * An opaque file-level snapshot marker that can be used to purge
722 * stale data up to a given file-level snapshot marker.
723*/
724typedef uint64_t fdb_snapshot_marker_t;
725
726/**
727 * Snapshot Information structure for a ForestDB database file.
728 */
729typedef struct {
730    /**
731     * Opaque file-level snapshot marker that can be passed to
732     * fdb_compact_upto() api.
733     */
734    fdb_snapshot_marker_t marker;
735    /**
736     * Number of KV store snapshot markers in the kvs_markers array.
737     */
738    int64_t num_kvs_markers;
739    /**
740     * Pointer to an array of {kv_store_name, committed_seqnum} pairs.
741     */
742    fdb_kvs_commit_marker_t *kvs_markers;
743} fdb_snapshot_info_t;
744
745#ifdef __cplusplus
746}
747#endif
748
749#endif
750