1/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/*
3 *     Copyright 2010 Couchbase, Inc
4 *
5 *   Licensed under the Apache License, Version 2.0 (the "License");
6 *   you may not use this file except in compliance with the License.
7 *   You may obtain a copy of the License at
8 *
9 *       http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *   Unless required by applicable law or agreed to in writing, software
12 *   distributed under the License is distributed on an "AS IS" BASIS,
13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *   See the License for the specific language governing permissions and
15 *   limitations under the License.
16 */
17
18#ifndef _FDB_TYPES_H
19#define _FDB_TYPES_H
20
21#include <stdint.h>
22#include <stddef.h>
23#ifndef _MSC_VER
24#include <stdbool.h>
25#else
26#ifndef __cplusplus
27#pragma once
28#define false (0)
29#define true (1)
30#define bool int
31#endif
32#endif
33
34/**
35 * Maximum key length supported.
36 */
37#define FDB_MAX_KEYLEN (65408) // 2^16 - 64*2 (64: max chunk size)
38/**
39 * Maximum metadata length supported.
40 */
41#define FDB_MAX_METALEN (65535UL) // 2^16 - 1
42/**
43 * Maximum value length supported.
44 */
45#define FDB_MAX_BODYLEN (4294967295UL) // 2^32 - 1
46
47#ifdef __cplusplus
48extern "C" {
49#endif
50
51/**
52 * Flags to be passed to fdb_open() API
53 */
54typedef uint32_t fdb_open_flags;
55enum {
56    /**
57     * Open a ForestDB file with read-write mode and
58     * create a new empty ForestDB file if it doesn't exist.
59     */
60    FDB_OPEN_FLAG_CREATE = 1,
61    /**
62     * Open a ForestDB file in read only mode, but
63     * return an error if a file doesn't exist.
64     */
65    FDB_OPEN_FLAG_RDONLY = 2,
66
67    /**
68     * Open a ForestDB file with legacy CRC.
69     *
70     * This flag is intended to be used by upgrade tests.
71     *
72     * This flag is only valid if the file to be opened is a new file or an
73     * existing file with legacy CRC.
74     *
75     * Opening existing files which use CRC32C with this flag results
76     * in FDB_RESULT_INVALID_ARGS.
77     */
78    FDB_OPEN_WITH_LEGACY_CRC = 4
79};
80
81/**
82 * Options to be passed to fdb_commit() API.
83 * Combinational options can be possible.
84 */
85typedef uint8_t fdb_commit_opt_t;
86enum {
87    /**
88     * Perform commit without any options.
89     */
90    FDB_COMMIT_NORMAL = 0x00,
91    /**
92     * Manually flush WAL entries even though it doesn't
93     * reach the configured threshold
94     */
95    FDB_COMMIT_MANUAL_WAL_FLUSH = 0x01
96};
97
98/**
99 * Flag to enable / disable a sequence btree.
100 */
101typedef uint8_t fdb_seqtree_opt_t;
102enum {
103    FDB_SEQTREE_NOT_USE = 0,
104    FDB_SEQTREE_USE = 1
105};
106
107/**
108 * Durability options for ForestDB.
109 */
110typedef uint8_t fdb_durability_opt_t;
111enum {
112    /**
113     * Synchronous commit through OS page cache.
114     */
115    FDB_DRB_NONE = 0x0,
116    /**
117     * Synchronous commit through the direct IO option to bypass
118     * the OS page cache.
119     */
120    FDB_DRB_ODIRECT = 0x1,
121    /**
122     * Asynchronous commit through OS page cache.
123     */
124    FDB_DRB_ASYNC = 0x2,
125    /**
126     * Asynchronous commit through the direct IO option to bypass
127     * the OS page cache.
128     */
129    FDB_DRB_ODIRECT_ASYNC = 0x3
130};
131
132/**
133 * Options for compaction mode.
134 */
135typedef uint8_t fdb_compaction_mode_t;
136enum {
137    FDB_COMPACTION_MANUAL = 0,
138    FDB_COMPACTION_AUTO = 1
139};
140
141/**
142 * Transaction isolation level.
143 * Note that both serializable and repeatable-read isolation levels are not
144 * supported at this moment. We plan to support them in the future releases.
145 */
146typedef uint8_t fdb_isolation_level_t;
147enum {
148    // FDB_ISOLATION_SERIALIZABLE = 0,
149    // FDB_ISOLATION_REPEATABLE_READ = 1,
150    /**
151     * Prevent a transaction from reading uncommitted data from other
152     * transactions.
153     */
154    FDB_ISOLATION_READ_COMMITTED = 2,
155    /**
156     * Allow a transaction to see uncommitted data from other transaction.
157     */
158    FDB_ISOLATION_READ_UNCOMMITTED = 3
159};
160
161/**
162 * Pointer type definition of a customized compare function for fixed size key.
163 */
164typedef int (*fdb_custom_cmp_fixed)(void *a, void *b);
165
166/**
167 * Pointer type definition of a customized compare function for variable length key.
168 */
169typedef int (*fdb_custom_cmp_variable)(void *a, size_t len_a,
170                                       void *b, size_t len_b);
171
172typedef uint64_t fdb_seqnum_t;
173#define FDB_SNAPSHOT_INMEM ((fdb_seqnum_t)(-1))
174
175/**
176 * ForestDB doc structure definition
177 */
178typedef struct fdb_doc_struct {
179    /**
180     * key length.
181     */
182    size_t keylen;
183    /**
184     * metadata length.
185     */
186    size_t metalen;
187    /**
188     * doc body length.
189     */
190    size_t bodylen;
191    /**
192     * actual doc size written on disk.
193     */
194    size_t size_ondisk;
195    /**
196     * Pointer to doc's key.
197     */
198    void *key;
199    /**
200     * Sequence number assigned to a doc.
201     */
202    fdb_seqnum_t seqnum;
203    /**
204     * Offset to the doc (header + key + metadata + body) on disk.
205     */
206    uint64_t offset;
207    /**
208     * Pointer to doc's metadata.
209     */
210    void *meta;
211    /**
212     * Pointer to doc's body.
213     */
214    void *body;
215    /**
216     * Is a doc deleted?
217     */
218    bool deleted;
219    /**
220     * Flags for miscellaneous doc properties.
221     */
222     uint32_t flags;
223    /**
224     * Use the seqnum set by user instead of auto-generating.
225     */
226#define FDB_CUSTOM_SEQNUM 0x01
227} fdb_doc;
228
229/**
230 * Opaque reference to a ForestDB file handle, which is exposed in public APIs.
231 */
232typedef struct _fdb_file_handle fdb_file_handle;
233
234/**
235 * Opaque reference to a ForestDB KV store handle, which is exposed in public APIs.
236 */
237typedef struct _fdb_kvs_handle fdb_kvs_handle;
238
239/**
240 * Compaction status for callback function.
241 */
242typedef uint32_t fdb_compaction_status;
243enum {
244    FDB_CS_BEGIN = 0x1,
245    FDB_CS_MOVE_DOC = 0x2,
246    FDB_CS_BATCH_MOVE = 0x4,
247    FDB_CS_FLUSH_WAL = 0x8,
248    FDB_CS_END = 0x10, // invoked at the end of every phase of compaction
249    FDB_CS_COMPLETE = 0x20 // invoked on completion of compaction
250};
251
252/**
253 * Compaction decision returned if FDB_CS_MOVE_DOC callback option is used.
254 * If this compaction callback option is used then it is upto its corresponding
255 * callback function to specify, using the given return values below, if a
256 * given document should be retained in the newly compacted file or dropped.
257 */
258typedef int fdb_compact_decision;
259enum {
260    FDB_CS_KEEP_DOC = 0x0,
261    FDB_CS_DROP_DOC = 0x1
262};
263
264/**
265 * Pointer type definition of a callback function for compaction.
266 */
267typedef fdb_compact_decision (*fdb_compaction_callback)(
268                               fdb_file_handle *fhandle,
269                               fdb_compaction_status status,
270                               const char *kv_store_name,
271                               fdb_doc *doc,
272                               uint64_t last_oldfile_offset,
273                               uint64_t last_newfile_offset,
274                               void *ctx);
275
276/**
277  * Encryption algorithms known to ForestDB.
278  */
279typedef int fdb_encryption_algorithm_t;
280enum {
281    FDB_ENCRYPTION_NONE = 0,    /**< No encryption (default) */
282    FDB_ENCRYPTION_AES256 = 1   /**< AES with 256-bit key */
283};
284
285/**
286  * File encryption key.
287  */
288typedef struct {
289    fdb_encryption_algorithm_t algorithm;
290    uint8_t bytes[32];
291} fdb_encryption_key;
292
293/**
294 * ForestDB config options that are passed to fdb_open API.
295 */
296typedef struct {
297    /**
298     * Chunk size (bytes) that is used to build B+-tree at each level.
299     * It is set to 8 bytes by default and has a min value of 4 bytes
300     * and a max value of 64 bytes.
301     * This is a local config to each ForestDB file.
302     */
303    uint16_t chunksize;
304    /**
305     * Size of block that is a unit of IO operations.
306     * It is set to 4KB by default and has a min value of 1KB and a max value of
307     * 128KB. This is a global config that is used across all ForestDB files.
308     */
309    uint32_t blocksize;
310    /**
311     * Buffer cache size in bytes. If the size is set to zero, then the buffer
312     * cache is disabled. This is a global config that is used across all
313     * ForestDB files.
314     */
315    uint64_t buffercache_size;
316    /**
317     * WAL index size threshold in memory (4096 entries by default).
318     * This is a local config to each ForestDB file.
319     */
320    uint64_t wal_threshold;
321    /**
322     * Flag to enable flushing the WAL whenever it reaches its threshold size.
323     * This reduces memory usage when a lot of data is written before a commit.
324     */
325    bool wal_flush_before_commit;
326    /**
327     * Flag to enable automatic commit.
328     * This is a local config to each ForestDB file.
329     */
330    bool auto_commit;
331    /**
332     * Interval for purging logically deleted documents in the unit of second.
333     * It is set to 0 second (purge during next compaction) by default.
334     * This is a local config to each ForestDB file.
335     */
336    uint32_t purging_interval;
337    /**
338     * Flag to enable or disable a sequence B+-Tree.
339     * This is a local config to each ForestDB file.
340     */
341    fdb_seqtree_opt_t seqtree_opt;
342    /**
343     * Flag to enable synchronous or asynchronous commit options.
344     * This is a local config to each ForestDB file.
345     */
346    fdb_durability_opt_t durability_opt;
347    /**
348     * Flags for fdb_open API. It can be used for specifying read-only mode.
349     * This is a local config to each ForestDB file.
350     */
351    fdb_open_flags flags;
352    /**
353     * Maximum size (bytes) of temporary buffer for compaction (4MB by default).
354     * This is a local config to each ForestDB file.
355     */
356    uint32_t compaction_buf_maxsize;
357    /**
358     * Destroy all the cached blocks in the global buffer cache when a ForestDB
359     * file is closed. It is set to true by default. This is a global config
360     * that is used across all ForestDB files.
361     */
362    bool cleanup_cache_onclose;
363    /**
364     * Compress the body of document when it is written on disk. The compression
365     * is disabled by default. This is a global config that is used across all
366     * ForestDB files.
367     */
368    bool compress_document_body;
369    /**
370     * Flag to enable auto compaction for the file. The auto compaction is disabled
371     * by default.
372     * This is a local config to each ForestDB file.
373     */
374    fdb_compaction_mode_t compaction_mode;
375    /**
376     * Compaction threshold in the unit of percentage (%). It can be calculated
377     * as '(stale data size)/(total file size)'. The compaction daemon triggers
378     * compaction if this threshold is satisfied.
379     * Compaction will not be performed when this value is set to zero or 100.
380     * This is a local config to each ForestDB file.
381     */
382    uint8_t compaction_threshold;
383    /**
384     * The minimum filesize to perform compaction.
385     * This is a local config to each ForestDB file.
386     */
387    uint64_t compaction_minimum_filesize;
388    /**
389     * Duration that the compaction daemon periodically wakes up, in the unit of
390     * second. This is a global config that is used across all ForestDB files.
391     */
392    uint64_t compactor_sleep_duration;
393    /**
394     * Flag to enable supporting multiple KV instances in a DB instance.
395     * This is a global config that is used across all ForestDB files.
396     */
397    bool multi_kv_instances;
398    /**
399     * Duration that prefetching of DB file will be performed when the file
400     * is opened, in the unit of second. If the duration is set to zero,
401     * prefetching is disabled. This is a local config to each ForestDB file.
402     */
403    uint64_t prefetch_duration;
404    /**
405     * Number of in-memory WAL index partitions for a DB file.
406     * This is a local config to each ForestDB file.
407     */
408    uint16_t num_wal_partitions;
409    /**
410     * Number of buffer cache partitions for each DB file.
411     * This is a local config to each ForestDB file.
412     */
413    uint16_t num_bcache_partitions;
414    /**
415     * Callback function for compaction.
416     * This is a local config to each ForestDB file.
417     */
418    fdb_compaction_callback compaction_cb;
419    /**
420     * Mask to select when to invoke callback function during compaction.
421     * Note that mask value is a combination of flags defined in
422     * fdb_compaction_status.
423     * This is a local config to each ForestDB file.
424     */
425    uint32_t compaction_cb_mask;
426    /**
427     * Auxiliary data for compaction callback function.
428     * This is a local config to each ForestDB file.
429     */
430    void *compaction_cb_ctx;
431    /**
432     * Maximum probability (range: 20% ~ 100%) for the compactor to grab
433     * the writer's lock during each batch write in case the writer's throughput
434     * is faster than the compactor, to make sure that the compactor can keep
435     * pace with the writer and eventually complete the compaction.
436     * Note that we plan to reduce the compaction overhead significantly soon
437     * and deprecate this parameter when it is not needed anymore.
438     * This is a local config to each ForestDB file.
439     */
440    size_t max_writer_lock_prob;
441    /**
442     * Number of daemon compactor threads. It is set to 4 threads by default.
443     * If many files are opened and accessed concurrently, then it is
444     * recommended to increase this value if the host machine has enough cores
445     * and disk I/O bandwidth.
446     * This is a global config that is configured across all ForestDB files.
447     */
448    size_t num_compactor_threads;
449    /**
450     * Number of background flusher threads. It is set to 4 threads by default.
451     * For write intensive workloads with large commit intervals and many files
452     * it is recommended to increase this value if the host machine has enough
453     * cores and disk I/O bandwidth.
454     * This is a global config that is configured across all ForestDB files.
455     */
456    size_t num_bgflusher_threads;
457    /**
458     * Encryption key for the database. Default value has algorithm = FDB_ENCRYPTION_NONE,
459     * i.e. no encryption. When a database file is being created, its contents will be
460     * encrypted with the given key. When a database is re-opened, the same key
461     * must be given, otherwise fdb_open will fail with error FDB_RESULT_NO_DB_HEADERS.
462     */
463    fdb_encryption_key encryption_key;
464    /**
465     * Circular block reusing threshold in the unit of percentage (%), which can be
466     * represented as '(stale data size)/(total file size)'. When stale data size
467     * grows beyond the threshold, circular block reusing is triggered so that stale
468     * blocks are reused for further block allocation. Block reusing is disabled if
469     * this threshold is set to zero or 100.
470     */
471    size_t block_reusing_threshold;
472    /**
473     * Number of the last commit headers whose stale blocks should be kept for
474     * snapshot readers.
475     */
476    size_t num_keeping_headers;
477
478} fdb_config;
479
480typedef struct {
481    /**
482     * Flag to create a new empty KV store instance in a DB instance,
483     * if it doesn't exist.
484     */
485    bool create_if_missing;
486    /**
487     * Customized compare function for an KV store instance.
488     */
489    fdb_custom_cmp_variable custom_cmp;
490} fdb_kvs_config;
491
492/**
493 * Pointer type definition of an error logging callback function.
494 */
495typedef void (*fdb_log_callback)(int err_code, const char *err_msg, void *ctx_data);
496
497/**
498 * Function pointer definition of the fatal error callback function.
499 */
500typedef void (*fdb_fatal_error_callback)(void);
501
502/**
503 * ForestDB iterator options.Combinational options can be passed to the iterator.
504 * For example, FDB_ITR_SKIP_MIN_KEY | FDB_ITR_SKIP_MAX_KEY means
505 * "The smallest and largest keys in the iteration ragne won't be returned by the
506 * iterator".
507 */
508typedef uint16_t fdb_iterator_opt_t;
509enum {
510    /**
511     * Return both key and value through iterator.
512     */
513    FDB_ITR_NONE = 0x00,
514    /**
515     * Return only non-deleted items through iterator.
516     */
517    FDB_ITR_NO_DELETES = 0x02,
518    /**
519     * The lowest key specified will not be returned by the iterator.
520     */
521    FDB_ITR_SKIP_MIN_KEY = 0x04,
522    /**
523     * The highest key specified will not be returned by the iterator.
524     */
525    FDB_ITR_SKIP_MAX_KEY = 0x08
526};
527
528/**
529 * ForestDB iterator seek options.
530 */
531typedef uint8_t fdb_iterator_seek_opt_t;
532enum {
533    /**
534     * If seek_key does not exist return the next sorted key higher than it.
535     */
536    FDB_ITR_SEEK_HIGHER = 0x00,
537    /**
538     * If seek_key does not exist return the previous sorted key lower than it.
539     */
540    FDB_ITR_SEEK_LOWER = 0x01
541};
542
543/**
544 * Opaque reference to ForestDB iterator structure definition, which is exposed
545 * in public APIs.
546 */
547typedef struct _fdb_iterator fdb_iterator;
548
549/**
550 * Using off_t turned out to be a real challenge. On "unix-like" systems
551 * its size is set by a combination of #defines like: _LARGE_FILE,
552 * _FILE_OFFSET_BITS and/or _LARGEFILE_SOURCE etc. The interesting
553 * part is however Windows.
554 *
555 * Windows follows the LLP64 data model:
556 * http://en.wikipedia.org/wiki/LLP64#64-bit_data_models
557 *
558 * This means both the int and long int types have a size of 32 bits
559 * regardless if it's a 32 or 64 bits Windows system.
560 *
561 * And Windows defines the type off_t as being a signed long integer:
562 * http://msdn.microsoft.com/en-us/library/323b6b3k.aspx
563 *
564 * This means we can't use off_t on Windows if we deal with files
565 * that can have a size of 2Gb or more.
566 */
567typedef int64_t cs_off_t;
568
569/**
570 * Information about a ForestDB file
571 */
572typedef struct {
573    /**
574     * A file name.
575     */
576    const char* filename;
577    /**
578     * A new file name that is used after compaction.
579     */
580    const char* new_filename;
581    /**
582     * Total number of non-deleted documents aggregated across all KV stores.
583     */
584    uint64_t doc_count;
585    /**
586     * Total number of deleted documents aggregated across all KV stores.
587     */
588    uint64_t deleted_count;
589    /**
590     * Disk space actively used by the file.
591     */
592    uint64_t space_used;
593    /**
594     * Total disk space used by the file, including stale btree nodes and docs.
595     */
596    uint64_t file_size;
597    /**
598     * Number of KV store instances in a ForestDB file
599     */
600    size_t num_kv_stores;
601} fdb_file_info;
602
603/**
604 * Information about a ForestDB KV store
605 */
606typedef struct {
607    /**
608     * A KV store name.
609     */
610    const char* name;
611    /**
612     * Last sequence number assigned.
613     */
614    fdb_seqnum_t last_seqnum;
615    /**
616     * Total number of non-deleted documents in a KV store.
617     */
618    uint64_t doc_count;
619    /**
620     * Total number of deleted documents in a KV store.
621     */
622    uint64_t deleted_count;
623    /**
624     * Disk space actively used by the KV store.
625     */
626    uint64_t space_used;
627    /**
628     * File handle that owns the KV store.
629     */
630    fdb_file_handle* file;
631} fdb_kvs_info;
632
633/**
634 * Information about a ForestDB KV store's operational counters
635 */
636typedef struct {
637    /**
638     * Number of fdb_set operations.
639     */
640    uint64_t num_sets;
641    /**
642     * Number of fdb_del operations.
643     */
644    uint64_t num_dels;
645    /**
646     * Number of fdb_commit operations.
647     */
648    uint64_t num_commits;
649    /**
650     * Number of fdb_compact operations on underlying file.
651     */
652    uint64_t num_compacts;
653    /**
654     * Number of fdb_get* (includes metaonly, byseq etc) operations.
655     */
656    uint64_t num_gets;
657    /**
658     * Number of fdb_iterator_get* (includes meta_only) operations.
659     */
660    uint64_t num_iterator_gets;
661    /**
662     * Number of fdb_iterator_moves (includes next,prev,seek) operations.
663     */
664    uint64_t num_iterator_moves;
665} fdb_kvs_ops_info;
666
667/**
668 * Latency stat type for each public API
669 */
670typedef uint8_t fdb_latency_stat_type;
671enum {
672    FDB_LATENCY_SETS      = 0, // fdb_set API
673    FDB_LATENCY_GETS      = 1, // fdb_get API
674    FDB_LATENCY_COMMITS   = 2, // fdb_commit API
675    FDB_LATENCY_SNAPSHOTS = 3, // fdb_snapshot_open API
676    FDB_LATENCY_COMPACTS  = 4  // fdb_compact API
677};
678
679// Number of latency stat types
680#define FDB_LATENCY_NUM_STATS 5
681
682/**
683 * Latency statistics of a specific ForestDB api call
684 */
685typedef struct {
686    /**
687     * Total number this call was invoked.
688     */
689    uint64_t lat_count;
690    /**
691     * The fastest call took this amount of time in micro seconds.
692     */
693    uint32_t lat_min;
694    /**
695     * The slowest call took this amount of time in micro seconds.
696     */
697    uint32_t lat_max;
698    /**
699     * The average time taken by this call in micro seconds.
700     */
701    uint32_t lat_avg;
702} fdb_latency_stat;
703
704/**
705 * List of ForestDB KV store names
706 */
707typedef struct {
708    /**
709     * Number of KV store names listed in kvs_names.
710     */
711    size_t num_kvs_names;
712    /**
713     * Pointer to array of KV store names.
714     */
715    char **kvs_names;
716} fdb_kvs_name_list;
717
718/**
719 * Persisted Snapshot Marker in file (Sequence number + KV Store name)
720 */
721typedef struct {
722    /**
723     * NULL-terminated KV Store name.
724     */
725    char *kv_store_name;
726    /**
727     * A Sequence number of the above KV store, which results from an
728     * fdb_commit operation.
729     */
730    fdb_seqnum_t seqnum;
731} fdb_kvs_commit_marker_t;
732
733/**
734 * An opaque file-level snapshot marker that can be used to purge
735 * stale data up to a given file-level snapshot marker.
736*/
737typedef uint64_t fdb_snapshot_marker_t;
738
739/**
740 * Snapshot Information structure for a ForestDB database file.
741 */
742typedef struct {
743    /**
744     * Opaque file-level snapshot marker that can be passed to
745     * fdb_compact_upto() api.
746     */
747    fdb_snapshot_marker_t marker;
748    /**
749     * Number of KV store snapshot markers in the kvs_markers array.
750     */
751    int64_t num_kvs_markers;
752    /**
753     * Pointer to an array of {kv_store_name, committed_seqnum} pairs.
754     */
755    fdb_kvs_commit_marker_t *kvs_markers;
756} fdb_snapshot_info_t;
757
758#ifdef __cplusplus
759}
760#endif
761
762#endif
763