xref: /6.6.0/forestdb/src/fdb_internal.h (revision 56236603)
1/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/*
3 *     Copyright 2010 Couchbase, Inc
4 *
5 *   Licensed under the Apache License, Version 2.0 (the "License");
6 *   you may not use this file except in compliance with the License.
7 *   You may obtain a copy of the License at
8 *
9 *       http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *   Unless required by applicable law or agreed to in writing, software
12 *   distributed under the License is distributed on an "AS IS" BASIS,
13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *   See the License for the specific language governing permissions and
15 *   limitations under the License.
16 */
17
18#ifndef _FDB_INTERNAL_H
19#define _FDB_INTERNAL_H
20
21#include <stdint.h>
22#include "common.h"
23#include "internal_types.h"
24#include "avltree.h"
25#include "btreeblock.h"
26#include "hbtrie.h"
27#include "docio.h"
28#include "staleblock.h"
29
30#ifdef __cplusplus
31extern "C" {
32#endif
33
34/* If non-NULL, callback invoked when handling a fatal error. */
35extern fdb_fatal_error_callback fatal_error_callback;
36
37void buf2kvid(size_t chunksize, void *buf, fdb_kvs_id_t *id);
38void kvid2buf(size_t chunksize, fdb_kvs_id_t id, void *buf);
39void buf2buf(size_t chunksize_src, void *buf_src,
40             size_t chunksize_dst, void *buf_dst);
41
42size_t _fdb_readkey_wrap(void *handle, uint64_t offset, void *buf);
43size_t _fdb_readseq_wrap(void *handle, uint64_t offset, void *buf);
44int _fdb_custom_cmp_wrap(void *key1, void *key2, void *aux);
45
46fdb_status fdb_log(err_log_callback *callback,
47                   fdb_status status,
48                   const char *format, ...);
49
50fdb_status _fdb_clone_snapshot(fdb_kvs_handle *handle_in,
51                               fdb_kvs_handle *handle_out);
52fdb_status _fdb_open(fdb_kvs_handle *handle,
53                     const char *filename,
54                     fdb_filename_mode_t filename_mode,
55                     const fdb_config *config);
56fdb_status _fdb_close_root(fdb_kvs_handle *handle);
57fdb_status _fdb_close(fdb_kvs_handle *handle);
58fdb_status _fdb_commit(fdb_kvs_handle *handle,
59                       fdb_commit_opt_t opt,
60                       bool sync);
61
62fdb_status fdb_check_file_reopen(fdb_kvs_handle *handle, file_status_t *status);
63void fdb_sync_db_header(fdb_kvs_handle *handle);
64
65void fdb_fetch_header(uint64_t version,
66                      void *header_buf,
67                      bid_t *trie_root_bid,
68                      bid_t *seq_root_bid,
69                      bid_t *stale_root_bid,
70                      uint64_t *ndocs,
71                      uint64_t *ndeletes,
72                      uint64_t *nlivenodes,
73                      uint64_t *datasize,
74                      uint64_t *last_wal_flush_hdr_bid,
75                      uint64_t *kv_info_offset,
76                      uint64_t *header_flags,
77                      char **new_filename,
78                      char **old_filename);
79uint64_t fdb_set_file_header(fdb_kvs_handle *handle, bool inc_revnum);
80
81void _fdb_invalidate_dbheader(fdb_kvs_handle *handle);
82
83fdb_status fdb_open_for_compactor(fdb_file_handle **ptr_fhandle,
84                                  const char *filename,
85                                  fdb_config *fconfig,
86                                  struct list *cmp_func_list);
87
88fdb_status fdb_compact_file(fdb_file_handle *fhandle,
89                            const char *new_filename,
90                            bool in_place_compaction,
91                            bid_t marker_bid,
92                            bool clone_docs,
93                            const fdb_encryption_key *new_encryption_key);
94
95fdb_status _fdb_abort_transaction(fdb_kvs_handle *handle);
96
97typedef enum {
98    FDB_RESTORE_NORMAL,
99    FDB_RESTORE_KV_INS,
100} fdb_restore_mode_t;
101
102void fdb_file_handle_init(fdb_file_handle *fhandle,
103                           fdb_kvs_handle *root);
104void fdb_file_handle_close_all(fdb_file_handle *fhandle);
105void fdb_file_handle_parse_cmp_func(fdb_file_handle *fhandle,
106                                    size_t n_func,
107                                    char **kvs_names,
108                                    fdb_custom_cmp_variable *functions);
109void fdb_file_handle_clone_cmp_func_list(fdb_file_handle *fhandle,
110                                         struct list *cmp_func_list);
111void fdb_file_handle_add_cmp_func(fdb_file_handle *fhandle,
112                                  char *kvs_name,
113                                  fdb_custom_cmp_variable cmp_func);
114void fdb_file_handle_free(fdb_file_handle *fhandle);
115
116void fdb_cmp_func_list_from_filemgr(struct filemgr *file,
117                                    struct list *cmp_func_list);
118void fdb_free_cmp_func_list(struct list *cmp_func_list);
119
120fdb_status fdb_kvs_cmp_check(fdb_kvs_handle *handle);
121hbtrie_cmp_func * fdb_kvs_find_cmp_chunk(void *chunk, void *aux);
122
123void fdb_kvs_info_create(fdb_kvs_handle *root_handle,
124                         fdb_kvs_handle *handle,
125                         struct filemgr *file,
126                         const char *kvs_name);
127void fdb_kvs_info_free(fdb_kvs_handle *handle);
128void fdb_kvs_header_reset_all_stats(struct filemgr *file);
129void fdb_kvs_header_create(struct filemgr *file);
130uint64_t fdb_kvs_header_append(fdb_kvs_handle *handle);
131
132struct kvs_header;
133
134void fdb_kvs_header_read(struct kvs_header *kv_header,
135                         struct docio_handle *dhandle,
136                         uint64_t kv_info_offset,
137                         uint64_t version,
138                         bool only_seq_nums);
139void fdb_kvs_header_copy(fdb_kvs_handle *handle,
140                         struct filemgr *new_file,
141                         struct docio_handle *new_dhandle,
142                         uint64_t *new_file_kv_info_offset,
143                         bool create_new);
144void _fdb_kvs_init_root(fdb_kvs_handle *handle, struct filemgr *file);
145void _fdb_kvs_header_create(struct kvs_header **kv_header_ptr);
146void _fdb_kvs_header_import(struct kvs_header *kv_header,
147                            void *data, size_t len, uint64_t version,
148                            bool only_seq_nums);
149
150fdb_status _fdb_kvs_get_snap_info(void *data, uint64_t version,
151                                  fdb_snapshot_info_t *snap_info);
152void _fdb_kvs_header_free(struct kvs_header *kv_header);
153fdb_seqnum_t _fdb_kvs_get_seqnum(struct kvs_header *kv_header,
154                                 fdb_kvs_id_t id);
155uint64_t _kvs_stat_get_sum_attr(void *data, uint64_t version,
156                                kvs_stat_attr_t attr);
157
158bool _fdb_kvs_is_busy(fdb_file_handle *fhandle);
159
160void fdb_kvs_header_free(struct filemgr *file);
161
162char* _fdb_kvs_get_name(fdb_kvs_handle *kv_ins, struct filemgr *file);
163/**
164 * Extracts the KV Store name from a key sample and offset to start of user key
165 * @param handle - pointer to root handle
166 * @param keybuf - pointer to key which may include the KV Store Id prefix
167 * @param key_offset - return variable of offset to where real key begins
168 */
169const char* _fdb_kvs_extract_name_off(fdb_kvs_handle *handle, void *keybuf,
170                                      size_t *key_offset);
171
172fdb_status _fdb_kvs_clone_snapshot(fdb_kvs_handle *handle_in,
173                                   fdb_kvs_handle *handle_out);
174fdb_status _fdb_kvs_open(fdb_kvs_handle *root_handle,
175                         fdb_config *config,
176                         fdb_kvs_config *kvs_config,
177                         struct filemgr *file,
178                         const char *filename,
179                         const char *kvs_name,
180                         fdb_kvs_handle *handle);
181
182/**
183 * Link a given KV Store handle into the file handle's list of
184 * handles. This list may be used to auto close all child KV Store
185 * handles when the file handle is closed.
186 * @param fhandle - parent file handle of the ForestDB database.
187 * @param handle - the newly opened KV Store handle
188 * @return pointer to the newly linked node
189 */
190struct kvs_opened_node * _fdb_kvs_createNLinkKVHandle(fdb_file_handle *fhandle,
191                                                      fdb_kvs_handle *handle);
192fdb_status _fdb_kvs_close(fdb_kvs_handle *handle);
193fdb_status fdb_kvs_close_all(fdb_kvs_handle *root_handle);
194
195fdb_seqnum_t fdb_kvs_get_seqnum(struct filemgr *file,
196                                fdb_kvs_id_t id);
197fdb_seqnum_t fdb_kvs_get_committed_seqnum(fdb_kvs_handle *handle);
198
199void fdb_kvs_set_seqnum(struct filemgr *file,
200                        fdb_kvs_id_t id,
201                        fdb_seqnum_t seqnum);
202
203fdb_status fdb_kvs_rollback(fdb_kvs_handle **handle_ptr, fdb_seqnum_t seqnum);
204
205/**
206 * Return the smallest commit revision number that are currently being referred.
207 *
208 * @param handle Pointer to ForestDB KV store handle.
209 * @return Header revision number and block ID.
210 */
211stale_header_info fdb_get_smallest_active_header(fdb_kvs_handle *handle);
212
213INLINE size_t _fdb_get_docsize(struct docio_length len)
214{
215    size_t ret =
216        len.keylen +
217        len.metalen +
218        len.bodylen_ondisk +
219        sizeof(struct docio_length);
220
221    ret += sizeof(timestamp_t);
222
223    ret += sizeof(fdb_seqnum_t);
224
225#ifdef __CRC32
226    ret += sizeof(uint32_t);
227#endif
228
229    return ret;
230}
231
232INLINE void _fdb_import_dirty_root(fdb_kvs_handle *handle,
233                                   bid_t dirty_idtree_root,
234                                   bid_t dirty_seqtree_root)
235{
236    if (dirty_idtree_root != BLK_NOT_FOUND) {
237        handle->trie->root_bid = dirty_idtree_root;
238    }
239    if (handle->config.seqtree_opt == FDB_SEQTREE_USE) {
240        if (dirty_seqtree_root != BLK_NOT_FOUND) {
241            if (handle->kvs) {
242                handle->seqtrie->root_bid = dirty_seqtree_root;
243            } else {
244                btree_init_from_bid(handle->seqtree,
245                                    handle->seqtree->blk_handle,
246                                    handle->seqtree->blk_ops,
247                                    handle->seqtree->kv_ops,
248                                    handle->seqtree->blksize,
249                                    dirty_seqtree_root);
250            }
251        }
252    }
253}
254
255INLINE void _fdb_export_dirty_root(fdb_kvs_handle *handle,
256                                   bid_t *dirty_idtree_root,
257                                   bid_t *dirty_seqtree_root)
258{
259    *dirty_idtree_root = handle->trie->root_bid;
260    if (handle->config.seqtree_opt == FDB_SEQTREE_USE) {
261        if (handle->kvs) {
262            *dirty_seqtree_root = handle->seqtrie->root_bid;
263        } else {
264            *dirty_seqtree_root = handle->seqtree->root_bid;
265        }
266    }
267}
268
269// 1. fetch dirty update if exist,
270// 2. and assign dirty root nodes to FDB handle
271INLINE void _fdb_dirty_update_ready(fdb_kvs_handle *handle,
272                                    struct filemgr_dirty_update_node **prev_node,
273                                    struct filemgr_dirty_update_node **new_node,
274                                    bid_t *dirty_idtree_root,
275                                    bid_t *dirty_seqtree_root,
276                                    bool dirty_wal_flush)
277{
278    *prev_node = *new_node = NULL;
279    *dirty_idtree_root = *dirty_seqtree_root = BLK_NOT_FOUND;
280
281    *prev_node = filemgr_dirty_update_get_latest(handle->file);
282
283    // discard all cached index blocks
284    // to avoid data inconsistency with other writers
285    btreeblk_discard_blocks(handle->bhandle);
286
287    // create a new dirty update entry if previous one exists
288    // (if we don't this, we cannot identify which block on
289    //  dirty copy or actual file is more recent during the WAL flushing.)
290
291    // on dirty wal flush, create a new dirty update entry
292    // although there is no previous immutable dirty updates.
293
294    if (*prev_node || dirty_wal_flush) {
295        *new_node = filemgr_dirty_update_new_node(handle->file);
296        // sync dirty root nodes
297        filemgr_dirty_update_get_root(handle->file, *prev_node,
298                                      dirty_idtree_root, dirty_seqtree_root);
299    }
300    btreeblk_set_dirty_update(handle->bhandle, *prev_node);
301    btreeblk_set_dirty_update_writer(handle->bhandle, *new_node);
302
303    // assign dirty root nodes to FDB handle
304    _fdb_import_dirty_root(handle, *dirty_idtree_root, *dirty_seqtree_root);
305}
306
307// 1. get dirty root from FDB handle,
308// 2. update corresponding dirty update entry,
309// 3. make new_node immutable, and close previous immutable node
310INLINE void _fdb_dirty_update_finalize(fdb_kvs_handle *handle,
311                                       struct filemgr_dirty_update_node *prev_node,
312                                       struct filemgr_dirty_update_node *new_node,
313                                       bid_t *dirty_idtree_root,
314                                       bid_t *dirty_seqtree_root,
315                                       bool commit)
316{
317    // read dirty root nodes from FDB handle
318    _fdb_export_dirty_root(handle, dirty_idtree_root, dirty_seqtree_root);
319    // assign dirty root nodes to dirty update entry
320    if (new_node) {
321        filemgr_dirty_update_set_root(handle->file, new_node,
322                                      *dirty_idtree_root, *dirty_seqtree_root);
323    }
324    // clear dirty update setting in bhandle
325    btreeblk_clear_dirty_update(handle->bhandle);
326    // finalize new_node
327    if (new_node) {
328        filemgr_dirty_update_set_immutable(handle->file, prev_node, new_node);
329    }
330    // close previous immutable node
331    if (prev_node) {
332        filemgr_dirty_update_close_node(handle->file, prev_node);
333    }
334    if (commit) {
335        // write back new_node's dirty blocks
336        filemgr_dirty_update_commit(handle->file, new_node, &handle->log_callback);
337    } else {
338        // if this update set is still dirty,
339        // discard all cached index blocks to avoid data inconsistency.
340        btreeblk_discard_blocks(handle->bhandle);
341    }
342}
343
344#ifdef __cplusplus
345}
346#endif
347
348#endif
349