xref: /6.0.3/forestdb/src/fdb_internal.h (revision ef4619e9)
1/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/*
3 *     Copyright 2010 Couchbase, Inc
4 *
5 *   Licensed under the Apache License, Version 2.0 (the "License");
6 *   you may not use this file except in compliance with the License.
7 *   You may obtain a copy of the License at
8 *
9 *       http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *   Unless required by applicable law or agreed to in writing, software
12 *   distributed under the License is distributed on an "AS IS" BASIS,
13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *   See the License for the specific language governing permissions and
15 *   limitations under the License.
16 */
17
18#ifndef _FDB_INTERNAL_H
19#define _FDB_INTERNAL_H
20
21#include <stdint.h>
22#include "common.h"
23#include "internal_types.h"
24#include "avltree.h"
25#include "btreeblock.h"
26#include "hbtrie.h"
27#include "docio.h"
28#include "staleblock.h"
29
30#ifdef __cplusplus
31extern "C" {
32#endif
33
34/* If non-NULL, callback invoked when handling a fatal error. */
35extern fdb_fatal_error_callback fatal_error_callback;
36
37void buf2kvid(size_t chunksize, void *buf, fdb_kvs_id_t *id);
38void kvid2buf(size_t chunksize, fdb_kvs_id_t id, void *buf);
39void buf2buf(size_t chunksize_src, void *buf_src,
40             size_t chunksize_dst, void *buf_dst);
41
42size_t _fdb_readkey_wrap(void *handle, uint64_t offset, void *buf);
43size_t _fdb_readseq_wrap(void *handle, uint64_t offset, void *buf);
44int _fdb_custom_cmp_wrap(void *key1, void *key2, void *aux);
45
46fdb_status fdb_log(err_log_callback *callback,
47                   fdb_status status,
48                   const char *format, ...);
49
50fdb_status _fdb_clone_snapshot(fdb_kvs_handle *handle_in,
51                               fdb_kvs_handle *handle_out);
52fdb_status _fdb_open(fdb_kvs_handle *handle,
53                     const char *filename,
54                     fdb_filename_mode_t filename_mode,
55                     const fdb_config *config);
56fdb_status _fdb_close_root(fdb_kvs_handle *handle);
57fdb_status _fdb_close(fdb_kvs_handle *handle);
58fdb_status _fdb_commit(fdb_kvs_handle *handle,
59                       fdb_commit_opt_t opt,
60                       bool sync);
61
62fdb_status fdb_check_file_reopen(fdb_kvs_handle *handle, file_status_t *status);
63void fdb_sync_db_header(fdb_kvs_handle *handle);
64
65void fdb_fetch_header(uint64_t version,
66                      void *header_buf,
67                      bid_t *trie_root_bid,
68                      bid_t *seq_root_bid,
69                      bid_t *stale_root_bid,
70                      uint64_t *ndocs,
71                      uint64_t *ndeletes,
72                      uint64_t *nlivenodes,
73                      uint64_t *datasize,
74                      uint64_t *last_wal_flush_hdr_bid,
75                      uint64_t *kv_info_offset,
76                      uint64_t *header_flags,
77                      char **new_filename,
78                      char **old_filename);
79uint64_t fdb_set_file_header(fdb_kvs_handle *handle, bool inc_revnum);
80
81fdb_status fdb_open_for_compactor(fdb_file_handle **ptr_fhandle,
82                                  const char *filename,
83                                  fdb_config *fconfig,
84                                  struct list *cmp_func_list);
85
86fdb_status fdb_compact_file(fdb_file_handle *fhandle,
87                            const char *new_filename,
88                            bool in_place_compaction,
89                            bid_t marker_bid,
90                            bool clone_docs,
91                            const fdb_encryption_key *new_encryption_key);
92
93fdb_status _fdb_abort_transaction(fdb_kvs_handle *handle);
94
95typedef enum {
96    FDB_RESTORE_NORMAL,
97    FDB_RESTORE_KV_INS,
98} fdb_restore_mode_t;
99
100void fdb_file_handle_init(fdb_file_handle *fhandle,
101                           fdb_kvs_handle *root);
102void fdb_file_handle_close_all(fdb_file_handle *fhandle);
103void fdb_file_handle_parse_cmp_func(fdb_file_handle *fhandle,
104                                    size_t n_func,
105                                    char **kvs_names,
106                                    fdb_custom_cmp_variable *functions);
107void fdb_file_handle_clone_cmp_func_list(fdb_file_handle *fhandle,
108                                         struct list *cmp_func_list);
109void fdb_file_handle_add_cmp_func(fdb_file_handle *fhandle,
110                                  char *kvs_name,
111                                  fdb_custom_cmp_variable cmp_func);
112void fdb_file_handle_free(fdb_file_handle *fhandle);
113
114void fdb_cmp_func_list_from_filemgr(struct filemgr *file,
115                                    struct list *cmp_func_list);
116void fdb_free_cmp_func_list(struct list *cmp_func_list);
117
118fdb_status fdb_kvs_cmp_check(fdb_kvs_handle *handle);
119hbtrie_cmp_func * fdb_kvs_find_cmp_chunk(void *chunk, void *aux);
120
121void fdb_kvs_info_create(fdb_kvs_handle *root_handle,
122                         fdb_kvs_handle *handle,
123                         struct filemgr *file,
124                         const char *kvs_name);
125void fdb_kvs_info_free(fdb_kvs_handle *handle);
126void fdb_kvs_header_reset_all_stats(struct filemgr *file);
127void fdb_kvs_header_create(struct filemgr *file);
128uint64_t fdb_kvs_header_append(fdb_kvs_handle *handle);
129
130struct kvs_header;
131
132void fdb_kvs_header_read(struct kvs_header *kv_header,
133                         struct docio_handle *dhandle,
134                         uint64_t kv_info_offset,
135                         uint64_t version,
136                         bool only_seq_nums);
137void fdb_kvs_header_copy(fdb_kvs_handle *handle,
138                         struct filemgr *new_file,
139                         struct docio_handle *new_dhandle,
140                         uint64_t *new_file_kv_info_offset,
141                         bool create_new);
142void _fdb_kvs_init_root(fdb_kvs_handle *handle, struct filemgr *file);
143void _fdb_kvs_header_create(struct kvs_header **kv_header_ptr);
144void _fdb_kvs_header_import(struct kvs_header *kv_header,
145                            void *data, size_t len, uint64_t version,
146                            bool only_seq_nums);
147
148fdb_status _fdb_kvs_get_snap_info(void *data, uint64_t version,
149                                  fdb_snapshot_info_t *snap_info);
150void _fdb_kvs_header_free(struct kvs_header *kv_header);
151fdb_seqnum_t _fdb_kvs_get_seqnum(struct kvs_header *kv_header,
152                                 fdb_kvs_id_t id);
153uint64_t _kvs_stat_get_sum_attr(void *data, uint64_t version,
154                                kvs_stat_attr_t attr);
155
156bool _fdb_kvs_is_busy(fdb_file_handle *fhandle);
157
158void fdb_kvs_header_free(struct filemgr *file);
159
160char* _fdb_kvs_get_name(fdb_kvs_handle *kv_ins, struct filemgr *file);
161/**
162 * Extracts the KV Store name from a key sample and offset to start of user key
163 * @param handle - pointer to root handle
164 * @param keybuf - pointer to key which may include the KV Store Id prefix
165 * @param key_offset - return variable of offset to where real key begins
166 */
167const char* _fdb_kvs_extract_name_off(fdb_kvs_handle *handle, void *keybuf,
168                                      size_t *key_offset);
169
170fdb_status _fdb_kvs_clone_snapshot(fdb_kvs_handle *handle_in,
171                                   fdb_kvs_handle *handle_out);
172fdb_status _fdb_kvs_open(fdb_kvs_handle *root_handle,
173                         fdb_config *config,
174                         fdb_kvs_config *kvs_config,
175                         struct filemgr *file,
176                         const char *filename,
177                         const char *kvs_name,
178                         fdb_kvs_handle *handle);
179
180/**
181 * Link a given KV Store handle into the file handle's list of
182 * handles. This list may be used to auto close all child KV Store
183 * handles when the file handle is closed.
184 * @param fhandle - parent file handle of the ForestDB database.
185 * @param handle - the newly opened KV Store handle
186 * @return pointer to the newly linked node
187 */
188struct kvs_opened_node * _fdb_kvs_createNLinkKVHandle(fdb_file_handle *fhandle,
189                                                      fdb_kvs_handle *handle);
190fdb_status _fdb_kvs_close(fdb_kvs_handle *handle);
191fdb_status fdb_kvs_close_all(fdb_kvs_handle *root_handle);
192
193fdb_seqnum_t fdb_kvs_get_seqnum(struct filemgr *file,
194                                fdb_kvs_id_t id);
195fdb_seqnum_t fdb_kvs_get_committed_seqnum(fdb_kvs_handle *handle);
196
197void fdb_kvs_set_seqnum(struct filemgr *file,
198                        fdb_kvs_id_t id,
199                        fdb_seqnum_t seqnum);
200
201fdb_status fdb_kvs_rollback(fdb_kvs_handle **handle_ptr, fdb_seqnum_t seqnum);
202
203/**
204 * Return the smallest commit revision number that are currently being referred.
205 *
206 * @param handle Pointer to ForestDB KV store handle.
207 * @return Header revision number and block ID.
208 */
209stale_header_info fdb_get_smallest_active_header(fdb_kvs_handle *handle);
210
211INLINE size_t _fdb_get_docsize(struct docio_length len)
212{
213    size_t ret =
214        len.keylen +
215        len.metalen +
216        len.bodylen_ondisk +
217        sizeof(struct docio_length);
218
219    ret += sizeof(timestamp_t);
220
221    ret += sizeof(fdb_seqnum_t);
222
223#ifdef __CRC32
224    ret += sizeof(uint32_t);
225#endif
226
227    return ret;
228}
229
230INLINE void _fdb_import_dirty_root(fdb_kvs_handle *handle,
231                                   bid_t dirty_idtree_root,
232                                   bid_t dirty_seqtree_root)
233{
234    if (dirty_idtree_root != BLK_NOT_FOUND) {
235        handle->trie->root_bid = dirty_idtree_root;
236    }
237    if (handle->config.seqtree_opt == FDB_SEQTREE_USE) {
238        if (dirty_seqtree_root != BLK_NOT_FOUND) {
239            if (handle->kvs) {
240                handle->seqtrie->root_bid = dirty_seqtree_root;
241            } else {
242                btree_init_from_bid(handle->seqtree,
243                                    handle->seqtree->blk_handle,
244                                    handle->seqtree->blk_ops,
245                                    handle->seqtree->kv_ops,
246                                    handle->seqtree->blksize,
247                                    dirty_seqtree_root);
248            }
249        }
250    }
251}
252
253INLINE void _fdb_export_dirty_root(fdb_kvs_handle *handle,
254                                   bid_t *dirty_idtree_root,
255                                   bid_t *dirty_seqtree_root)
256{
257    *dirty_idtree_root = handle->trie->root_bid;
258    if (handle->config.seqtree_opt == FDB_SEQTREE_USE) {
259        if (handle->kvs) {
260            *dirty_seqtree_root = handle->seqtrie->root_bid;
261        } else {
262            *dirty_seqtree_root = handle->seqtree->root_bid;
263        }
264    }
265}
266
267// 1. fetch dirty update if exist,
268// 2. and assign dirty root nodes to FDB handle
269INLINE void _fdb_dirty_update_ready(fdb_kvs_handle *handle,
270                                    struct filemgr_dirty_update_node **prev_node,
271                                    struct filemgr_dirty_update_node **new_node,
272                                    bid_t *dirty_idtree_root,
273                                    bid_t *dirty_seqtree_root,
274                                    bool dirty_wal_flush)
275{
276    *prev_node = *new_node = NULL;
277    *dirty_idtree_root = *dirty_seqtree_root = BLK_NOT_FOUND;
278
279    *prev_node = filemgr_dirty_update_get_latest(handle->file);
280
281    // discard all cached index blocks
282    // to avoid data inconsistency with other writers
283    btreeblk_discard_blocks(handle->bhandle);
284
285    // create a new dirty update entry if previous one exists
286    // (if we don't this, we cannot identify which block on
287    //  dirty copy or actual file is more recent during the WAL flushing.)
288
289    // on dirty wal flush, create a new dirty update entry
290    // although there is no previous immutable dirty updates.
291
292    if (*prev_node || dirty_wal_flush) {
293        *new_node = filemgr_dirty_update_new_node(handle->file);
294        // sync dirty root nodes
295        filemgr_dirty_update_get_root(handle->file, *prev_node,
296                                      dirty_idtree_root, dirty_seqtree_root);
297    }
298    btreeblk_set_dirty_update(handle->bhandle, *prev_node);
299    btreeblk_set_dirty_update_writer(handle->bhandle, *new_node);
300
301    // assign dirty root nodes to FDB handle
302    _fdb_import_dirty_root(handle, *dirty_idtree_root, *dirty_seqtree_root);
303}
304
305// 1. get dirty root from FDB handle,
306// 2. update corresponding dirty update entry,
307// 3. make new_node immutable, and close previous immutable node
308INLINE void _fdb_dirty_update_finalize(fdb_kvs_handle *handle,
309                                       struct filemgr_dirty_update_node *prev_node,
310                                       struct filemgr_dirty_update_node *new_node,
311                                       bid_t *dirty_idtree_root,
312                                       bid_t *dirty_seqtree_root,
313                                       bool commit)
314{
315    // read dirty root nodes from FDB handle
316    _fdb_export_dirty_root(handle, dirty_idtree_root, dirty_seqtree_root);
317    // assign dirty root nodes to dirty update entry
318    if (new_node) {
319        filemgr_dirty_update_set_root(handle->file, new_node,
320                                      *dirty_idtree_root, *dirty_seqtree_root);
321    }
322    // clear dirty update setting in bhandle
323    btreeblk_clear_dirty_update(handle->bhandle);
324    // finalize new_node
325    if (new_node) {
326        filemgr_dirty_update_set_immutable(handle->file, prev_node, new_node);
327    }
328    // close previous immutable node
329    if (prev_node) {
330        filemgr_dirty_update_close_node(handle->file, prev_node);
331    }
332    if (commit) {
333        // write back new_node's dirty blocks
334        filemgr_dirty_update_commit(handle->file, new_node, &handle->log_callback);
335    } else {
336        // if this update set is still dirty,
337        // discard all cached index blocks to avoid data inconsistency.
338        btreeblk_discard_blocks(handle->bhandle);
339    }
340}
341
342#ifdef __cplusplus
343}
344#endif
345
346#endif
347