1/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/*
3 *     Copyright 2010 Couchbase, Inc
4 *
5 *   Licensed under the Apache License, Version 2.0 (the "License");
6 *   you may not use this file except in compliance with the License.
7 *   You may obtain a copy of the License at
8 *
9 *       http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *   Unless required by applicable law or agreed to in writing, software
12 *   distributed under the License is distributed on an "AS IS" BASIS,
13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *   See the License for the specific language governing permissions and
15 *   limitations under the License.
16 */
17
18#include <stdio.h>
19#include <stdlib.h>
20#include <string.h>
21#include <stdint.h>
22#include <time.h>
23#if !defined(WIN32) && !defined(_WIN32)
24#include <unistd.h>
25#endif
26
27#include "libforestdb/forestdb.h"
28#include "test.h"
29#include "internal_types.h"
30#include "functional_util.h"
31
32void basic_test()
33{
34    TEST_INIT();
35
36    memleak_start();
37
38    int i, r;
39    int n = 10;
40    fdb_file_handle *dbfile, *dbfile_rdonly;
41    fdb_kvs_handle *db;
42    fdb_kvs_handle *db_rdonly;
43    fdb_doc **doc = alca(fdb_doc*, n);
44    fdb_doc *rdoc = NULL;
45    fdb_status status;
46
47    char keybuf[256], metabuf[256], bodybuf[256];
48
49    // remove previous dummy test files
50    r = system(SHELL_DEL" dummy* > errorlog.txt");
51    (void)r;
52
53    // Get the ForestDB version
54    const char *version = fdb_get_lib_version();
55    TEST_CHK(version != NULL && strlen(version) > 0);
56
57    fdb_config fconfig = fdb_get_default_config();
58    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
59    fconfig.wal_threshold = 1024;
60    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
61    fconfig.compaction_threshold = 0;
62    fconfig.purging_interval = 1;
63
64    // Read-Write mode test without a create flag.
65    fconfig.flags = 0;
66    status = fdb_open(&dbfile, "./dummy1", &fconfig);
67    TEST_CHK(status == FDB_RESULT_NO_SUCH_FILE);
68    TEST_CHK(!strcmp(fdb_error_msg(status), "no such file"));
69
70    // Read-Only mode test: Must not create new file.
71    fconfig.flags = FDB_OPEN_FLAG_RDONLY;
72    status = fdb_open(&dbfile, "./dummy1", &fconfig);
73    TEST_CHK(status == FDB_RESULT_NO_SUCH_FILE);
74
75    // Read-Only and Create mode: Must not create a new file.
76    fconfig.flags = FDB_OPEN_FLAG_RDONLY | FDB_OPEN_FLAG_CREATE;
77    status = fdb_open(&dbfile, "./dummy1", &fconfig);
78    TEST_CHK(status == FDB_RESULT_INVALID_CONFIG);
79    TEST_CHK(!strcmp(fdb_error_msg(status), "invalid configuration"));
80
81    // open and close db with a create flag.
82    fconfig.flags = FDB_OPEN_FLAG_CREATE;
83    status = fdb_open(&dbfile, "./dummy1", &fconfig);
84    TEST_CHK(status == FDB_RESULT_SUCCESS);
85    TEST_CHK(!strcmp(fdb_error_msg(status), "success"));
86    const char *file_version = fdb_get_file_version(dbfile);
87    TEST_CHK(file_version != NULL && strlen(file_version) > 0);
88    fdb_close(dbfile);
89
90    // reopen db
91    r = system(SHELL_DEL" dummy* > errorlog.txt");
92    (void)r;
93    fdb_open(&dbfile, "./dummy1",&fconfig);
94    fdb_kvs_open_default(dbfile, &db, &kvs_config);
95    status = fdb_set_log_callback(db, logCallbackFunc, (void *) "basic_test");
96    TEST_CHK(status == FDB_RESULT_SUCCESS);
97
98    // insert documents
99    for (i=0;i<n;++i){
100        sprintf(keybuf, "key%d", i);
101        sprintf(metabuf, "meta%d", i);
102        sprintf(bodybuf, "body%d", i);
103        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
104            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
105        fdb_set(db, doc[i]);
106    }
107
108    // remove document #5
109    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, doc[5]->meta,
110                   doc[5]->metalen, NULL, 0);
111    status = fdb_del(db, rdoc);
112    TEST_CHK(status == FDB_RESULT_SUCCESS);
113    fdb_doc_free(rdoc);
114    rdoc = NULL;
115
116    // commit
117    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
118
119    // check the file info
120    fdb_file_info info;
121    fdb_get_file_info(dbfile, &info);
122    TEST_CHK(info.doc_count == 9);
123    TEST_CHK(info.deleted_count == 1);
124    TEST_CHK(info.space_used > 0);
125    TEST_CHK(info.num_kv_stores == 1);
126
127    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, NULL, 0, NULL, 0);
128    status = fdb_get_metaonly(db, rdoc);
129    TEST_CHK(status == FDB_RESULT_SUCCESS);
130    TEST_CHK(rdoc->deleted == true);
131    TEST_CMP(rdoc->meta, doc[5]->meta, rdoc->metalen);
132    fdb_doc_free(rdoc);
133    rdoc = NULL;
134
135    // close the db
136    fdb_kvs_close(db);
137    fdb_close(dbfile);
138
139    // reopen
140    fdb_open(&dbfile, "./dummy1", &fconfig);
141    fdb_kvs_open_default(dbfile, &db, &kvs_config);
142    status = fdb_set_log_callback(db, logCallbackFunc, (void *) "basic_test");
143    TEST_CHK(status == FDB_RESULT_SUCCESS);
144
145    // update document #0 and #1
146    for (i=0;i<2;++i){
147        sprintf(metabuf, "meta2%d", i);
148        sprintf(bodybuf, "body2%d", i);
149        fdb_doc_update(&doc[i], (void *)metabuf, strlen(metabuf),
150            (void *)bodybuf, strlen(bodybuf));
151        fdb_set(db, doc[i]);
152    }
153
154    // commit
155    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
156
157    // retrieve documents
158    for (i=0;i<n;++i){
159        // search by key
160        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
161        status = fdb_get(db, rdoc);
162
163        if (i != 5) {
164            // updated documents
165            TEST_CHK(status == FDB_RESULT_SUCCESS);
166            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
167            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
168        } else {
169            // removed document
170            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
171            TEST_CHK(!strcmp(fdb_error_msg(status), "key not found"));
172        }
173
174        // free result document
175        fdb_doc_free(rdoc);
176        rdoc = NULL;
177    }
178
179    // do compaction
180    fdb_compact(dbfile, (char *) "./dummy2");
181
182    // retrieve documents after compaction
183    for (i=0;i<n;++i){
184        // search by key
185        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
186        status = fdb_get(db, rdoc);
187
188        if (i != 5) {
189            // updated documents
190            TEST_CHK(status == FDB_RESULT_SUCCESS);
191            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
192            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
193        } else {
194            // removed document
195            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
196        }
197
198        // free result document
199        fdb_doc_free(rdoc);
200        rdoc = NULL;
201    }
202
203    // retrieve documents by sequence number
204    for (i=0; i < n+3; ++i){
205        // search by seq
206        fdb_doc_create(&rdoc, NULL, 0, NULL, 0, NULL, 0);
207        rdoc->seqnum = i + 1;
208        status = fdb_get_byseq(db, rdoc);
209        if ( (i>=2 && i<=4) || (i>=6 && i<=9) || (i>=11 && i<=12)) {
210            // updated documents
211            TEST_CHK(status == FDB_RESULT_SUCCESS);
212        } else {
213            // removed document
214            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
215        }
216
217        // free result document
218        fdb_doc_free(rdoc);
219        rdoc = NULL;
220    }
221
222    // update document #5 with an empty doc body.
223    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, doc[5]->meta,
224                   doc[5]->metalen, NULL, 0);
225    status = fdb_set(db, rdoc);
226    TEST_CHK(status == FDB_RESULT_SUCCESS);
227    fdb_doc_free(rdoc);
228    rdoc = NULL;
229    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
230
231    // Check document #5 with respect to metadata and doc body.
232    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, NULL, 0, NULL, 0);
233    status = fdb_get(db, rdoc);
234    TEST_CHK(status == FDB_RESULT_SUCCESS);
235    TEST_CHK(memcmp(rdoc->meta, doc[5]->meta, rdoc->metalen) == 0);
236    TEST_CHK(rdoc->body == NULL);
237    TEST_CHK(rdoc->bodylen == 0);
238    fdb_doc_free(rdoc);
239    rdoc = NULL;
240
241    // Read-Only mode test: Open succeeds if file exists, but disallow writes
242    fconfig.flags = FDB_OPEN_FLAG_RDONLY;
243    status = fdb_open(&dbfile_rdonly, "./dummy2", &fconfig);
244    TEST_CHK(status == FDB_RESULT_SUCCESS);
245    status = fdb_kvs_open_default(dbfile_rdonly, &db_rdonly, &kvs_config);
246    TEST_CHK(status == FDB_RESULT_SUCCESS);
247
248    fdb_doc_create(&rdoc, doc[0]->key, doc[0]->keylen, NULL, 0, NULL, 0);
249    status = fdb_get(db_rdonly, rdoc);
250    TEST_CHK(status == FDB_RESULT_SUCCESS);
251
252    status = fdb_set_log_callback(db_rdonly, logCallbackFunc,
253                                  (void *) "basic_test");
254    TEST_CHK(status == FDB_RESULT_SUCCESS);
255
256    status = fdb_set(db_rdonly, doc[i]);
257    TEST_CHK(status == FDB_RESULT_RONLY_VIOLATION);
258    TEST_CHK(!strcmp(fdb_error_msg(status), "database is read-only"));
259
260    status = fdb_commit(dbfile_rdonly, FDB_COMMIT_NORMAL);
261    TEST_CHK(status == FDB_RESULT_RONLY_VIOLATION);
262
263    fdb_doc_free(rdoc);
264    rdoc = NULL;
265    fdb_kvs_close(db_rdonly);
266    fdb_close(dbfile_rdonly);
267
268    // free all documents
269    for (i=0;i<n;++i){
270        fdb_doc_free(doc[i]);
271    }
272
273    // do one more compaction
274    fdb_compact(dbfile, (char *) "./dummy3");
275
276    // close db file
277    fdb_kvs_close(db);
278    fdb_close(dbfile);
279
280    // free all resources
281    fdb_shutdown();
282
283    memleak_end();
284
285    TEST_RESULT("basic test");
286}
287
288void init_test()
289{
290    TEST_INIT();
291    memleak_start();
292
293    int r;
294    fdb_status status;
295    fdb_file_handle *dbfile;
296    fdb_kvs_handle *db;
297    fdb_config fconfig = fdb_get_default_config();
298    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
299
300    r = system(SHELL_DEL" dummy* > errorlog.txt");
301    (void)r;
302
303    status = fdb_init(&fconfig);
304    TEST_CHK(status == FDB_RESULT_SUCCESS);
305
306    status = fdb_open(&dbfile, "./dummy1", &fconfig);
307    TEST_CHK(status == FDB_RESULT_SUCCESS);
308    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
309    TEST_CHK(status == FDB_RESULT_SUCCESS);
310
311    status = fdb_close(dbfile);
312    TEST_CHK(status == FDB_RESULT_SUCCESS);
313    status = fdb_shutdown();
314    TEST_CHK(status == FDB_RESULT_SUCCESS);
315
316    memleak_end();
317    TEST_RESULT("init test");
318}
319
320void set_get_max_keylen()
321{
322    TEST_INIT();
323    memleak_start();
324
325    int r;
326    static const int len = FDB_MAX_KEYLEN;
327    char keybuf[len];
328    void *rvalue;
329    size_t rvalue_len;
330    static const char *achar = "a";
331
332    fdb_status status;
333    fdb_file_handle *dbfile;
334    fdb_kvs_handle *db;
335    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
336    fdb_config fconfig = fdb_get_default_config();
337    fconfig.chunksize = 16;
338
339
340    r = system(SHELL_DEL" dummy* > errorlog.txt");
341    (void)r;
342
343    for (int i = 0; i < len; ++i) {
344        keybuf[i] = *achar;
345    }
346    keybuf[len-1] = '\0';
347
348    // open db
349    status = fdb_open(&dbfile, "./dummy1", &fconfig);
350    TEST_CHK(status == FDB_RESULT_SUCCESS);
351    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
352    TEST_CHK(status == FDB_RESULT_SUCCESS);
353
354    // set kv
355    status = fdb_set_kv(db, keybuf, strlen(keybuf), NULL, 0);
356    TEST_CHK(status == FDB_RESULT_SUCCESS);
357
358    // get NULL pointer
359    status = fdb_get(db, NULL);
360    TEST_CHK(status == FDB_RESULT_INVALID_ARGS);
361
362    // get kv
363    status = fdb_get_kv(db, keybuf, strlen(keybuf), &rvalue, &rvalue_len);
364    TEST_CHK(status == FDB_RESULT_SUCCESS);
365
366    fdb_close(dbfile);
367    fdb_shutdown();
368
369    memleak_end();
370    TEST_RESULT("set get max keylen");
371}
372
373void config_test()
374{
375    TEST_INIT();
376
377    memleak_start();
378
379    fdb_file_handle *dbfile;
380    fdb_kvs_handle *db;
381    fdb_status status;
382    fdb_config fconfig;
383    fdb_kvs_config kvs_config;
384    int nfiles = 4;
385    int i;
386    size_t bcache_space_used;
387    char fname[256];
388
389    // remove previous dummy test files
390    int r = system(SHELL_DEL" dummy* > errorlog.txt");
391    (void)r;
392
393    bcache_space_used = fdb_get_buffer_cache_used();
394    TEST_CHK(bcache_space_used == 0);
395
396    fconfig = fdb_get_default_config();
397    fconfig.buffercache_size= (uint64_t) -1;
398    status = fdb_open(&dbfile, "./dummy1", &fconfig);
399    TEST_CHK(status == FDB_RESULT_TOO_BIG_BUFFER_CACHE);
400
401    fconfig = fdb_get_default_config();
402    fconfig.max_writer_lock_prob = 120;
403    status = fdb_open(&dbfile, "./dummy1", &fconfig);
404    TEST_CHK(status == FDB_RESULT_INVALID_CONFIG);
405
406    fconfig = fdb_get_default_config();
407    kvs_config = fdb_get_default_kvs_config();
408    for (i = nfiles; i; --i) {
409        sprintf(fname, "dummy%d", i);
410        status = fdb_open(&dbfile, fname, &fconfig);
411        TEST_CHK(status == FDB_RESULT_SUCCESS);
412        status = fdb_kvs_open(dbfile, &db, "justonekv", &kvs_config);
413        TEST_CHK(status == FDB_RESULT_SUCCESS);
414
415        bcache_space_used = fdb_get_buffer_cache_used();
416
417        fdb_file_info finfo;
418        status = fdb_get_file_info(dbfile, &finfo);
419        TEST_CHK(status == FDB_RESULT_SUCCESS);
420        // Since V3 magic number, 7 blocks are used:
421        // 4 superblocks + KV name header + Stale-tree root node + DB header
422        TEST_CHK(finfo.file_size == fconfig.blocksize * 7);
423        // Buffercache must only have KV name header + stale-tree root
424        TEST_CHK(bcache_space_used == fconfig.blocksize * 2);
425
426        status = fdb_close(dbfile);
427        TEST_CHK(status == FDB_RESULT_SUCCESS);
428    }
429
430    status = fdb_open(&dbfile, fname, &fconfig);
431    TEST_CHK(status == FDB_RESULT_SUCCESS);
432    status = fdb_kvs_open(dbfile, &db, "justonekv", &kvs_config);
433    TEST_CHK(status == FDB_RESULT_SUCCESS);
434    status = fdb_set_kv(db, (void*)"key", 3, (void*)"body", 5);
435    TEST_CHK(status == FDB_RESULT_SUCCESS);
436
437    bcache_space_used = fdb_get_buffer_cache_used();
438
439    // Since V3 magic number, 8 blocks are used:
440    // 7 blocks created eariler + document block for KV pair
441    TEST_CHK(bcache_space_used == fconfig.blocksize * 8);
442
443    fdb_close(dbfile);
444
445    fdb_shutdown();
446
447    memleak_end();
448    TEST_RESULT("forestdb config test");
449}
450
451void delete_reopen_test()
452{
453    TEST_INIT();
454    memleak_start();
455
456    int r;
457    fdb_file_handle *fh;
458    fdb_kvs_handle *db;
459    fdb_status status;
460    fdb_config fconfig;
461
462    r = system(SHELL_DEL " dummy* > errorlog.txt");
463    (void)r;
464
465    fconfig = fdb_get_default_config();
466    fconfig.buffercache_size = 0;
467    fconfig.num_compactor_threads = 1;
468    status = fdb_open(&fh, "./dummy3", &fconfig);
469    TEST_CHK(status == FDB_RESULT_SUCCESS);
470
471    status = fdb_kvs_open_default(fh, &db, NULL);
472    TEST_CHK(status == FDB_RESULT_SUCCESS);
473
474    status = fdb_begin_transaction(fh, FDB_ISOLATION_READ_COMMITTED);
475    TEST_CHK(status == FDB_RESULT_SUCCESS);
476
477    status = fdb_set_kv(db, (void *) "foo", 3, (void *)"value", 5);
478    TEST_CHK(status == FDB_RESULT_SUCCESS);
479
480    status = fdb_end_transaction(fh, FDB_COMMIT_NORMAL);
481    TEST_CHK(status == FDB_RESULT_SUCCESS);
482
483    void *value;
484    size_t valueSize;
485    status = fdb_get_kv(db, (void*)"foo", 3, &value, &valueSize);
486    TEST_CHK(status == FDB_RESULT_SUCCESS);
487
488    TEST_CHK(valueSize == 5);
489    TEST_CMP(value, "value", 5);
490    fdb_free_block(value);
491
492    status = fdb_begin_transaction(fh, FDB_ISOLATION_READ_COMMITTED);
493    TEST_CHK(status == FDB_RESULT_SUCCESS);
494
495    status = fdb_del_kv(db, "foo", 3);
496    TEST_CHK(status == FDB_RESULT_SUCCESS);
497
498    status = fdb_end_transaction(fh, FDB_COMMIT_NORMAL);
499    TEST_CHK(status == FDB_RESULT_SUCCESS);
500
501    status = fdb_get_kv(db, "foo", 3, &value, &valueSize);
502    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
503
504    status = fdb_close(fh);
505    TEST_CHK(status == FDB_RESULT_SUCCESS);
506
507    // Reopen:
508    status = fdb_open(&fh, "./dummy3", &fconfig);
509    TEST_CHK(status == FDB_RESULT_SUCCESS);
510
511    status = fdb_kvs_open_default(fh, &db, NULL);
512    TEST_CHK(status == FDB_RESULT_SUCCESS);
513
514    status = fdb_get_kv(db, "foo", 3, &value, &valueSize);
515    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
516
517    status = fdb_close(fh);
518    TEST_CHK(status == FDB_RESULT_SUCCESS);
519
520    fdb_shutdown();
521
522    memleak_end();
523    TEST_RESULT("end trans delete & reopen passed");
524}
525
526void deleted_doc_get_api_test()
527{
528    TEST_INIT();
529    memleak_start();
530
531    int r;
532    fdb_file_handle *dbfile;
533    fdb_kvs_handle *db;
534    fdb_doc _doc;
535    fdb_doc *doc = &_doc;
536    fdb_doc *rdoc;
537    fdb_status status;
538    fdb_config fconfig;
539    fdb_kvs_config kvs_config;
540    char keybuf[256], bodybuf[256];
541
542    r = system(SHELL_DEL " dummy* > errorlog.txt");
543    (void)r;
544
545    memset(doc, 0, sizeof(fdb_doc));
546    doc->key = &keybuf[0];
547    doc->body = &bodybuf[0];
548    doc->seqnum = SEQNUM_NOT_USED;
549
550    // open dbfile
551    fconfig = fdb_get_default_config();
552    fconfig.purging_interval = 1;
553    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
554    kvs_config = fdb_get_default_kvs_config();
555    status = fdb_open(&dbfile, "./dummy1", &fconfig);
556    TEST_CHK(status == FDB_RESULT_SUCCESS);
557    status = fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
558    TEST_CHK(status == FDB_RESULT_SUCCESS);
559
560    sprintf(keybuf, "key");
561    sprintf(bodybuf, "body");
562    doc->keylen = strlen(keybuf);
563    doc->bodylen = strlen(bodybuf);
564    status = fdb_set(db, doc);
565    TEST_CHK(status == FDB_RESULT_SUCCESS);
566
567    // Commit the doc so it goes into main index
568    status = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
569    TEST_CHK(status == FDB_RESULT_SUCCESS);
570
571    // Delete the doc
572    status = fdb_del(db, doc);
573    TEST_CHK(status == FDB_RESULT_SUCCESS);
574
575    // Commit the doc with wal flush so the delete is appended into the file
576    status = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
577    TEST_CHK(status == FDB_RESULT_SUCCESS);
578
579
580    fdb_doc_create(&rdoc, keybuf, doc->keylen, NULL, 0, NULL, 0);
581
582    // Deleted document should be accessible via fdb_get_metaonly()
583    status = fdb_get_metaonly(db, rdoc);
584    TEST_CHK(status == FDB_RESULT_SUCCESS);
585    TEST_CHK(rdoc->deleted);
586    rdoc->deleted = false;
587
588    // Deleted document should be accessible via fdb_get_metaonly_byseq()
589    status = fdb_get_metaonly_byseq(db, rdoc);
590    TEST_CHK(status == FDB_RESULT_SUCCESS);
591    TEST_CHK(rdoc->deleted);
592    rdoc->deleted = false;
593
594    // Deleted document should be accessible via fdb_get_byoffset()
595    // But the return code must be FDB_RESULT_KEY_NOT_FOUND!
596    status = fdb_get_byoffset(db, rdoc);
597    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
598    TEST_CHK(rdoc->deleted);
599    rdoc->deleted = false;
600
601    // Deleted document should NOT be accessible via fdb_get()
602    status = fdb_get(db, rdoc);
603    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
604    TEST_CHK(!rdoc->deleted);
605    rdoc->deleted = false;
606
607    status = fdb_get_byseq(db, rdoc);
608    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
609    TEST_CHK(!rdoc->deleted);
610
611    fdb_doc_free(rdoc);
612    // close without commit
613    status = fdb_kvs_close(db);
614    TEST_CHK(status == FDB_RESULT_SUCCESS);
615    status = fdb_close(dbfile);
616    TEST_CHK(status == FDB_RESULT_SUCCESS);
617    fdb_shutdown();
618    memleak_end();
619    TEST_RESULT("deleted doc get api test");
620}
621
622void deleted_doc_stat_test()
623{
624    TEST_INIT();
625    memleak_start();
626
627    int r;
628    fdb_file_handle *dbfile;
629    fdb_kvs_handle *db;
630    fdb_doc _doc;
631    fdb_doc *doc = &_doc;
632    fdb_doc *rdoc;
633    fdb_status status;
634    fdb_config fconfig;
635    fdb_file_info info;
636    fdb_kvs_config kvs_config;
637    char keybuf[256], bodybuf[256];
638
639    r = system(SHELL_DEL " dummy* > errorlog.txt");
640    (void)r;
641
642    memset(doc, 0, sizeof(fdb_doc));
643    doc->key = &keybuf[0];
644    doc->body = &bodybuf[0];
645    doc->seqnum = SEQNUM_NOT_USED;
646
647    // open dbfile
648    fconfig = fdb_get_default_config();
649    fconfig.purging_interval = 0;
650    kvs_config = fdb_get_default_kvs_config();
651    status = fdb_open(&dbfile, "./dummy1", &fconfig);
652    TEST_CHK(status == FDB_RESULT_SUCCESS);
653    status = fdb_kvs_open(dbfile, &db, "main", &kvs_config);
654    TEST_CHK(status == FDB_RESULT_SUCCESS);
655
656    sprintf(keybuf, "K"); // This is necessary to set keysize to 2 bytes so
657    sprintf(bodybuf, "body"); // it matches KV_header doc's keysize of 10
658    doc->keylen = strlen(keybuf) + 1; // in multi-kv mode and hits MB-16491
659    doc->bodylen = strlen(bodybuf) + 1;
660    status = fdb_set(db, doc);
661    TEST_CHK(status == FDB_RESULT_SUCCESS);
662
663    // Delete the doc
664    status = fdb_del(db, doc);
665    TEST_CHK(status == FDB_RESULT_SUCCESS);
666
667    // Fetch the doc back
668    fdb_doc_create(&rdoc, doc->key, doc->keylen, NULL, 0, NULL, 0);
669    status = fdb_get(db, rdoc);
670    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
671    fdb_doc_free(rdoc);
672
673    // check the file info
674    fdb_get_file_info(dbfile, &info);
675    TEST_CHK(info.doc_count == 0);
676
677    status = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
678    TEST_CHK(status == FDB_RESULT_SUCCESS);
679    // check the file info again after commit..
680    fdb_get_file_info(dbfile, &info);
681    TEST_CHK(info.doc_count == 0);
682
683    status = fdb_kvs_close(db);
684    TEST_CHK(status == FDB_RESULT_SUCCESS);
685    status = fdb_close(dbfile);
686    TEST_CHK(status == FDB_RESULT_SUCCESS);
687    fdb_shutdown();
688    memleak_end();
689    TEST_RESULT("deleted doc stat test");
690}
691
692// MB-16312
693void complete_delete_test()
694{
695    TEST_INIT();
696
697    int i, r, n = 1000;
698    fdb_file_handle *dbfile;
699    fdb_kvs_handle *db;
700    fdb_iterator *fit;
701    fdb_config config;
702    fdb_kvs_config kvs_config;
703    fdb_status s; (void)s;
704    char path[256];
705    char keybuf[256], valuebuf[256];
706
707    memleak_start();
708
709    sprintf(path, "./dummy1");
710
711    r = system(SHELL_DEL " dummy* > errorlog.txt");
712    (void)r;
713
714    config = fdb_get_default_config();
715    config.buffercache_size = 0;
716    kvs_config = fdb_get_default_kvs_config();
717
718    fdb_open(&dbfile, path, &config);
719    s = fdb_kvs_open(dbfile, &db, "db1", &kvs_config);
720    TEST_CHK(s == FDB_RESULT_SUCCESS);
721
722    for (i=0;i<n;++i){
723        sprintf(keybuf, "key%05d", i);
724        sprintf(valuebuf, "value%05d", i);
725        s = fdb_set_kv(db, keybuf, strlen(keybuf)+1, valuebuf, strlen(valuebuf)+1);
726        TEST_CHK(s == FDB_RESULT_SUCCESS);
727    }
728    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
729
730    for (i=0;i<n;++i){
731        sprintf(keybuf, "key%05d", i);
732        sprintf(valuebuf, "value%05d", i);
733        s = fdb_del_kv(db, keybuf, strlen(keybuf)+1);
734        TEST_CHK(s == FDB_RESULT_SUCCESS);
735    }
736    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
737
738    s = fdb_iterator_init(db, &fit, NULL, 0, NULL, 0, FDB_ITR_NO_DELETES);
739    TEST_CHK(s == FDB_RESULT_SUCCESS);
740    s = fdb_iterator_close(fit);
741    TEST_CHK(s == FDB_RESULT_SUCCESS);
742
743    fdb_close(dbfile);
744    fdb_shutdown();
745
746    memleak_end();
747    TEST_RESULT("complete delete");
748}
749
750void large_batch_write_no_commit_test()
751{
752    TEST_INIT();
753    memleak_start();
754
755    int i, r;
756    int n = 500000;
757    fdb_file_handle *dbfile;
758    fdb_kvs_handle *db;
759    fdb_doc **doc = (fdb_doc **) malloc(sizeof(fdb_doc *) * n);
760    fdb_status status;
761    fdb_config fconfig;
762    fdb_kvs_config kvs_config;
763    char keybuf[256], metabuf[256], bodybuf[256];
764
765    r = system(SHELL_DEL " dummy* > errorlog.txt");
766    (void)r;
767
768    // open dbfile
769    fconfig = fdb_get_default_config();
770    kvs_config = fdb_get_default_kvs_config();
771    status = fdb_open(&dbfile, "./dummy1", &fconfig);
772    TEST_CHK(status == FDB_RESULT_SUCCESS);
773    status = fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
774    TEST_CHK(status == FDB_RESULT_SUCCESS);
775
776    // Write 500K docs to eject and flush some dirty pages into disk.
777    for (i=0;i<n;++i){
778        sprintf(keybuf, "key%128d", i);
779        sprintf(metabuf, "meta%128d", i);
780        sprintf(bodybuf, "body%128d", i);
781        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
782            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
783        fdb_set(db, doc[i]);
784        fdb_doc_free(doc[i]);
785    }
786
787    // close without commit
788    status = fdb_kvs_close(db);
789    TEST_CHK(status == FDB_RESULT_SUCCESS);
790    status = fdb_close(dbfile);
791    TEST_CHK(status == FDB_RESULT_SUCCESS);
792
793    status = fdb_open(&dbfile, "./dummy1", &fconfig);
794    TEST_CHK(status == FDB_RESULT_NO_DB_HEADERS ||
795             status == FDB_RESULT_SUCCESS); // No dirty pages are flushed into disk.
796    if (status == FDB_RESULT_SUCCESS) {
797        status = fdb_close(dbfile);
798        TEST_CHK(status == FDB_RESULT_SUCCESS);
799    }
800
801    free(doc);
802    fdb_shutdown();
803    memleak_end();
804    TEST_RESULT("large batch write test with no commits");
805}
806
807void set_get_meta_test()
808{
809    TEST_INIT();
810    memleak_start();
811
812    int r;
813    char keybuf[256];
814    fdb_file_handle *dbfile;
815    fdb_kvs_handle *db;
816    fdb_doc *rdoc;
817    fdb_status status;
818    fdb_config fconfig = fdb_get_default_config();
819    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
820    fconfig.wal_threshold = 1024;
821    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
822    fconfig.purging_interval = 1;
823    fconfig.flags = FDB_OPEN_FLAG_CREATE;
824
825    // remove previous dummy files
826    r = system(SHELL_DEL" dummy* > errorlog.txt");
827    (void)r;
828
829    // open db
830    fdb_open(&dbfile, "./dummy1", &fconfig);
831    fdb_kvs_open(dbfile, &db, "db1", &kvs_config);
832
833    sprintf(keybuf, "key%d", 0);
834    fdb_doc_create(&rdoc, keybuf, strlen(keybuf), NULL, 0, NULL, 0);
835    fdb_set(db, rdoc);
836    status = fdb_get(db, rdoc);
837    assert(status == FDB_RESULT_SUCCESS);
838    status = fdb_get_byoffset(db, rdoc);
839    TEST_CHK(status == FDB_RESULT_SUCCESS);
840    status = fdb_get_metaonly(db, rdoc);
841    assert(status == FDB_RESULT_SUCCESS);
842    status = fdb_get_metaonly_byseq(db, rdoc);
843    TEST_CHK(status == FDB_RESULT_SUCCESS);
844
845    fdb_del(db, rdoc);
846    status = fdb_get(db, rdoc);
847    assert(status == FDB_RESULT_KEY_NOT_FOUND);
848    assert(rdoc->deleted == true);
849
850    status = fdb_get_metaonly(db, rdoc);
851    assert(status == FDB_RESULT_SUCCESS);
852    assert(rdoc->deleted == true);
853
854    status = fdb_get_metaonly_byseq(db, rdoc);
855    TEST_CHK(status == FDB_RESULT_SUCCESS);
856    assert(rdoc->deleted == true);
857
858    status = fdb_get_byoffset(db, rdoc);
859    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
860    assert(rdoc->deleted == true);
861
862
863    fdb_doc_free(rdoc);
864    fdb_kvs_close(db);
865    fdb_close(dbfile);
866    fdb_shutdown();
867
868    memleak_end();
869    TEST_RESULT("set get meta test");
870}
871
872void long_filename_test()
873{
874    TEST_INIT();
875    memleak_start();
876
877    int i, j, r;
878    int n=15, m=1000;
879    char keyword[] = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
880    char filename[4096], cmd[4096], temp[4096];
881    fdb_file_handle *dbfile;
882    fdb_kvs_handle *db;
883    fdb_config config;
884    fdb_kvs_config kvs_config;
885    fdb_status s;
886    size_t rvalue_len;
887    char key[256], value[256];
888    void *rvalue;
889
890    config = fdb_get_default_config();
891    kvs_config = fdb_get_default_kvs_config();
892    sprintf(temp, SHELL_DMT"%s", keyword);
893
894    // filename longer than 1024 bytes
895    sprintf(filename, "%s", keyword);
896    while (strlen(filename) < 1024) {
897        strcat(filename, keyword);
898    }
899    s = fdb_open(&dbfile, filename, &config);
900    TEST_CHK(s == FDB_RESULT_TOO_LONG_FILENAME);
901
902    // make nested directories for long path
903    // but shorter than 1024 bytes (windows: 256 bytes)
904    sprintf(cmd, SHELL_RMDIR" %s", keyword);
905    r = system(cmd);
906    (void)r;
907    for (i=0;i<n;++i) {
908        sprintf(cmd, SHELL_MKDIR" %s", keyword);
909        for (j=0;j<i;++j){
910            strcat(cmd, temp);
911        }
912        if (strlen(cmd) > SHELL_MAX_PATHLEN) break;
913        r = system(cmd);
914        (void)r;
915    }
916
917    // create DB file
918    sprintf(filename, "%s", keyword);
919    for (j=0;j<i-1;++j){
920        strcat(filename, temp);
921    }
922    strcat(filename, SHELL_DMT"dbfile");
923    s = fdb_open(&dbfile, filename, &config);
924    TEST_CHK(s == FDB_RESULT_SUCCESS);
925    s = fdb_kvs_open_default(dbfile, &db, &kvs_config);
926    TEST_CHK(s == FDB_RESULT_SUCCESS);
927
928    // === write ===
929    for (i=0;i<m;++i){
930        sprintf(key, "key%08d", i);
931        sprintf(value, "value%08d", i);
932        s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
933        TEST_CHK(s == FDB_RESULT_SUCCESS);
934    }
935    s = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
936    TEST_CHK(s == FDB_RESULT_SUCCESS);
937
938    // === read ===
939    for (i=0;i<m;++i){
940        sprintf(key, "key%08d", i);
941        s = fdb_get_kv(db, key, strlen(key)+1, &rvalue, &rvalue_len);
942        TEST_CHK(s == FDB_RESULT_SUCCESS);
943        fdb_free_block(rvalue);
944    }
945
946    s = fdb_kvs_close(db);
947    TEST_CHK(s == FDB_RESULT_SUCCESS);
948    s = fdb_close(dbfile);
949    TEST_CHK(s == FDB_RESULT_SUCCESS);
950    s = fdb_shutdown();
951    TEST_CHK(s == FDB_RESULT_SUCCESS);
952
953    sprintf(cmd, SHELL_RMDIR" %s", keyword);
954    r = system(cmd);
955    (void)r;
956
957    memleak_end();
958    TEST_RESULT("long filename test");
959}
960
961void error_to_str_test()
962{
963    TEST_INIT();
964    memleak_start();
965    int i;
966    const char *err_msg;
967
968    for (i = FDB_RESULT_SUCCESS; i >= FDB_RESULT_LAST; --i) {
969        err_msg = fdb_error_msg((fdb_status)i);
970        // Verify that all error codes have corresponding error messages
971        TEST_CHK(strcmp(err_msg, "unknown error"));
972    }
973
974    err_msg = fdb_error_msg((fdb_status)i);
975    // Verify that the last error code has been checked
976    TEST_CHK(!strcmp(err_msg, "unknown error"));
977
978    memleak_end();
979    TEST_RESULT("error to string message test");
980}
981
982void seq_tree_exception_test()
983{
984    TEST_INIT();
985
986    memleak_start();
987
988    int i, r;
989    int n = 10;
990    fdb_file_handle *dbfile;
991    fdb_kvs_handle *db;
992    fdb_doc **doc = alca(fdb_doc*, n);
993    fdb_doc *rdoc = NULL;
994    fdb_status status;
995    fdb_iterator *it;
996
997    char keybuf[256], metabuf[256], bodybuf[256];
998
999    // remove previous dummy files
1000    r = system(SHELL_DEL" dummy* > errorlog.txt");
1001    (void)r;
1002
1003    fdb_config fconfig = fdb_get_default_config();
1004    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1005    fconfig.seqtree_opt = FDB_SEQTREE_NOT_USE;
1006
1007    // open db
1008    fdb_open(&dbfile, "./dummy1", &fconfig);
1009    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1010    status = fdb_set_log_callback(db, logCallbackFunc,
1011                                  (void *) "seq_tree_exception_test");
1012    TEST_CHK(status == FDB_RESULT_SUCCESS);
1013
1014    // insert documents
1015    for (i=0;i<n;++i){
1016        sprintf(keybuf, "key%d", i);
1017        sprintf(metabuf, "meta%d", i);
1018        sprintf(bodybuf, "body%d", i);
1019        fdb_doc_create(&doc[i],
1020                       (void *)keybuf,  strlen(keybuf),
1021                       (void *)metabuf, strlen(metabuf),
1022                       (void *)bodybuf, strlen(bodybuf));
1023        fdb_set(db, doc[i]);
1024    }
1025
1026    // commit
1027    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1028
1029    status = fdb_compact(dbfile, NULL);
1030    TEST_CHK(status == FDB_RESULT_SUCCESS);
1031
1032    // close the db
1033    fdb_kvs_close(db);
1034    fdb_close(dbfile);
1035
1036    // reopen with seq tree option
1037    fconfig.seqtree_opt = FDB_SEQTREE_USE;
1038    status = fdb_open(&dbfile, "./dummy1", &fconfig);
1039    // must succeed
1040    TEST_CHK(status == FDB_RESULT_SUCCESS);
1041    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
1042    TEST_CHK(status == FDB_RESULT_SUCCESS);
1043    status = fdb_set_log_callback(db, logCallbackFunc,
1044                                  (void *) "seq_tree_exception_test");
1045    TEST_CHK(status == FDB_RESULT_SUCCESS);
1046
1047    // search by seq
1048    fdb_doc_create(&rdoc, NULL, 0, NULL, 0, NULL, 0);
1049    rdoc->seqnum = 1;
1050    status = fdb_get_byseq(db, rdoc);
1051    // must fail
1052    TEST_CHK(status != FDB_RESULT_SUCCESS);
1053
1054    // search meta by seq
1055    status = fdb_get_metaonly_byseq(db, rdoc);
1056    // must fail
1057    TEST_CHK(status != FDB_RESULT_SUCCESS);
1058
1059    // init iterator by seq
1060    status = fdb_iterator_sequence_init(db , &it, 0, 0, FDB_ITR_NONE);
1061    // must fail
1062    TEST_CHK(status != FDB_RESULT_SUCCESS);
1063
1064    // close db file
1065    fdb_kvs_close(db);
1066    fdb_close(dbfile);
1067
1068    // free all documents
1069    free(rdoc);
1070    for (i=0;i<n;++i){
1071        fdb_doc_free(doc[i]);
1072    }
1073
1074    // remove previous dummy files
1075    r = system(SHELL_DEL" dummy* > errorlog.txt");
1076    (void)r;
1077
1078    // open db
1079    fconfig.seqtree_opt = FDB_SEQTREE_USE;
1080    fdb_open(&dbfile, "./dummy1", &fconfig);
1081    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1082    status = fdb_set_log_callback(db, logCallbackFunc,
1083                                  (void *) "seq_tree_exception_test");
1084    TEST_CHK(status == FDB_RESULT_SUCCESS);
1085
1086    // insert documents
1087    for (i=0;i<n;++i){
1088        sprintf(keybuf, "key%d", i);
1089        sprintf(metabuf, "meta%d", i);
1090        sprintf(bodybuf, "body%d", i);
1091        fdb_doc_create(&doc[i],
1092                       (void *)keybuf,  strlen(keybuf),
1093                       (void *)metabuf, strlen(metabuf),
1094                       (void *)bodybuf, strlen(bodybuf));
1095        fdb_set(db, doc[i]);
1096    }
1097
1098    // commit
1099    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1100
1101    // close the db
1102    fdb_kvs_close(db);
1103    fdb_close(dbfile);
1104
1105    // reopen with an option disabling seq tree
1106    fconfig.seqtree_opt = FDB_SEQTREE_NOT_USE;
1107    status = fdb_open(&dbfile, "./dummy1", &fconfig);
1108    // must succeed
1109    TEST_CHK(status == FDB_RESULT_SUCCESS);
1110    status = fdb_close(dbfile);
1111    TEST_CHK(status == FDB_RESULT_SUCCESS);
1112
1113    // free all documents
1114    for (i=0;i<n;++i){
1115        fdb_doc_free(doc[i]);
1116    }
1117
1118    // free all resources
1119    fdb_shutdown();
1120
1121    memleak_end();
1122
1123    TEST_RESULT("sequence tree exception test");
1124}
1125
1126void wal_commit_test()
1127{
1128    TEST_INIT();
1129
1130    memleak_start();
1131
1132    int i, r;
1133    int n = 10;
1134    fdb_file_handle *dbfile;
1135    fdb_kvs_handle *db;
1136    fdb_doc **doc = alca(fdb_doc*, n);
1137    fdb_doc *rdoc = NULL;
1138    fdb_status status;
1139
1140    char keybuf[256], metabuf[256], bodybuf[256];
1141
1142    // remove previous dummy files
1143    r = system(SHELL_DEL" dummy* > errorlog.txt");
1144    (void)r;
1145
1146    fdb_config fconfig = fdb_get_default_config();
1147    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1148    fconfig.buffercache_size = 0;
1149    fconfig.wal_threshold = 1024;
1150    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1151    fconfig.compaction_threshold = 0;
1152
1153    // open db
1154    fdb_open(&dbfile, "./dummy1", &fconfig);
1155    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1156    status = fdb_set_log_callback(db, logCallbackFunc, (void *) "wal_commit_test");
1157    TEST_CHK(status == FDB_RESULT_SUCCESS);
1158
1159    // insert half documents
1160    for (i=0;i<n/2;++i){
1161        sprintf(keybuf, "key%d", i);
1162        sprintf(metabuf, "meta%d", i);
1163        sprintf(bodybuf, "body%d", i);
1164        fdb_doc_create(&doc[i], (void *)keybuf, strlen(keybuf),
1165            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1166        fdb_set(db, doc[i]);
1167    }
1168
1169    // commit
1170    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1171
1172    // insert the other half documents
1173    for (i=n/2;i<n;++i){
1174        sprintf(keybuf, "key%d", i);
1175        sprintf(metabuf, "meta%d", i);
1176        sprintf(bodybuf, "body%d", i);
1177        fdb_doc_create(&doc[i], (void *)keybuf, strlen(keybuf),
1178            (void *)metabuf, strlen(metabuf), (void *)bodybuf, strlen(bodybuf));
1179        fdb_set(db, doc[i]);
1180    }
1181
1182    // close the db
1183    fdb_kvs_close(db);
1184    fdb_close(dbfile);
1185
1186    // reopen
1187    fdb_open(&dbfile, "./dummy1", &fconfig);
1188    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1189    status = fdb_set_log_callback(db, logCallbackFunc, (void *) "wal_commit_test");
1190    TEST_CHK(status == FDB_RESULT_SUCCESS);
1191
1192    // retrieve documents
1193    for (i=0;i<n;++i){
1194        // search by key
1195        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
1196        status = fdb_get(db, rdoc);
1197
1198        if (i < n/2) {
1199            // committed documents
1200            TEST_CHK(status == FDB_RESULT_SUCCESS);
1201            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
1202            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
1203        } else {
1204            // not committed document
1205            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
1206        }
1207
1208        // free result document
1209        fdb_doc_free(rdoc);
1210        rdoc = NULL;
1211    }
1212
1213    // free all documents
1214    for (i=0;i<n;++i){
1215        fdb_doc_free(doc[i]);
1216    }
1217
1218    // close db file
1219    fdb_kvs_close(db);
1220    fdb_close(dbfile);
1221
1222    // free all resources
1223    fdb_shutdown();
1224
1225    memleak_end();
1226
1227    TEST_RESULT("WAL commit test");
1228}
1229
1230void db_close_and_remove()
1231{
1232
1233    TEST_INIT();
1234    memleak_start();
1235
1236    int i, r;
1237    int n = 10;
1238    fdb_file_handle *dbfile;
1239    fdb_kvs_handle *db;
1240    fdb_doc **doc = alca(fdb_doc *, n);
1241    fdb_status status;
1242    fdb_config fconfig;
1243    fdb_kvs_config kvs_config;
1244    char keybuf[256], metabuf[256], bodybuf[256];
1245
1246    r = system(SHELL_DEL " dummy* > errorlog.txt");
1247    (void)r;
1248
1249    // open dbfile
1250    fconfig = fdb_get_default_config();
1251    kvs_config = fdb_get_default_kvs_config();
1252    fconfig.cleanup_cache_onclose = false;
1253    fdb_open(&dbfile, "./dummy1", &fconfig);
1254    fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
1255
1256    // write to db
1257    for (i=0;i<n;++i){
1258        sprintf(keybuf, "key%d", i);
1259        sprintf(metabuf, "meta%d", i);
1260        sprintf(bodybuf, "body%d", i);
1261        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1262            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1263        fdb_set(db, doc[i]);
1264        fdb_doc_free(doc[i]);
1265    }
1266    status = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1267    TEST_CHK(status == FDB_RESULT_SUCCESS);
1268
1269    // close
1270    fdb_kvs_close(db);
1271    fdb_close(dbfile);
1272
1273    // remove dbfile
1274    r = system(SHELL_DEL " dummy* > errorlog.txt");
1275    (void)r;
1276
1277    // re-open read-only
1278    fconfig.flags = FDB_OPEN_FLAG_RDONLY;
1279    status = fdb_open(&dbfile, "./dummy1", &fconfig);
1280    TEST_CHK(status == FDB_RESULT_NO_SUCH_FILE);
1281
1282    fdb_shutdown();
1283    memleak_end();
1284    TEST_RESULT("db close and remove");
1285}
1286
1287void db_drop_test()
1288{
1289    TEST_INIT();
1290
1291    memleak_start();
1292
1293    int i, r;
1294    int n = 3;
1295    fdb_file_handle *dbfile;
1296    fdb_kvs_handle *db;
1297    fdb_doc **doc = alca(fdb_doc *, n);
1298    fdb_doc *rdoc = NULL;
1299    fdb_status status;
1300
1301    char keybuf[256], metabuf[256], bodybuf[256];
1302
1303    // remove previous dummy files
1304    r = system(SHELL_DEL " dummy* > errorlog.txt");
1305    (void)r;
1306
1307    fdb_config fconfig = fdb_get_default_config();
1308    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1309    fconfig.buffercache_size = 16777216;
1310    fconfig.wal_threshold = 1024;
1311    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1312    fconfig.compaction_threshold = 0;
1313
1314    // open db
1315    fdb_open(&dbfile, "./dummy1", &fconfig);
1316    fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
1317    status = fdb_set_log_callback(db, logCallbackFunc,
1318                                  (void *) "db_drop_test");
1319    TEST_CHK(status == FDB_RESULT_SUCCESS);
1320
1321    // insert first two documents
1322    for (i=0;i<2;++i){
1323        sprintf(keybuf, "key%d", i);
1324        sprintf(metabuf, "meta%d", i);
1325        sprintf(bodybuf, "body%d", i);
1326        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1327            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1328        fdb_set(db, doc[i]);
1329    }
1330
1331    // commit
1332    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1333    fdb_kvs_close(db);
1334    fdb_close(dbfile);
1335
1336    // Remove the database file manually.
1337    r = system(SHELL_DEL " dummy1 > errorlog.txt");
1338    (void)r;
1339
1340    // Open the empty db with the same name.
1341    fdb_open(&dbfile, "./dummy1", &fconfig);
1342    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1343    status = fdb_set_log_callback(db, logCallbackFunc,
1344                                  (void *) "db_drop_test");
1345    TEST_CHK(status == FDB_RESULT_SUCCESS);
1346
1347    // now insert a new doc.
1348    sprintf(keybuf, "key%d", 0);
1349    sprintf(metabuf, "meta%d", 0);
1350    sprintf(bodybuf, "body%d", 0);
1351    fdb_doc_free(doc[0]);
1352    fdb_doc_create(&doc[0], (void*)keybuf, strlen(keybuf),
1353        (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1354    fdb_set(db, doc[0]);
1355
1356    // commit
1357    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1358
1359    // search by key
1360    fdb_doc_create(&rdoc, doc[0]->key, doc[0]->keylen, NULL, 0, NULL, 0);
1361    status = fdb_get(db, rdoc);
1362    TEST_CHK(status == FDB_RESULT_SUCCESS);
1363    // Make sure that a doc seqnum starts with one.
1364    TEST_CHK(rdoc->seqnum == 1);
1365
1366    fdb_kvs_close(db);
1367    fdb_close(dbfile);
1368
1369    // free all documents
1370    fdb_doc_free(rdoc);
1371    rdoc = NULL;
1372    for (i=0;i<2;++i){
1373        fdb_doc_free(doc[i]);
1374    }
1375
1376    // free all resources
1377    fdb_shutdown();
1378
1379    memleak_end();
1380
1381    TEST_RESULT("Database drop test");
1382}
1383
1384void db_destroy_test()
1385{
1386    TEST_INIT();
1387
1388    memleak_start();
1389
1390    int i, r;
1391    int n = 30;
1392    fdb_file_handle *dbfile, *dbfile2;
1393    fdb_kvs_handle *db, *db2;
1394    fdb_doc **doc = alca(fdb_doc *, n);
1395    fdb_doc *rdoc = NULL;
1396    fdb_status status;
1397    fdb_config fconfig;
1398    fdb_kvs_config kvs_config;
1399
1400    char keybuf[256], metabuf[256], bodybuf[256];
1401
1402    // remove previous dummy files
1403    r = system(SHELL_DEL " dummy* > errorlog.txt");
1404    (void)r;
1405
1406    fconfig = fdb_get_default_config();
1407    kvs_config = fdb_get_default_kvs_config();
1408    fconfig.buffercache_size = 16777216;
1409    fconfig.wal_threshold = 1024;
1410    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1411    fconfig.compaction_threshold = 0;
1412
1413    // open db
1414    fdb_open(&dbfile, "./dummy1", &fconfig);
1415    fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
1416    status = fdb_set_log_callback(db, logCallbackFunc,
1417                                  (void *) "db_destroy_test");
1418    TEST_CHK(status == FDB_RESULT_SUCCESS);
1419
1420    // insert 30 documents
1421    for (i=0;i<n;++i){
1422        sprintf(keybuf, "key%d", i);
1423        sprintf(metabuf, "meta%d", i);
1424        sprintf(bodybuf, "body%d", i);
1425        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1426            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1427        fdb_set(db, doc[i]);
1428    }
1429
1430    // commit
1431    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1432
1433    // Open the empty db with the same name.
1434    fdb_open(&dbfile2, "./dummy2", &fconfig);
1435    fdb_kvs_open(dbfile2, &db2, NULL, &kvs_config);
1436    status = fdb_set_log_callback(db2, logCallbackFunc,
1437                                  (void *) "db_destroy_test");
1438    TEST_CHK(status == FDB_RESULT_SUCCESS);
1439    // insert 30 documents
1440    for (i=0;i<n;++i){
1441        fdb_set(db2, doc[i]);
1442    }
1443
1444    // commit
1445    fdb_commit(dbfile2, FDB_COMMIT_NORMAL);
1446
1447    // Only close db not db2 and try to destroy
1448    fdb_close(dbfile);
1449
1450    status = fdb_destroy("./dummy2", &fconfig);
1451    TEST_CHK(status == FDB_RESULT_FILE_IS_BUSY);
1452
1453    //Now close the open db file
1454    fdb_close(dbfile2);
1455
1456    status = fdb_destroy("./dummy1", &fconfig);
1457    TEST_CHK(status == FDB_RESULT_SUCCESS);
1458
1459    // Open the same db with the same names.
1460    fdb_open(&dbfile, "./dummy1", &fconfig);
1461    fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
1462    status = fdb_set_log_callback(db, logCallbackFunc,
1463                                  (void *) "db_destroy_test");
1464    TEST_CHK(status == FDB_RESULT_SUCCESS);
1465
1466    // search by key
1467    fdb_doc_create(&rdoc, doc[0]->key, doc[0]->keylen, NULL, 0, NULL, 0);
1468    status = fdb_get(db, rdoc);
1469    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
1470    fdb_close(dbfile);
1471
1472    // free all documents
1473    fdb_doc_free(rdoc);
1474    rdoc = NULL;
1475    for (i=0;i<n;++i){
1476        fdb_doc_free(doc[i]);
1477    }
1478
1479    // free all resources
1480    fdb_shutdown();
1481
1482    memleak_end();
1483
1484    TEST_RESULT("Database destroy test");
1485}
1486
1487// Test for MB-16348
1488void db_destroy_test_full_path()
1489{
1490    TEST_INIT();
1491
1492    memleak_start();
1493    randomize();
1494
1495    int r;
1496    fdb_file_handle *dbfile;
1497    fdb_config config;
1498    fdb_status s;
1499    char path[256];
1500    char cmd[256];
1501
1502    sprintf(path, "/tmp/fdb_destroy_test_%d", random(10000));
1503
1504    sprintf(cmd, "rm -rf %s*", path);
1505    r = system(cmd); (void)r;
1506
1507    config = fdb_get_default_config();
1508    config.compaction_mode = FDB_COMPACTION_AUTO;
1509
1510    fdb_open(&dbfile, path, &config);
1511    fdb_close(dbfile);
1512
1513    s = fdb_destroy(path, &config);
1514    TEST_CHK(s == FDB_RESULT_SUCCESS);
1515
1516    fdb_shutdown();
1517
1518    memleak_end();
1519
1520    TEST_RESULT("Database destroy (full path) test");
1521}
1522
1523void operational_stats_test(bool multi_kv)
1524{
1525    TEST_INIT();
1526
1527    memleak_start();
1528
1529    int i, r;
1530    int n = 10;
1531    int num_kv = 4;
1532    fdb_file_handle *dbfile;
1533    fdb_kvs_handle **db = alca(fdb_kvs_handle*, num_kv);
1534    fdb_doc **doc = alca(fdb_doc*, n);
1535    fdb_doc real_doc;
1536    fdb_doc *rdoc = &real_doc;
1537    fdb_status status;
1538    fdb_iterator *iterator;
1539    fdb_kvs_ops_info info, rinfo;
1540
1541    char keybuf[256], bodybuf[256];
1542    memset(&info, 0, sizeof(fdb_kvs_ops_info));
1543    memset(&real_doc, 0, sizeof(fdb_doc));
1544    real_doc.key = &keybuf;
1545    real_doc.body = &bodybuf;
1546
1547    // remove previous dummy files
1548    r = system(SHELL_DEL" dummy* > errorlog.txt");
1549    (void)r;
1550
1551    fdb_config fconfig = fdb_get_default_config();
1552    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1553
1554    fconfig.buffercache_size = 0;
1555    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
1556    fconfig.wal_threshold = 1024;
1557    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1558    fconfig.compaction_threshold = 0;
1559    fconfig.multi_kv_instances = multi_kv;
1560    r = 0;
1561
1562    fdb_open(&dbfile, "./dummy1", &fconfig);
1563    if (multi_kv) {
1564        num_kv = 4;
1565        for (r = num_kv - 1; r >= 0; --r) {
1566            char tmp[16];
1567            sprintf(tmp, "kv%d", r);
1568            status = fdb_kvs_open(dbfile, &db[r], tmp, &kvs_config);
1569            TEST_CHK(status == FDB_RESULT_SUCCESS);
1570            status = fdb_set_log_callback(db[r], logCallbackFunc,
1571                                          (void *) "operational_stats_test");
1572            TEST_CHK(status == FDB_RESULT_SUCCESS);
1573        }
1574    } else {
1575        num_kv = 1;
1576        status = fdb_kvs_open_default(dbfile, &db[r], &kvs_config);
1577        TEST_CHK(status == FDB_RESULT_SUCCESS);
1578        status = fdb_set_log_callback(db[r], logCallbackFunc,
1579                (void *) "operational_stats_test");
1580        TEST_CHK(status == FDB_RESULT_SUCCESS);
1581    }
1582
1583    for (i = 0; i < n; ++i){
1584        sprintf(keybuf, "key%d", i);
1585        sprintf(bodybuf, "body%d", i);
1586        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf) + 1, NULL, 0,
1587            (void*)bodybuf, strlen(bodybuf)+1);
1588        for (r = num_kv - 1; r >= 0; --r) {
1589            status = fdb_set(db[r], doc[i]);
1590            TEST_CHK(status == FDB_RESULT_SUCCESS);
1591            status = fdb_get_kvs_ops_info(db[r], &rinfo);
1592            TEST_CHK(status == FDB_RESULT_SUCCESS);
1593            info.num_sets = i + 1;
1594            TEST_CMP(&rinfo, &info, sizeof(fdb_kvs_ops_info));
1595        }
1596    }
1597
1598    for (r = num_kv - 1; r >= 0; --r) {
1599        // range scan (before flushing WAL)
1600        fdb_iterator_init(db[r], &iterator, NULL, 0, NULL, 0, 0x0);
1601        i = 0;
1602        do {
1603            status = fdb_iterator_get(iterator, &rdoc);
1604            TEST_CHK(status == FDB_RESULT_SUCCESS);
1605            TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
1606            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
1607            status = fdb_get_kvs_ops_info(db[r], &rinfo);
1608            TEST_CHK(status == FDB_RESULT_SUCCESS);
1609            ++info.num_iterator_gets;
1610            ++info.num_iterator_moves;
1611            TEST_CMP(&rinfo, &info, sizeof(fdb_kvs_ops_info));
1612            ++i;
1613        } while(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
1614        ++info.num_iterator_moves; // account for the last move that failed
1615        fdb_iterator_close(iterator);
1616
1617        fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1618        ++info.num_commits;
1619
1620        status = fdb_get_kvs_ops_info(db[r], &rinfo);
1621        TEST_CHK(status == FDB_RESULT_SUCCESS);
1622        TEST_CMP(&rinfo, &info, sizeof(fdb_kvs_ops_info));
1623
1624        if (r) {
1625            info.num_iterator_gets = 0;
1626            info.num_iterator_moves = 0;
1627        }
1628    }
1629
1630    ++info.num_compacts;
1631    // do compaction
1632    fdb_compact(dbfile, (char *) "./dummy2");
1633
1634    status = fdb_get_kvs_ops_info(db[0], &rinfo);
1635    TEST_CHK(status == FDB_RESULT_SUCCESS);
1636    TEST_CMP(&rinfo, &info, sizeof(fdb_kvs_ops_info));
1637
1638    for (i = 0; i < n; ++i){
1639        sprintf(keybuf, "key%d", i);
1640        for (r = num_kv - 1; r >= 0; --r) {
1641            if (i % 2 == 0) {
1642                if (i % 4 == 0) {
1643                    status = fdb_get_metaonly(db[r], rdoc);
1644                } else {
1645                    rdoc->seqnum = i + 1;
1646                    status = fdb_get_byseq(db[r], rdoc);
1647                }
1648            } else {
1649                status = fdb_get(db[r], rdoc);
1650            }
1651            TEST_CHK(status == FDB_RESULT_SUCCESS);
1652            status = fdb_get_kvs_ops_info(db[r], &rinfo);
1653            TEST_CHK(status == FDB_RESULT_SUCCESS);
1654            info.num_gets = i + 1;
1655            TEST_CMP(&rinfo, &info, sizeof(fdb_kvs_ops_info));
1656        }
1657    }
1658    // also get latency stats..
1659    for (int i = 0; i < FDB_LATENCY_NUM_STATS; ++i) {
1660        fdb_latency_stat stat;
1661        memset(&stat, 0, sizeof(fdb_latency_stat));
1662        status = fdb_get_latency_stats(dbfile, &stat, i);
1663        TEST_CHK(status == FDB_RESULT_SUCCESS);
1664        fprintf(stderr, "%d:\t%u\t%u\t%u\t%" _F64 "\n", i,
1665                stat.lat_max, stat.lat_avg, stat.lat_max, stat.lat_count);
1666    }
1667
1668    fdb_close(dbfile);
1669
1670    // free all documents
1671    for (i=0;i<n;++i){
1672        fdb_doc_free(doc[i]);
1673    }
1674
1675    // free all resources
1676    fdb_shutdown();
1677
1678    memleak_end();
1679
1680    sprintf(bodybuf,"Operational stats test %s", multi_kv ?
1681            "multiple kv instances" : "single kv instance");
1682    TEST_RESULT(bodybuf);
1683}
1684
1685struct work_thread_args{
1686    int tid;
1687    size_t nthreads;
1688    size_t ndocs;
1689    size_t writer;
1690    fdb_doc **doc;
1691    size_t time_sec;
1692    size_t nbatch;
1693    size_t compact_term;
1694    int *n_opened;
1695    int *filename_count;
1696    spin_t *filename_count_lock;
1697    size_t nops;
1698    fdb_config *config;
1699    fdb_kvs_config *kvs_config;
1700};
1701
1702//#define FILENAME "./hdd/dummy"
1703#define FILENAME "dummy"
1704
1705#define KSIZE (100)
1706#define VSIZE (100)
1707#define IDX_DIGIT (7)
1708#define IDX_DIGIT_STR "7"
1709
1710void *_worker_thread(void *voidargs)
1711{
1712    TEST_INIT();
1713
1714    struct work_thread_args *args = (struct work_thread_args *)voidargs;
1715    int i, c, commit_count, filename_count;
1716    struct timeval ts_begin, ts_cur, ts_gap;
1717    fdb_file_handle *dbfile;
1718    fdb_kvs_handle *db;
1719    fdb_status status;
1720    fdb_doc *rdoc = NULL;
1721    char temp[1024];
1722
1723    char cnt_str[IDX_DIGIT+1];
1724    int cnt_int;
1725
1726    filename_count = *args->filename_count;
1727    sprintf(temp, FILENAME"%d", filename_count);
1728    fdb_open(&dbfile, temp, args->config);
1729    fdb_kvs_open_default(dbfile, &db, args->kvs_config);
1730    status = fdb_set_log_callback(db, logCallbackFunc,
1731                                  (void *) "worker_thread");
1732    TEST_CHK(status == FDB_RESULT_SUCCESS);
1733
1734    // wait until all other threads open the DB file.
1735    // (to avoid performing compaction before opening the file)
1736    spin_lock(args->filename_count_lock);
1737    *args->n_opened += 1;
1738    spin_unlock(args->filename_count_lock);
1739    do {
1740        spin_lock(args->filename_count_lock);
1741        if ((size_t)(*args->n_opened) == args->nthreads) {
1742            // all threads open the DB file
1743            spin_unlock(args->filename_count_lock);
1744            break;
1745        }
1746        spin_unlock(args->filename_count_lock);
1747        // sleep 1 sec
1748        sleep(1);
1749    } while (1);
1750
1751    gettimeofday(&ts_begin, NULL);
1752
1753    c = cnt_int = commit_count = 0;
1754    cnt_str[IDX_DIGIT] = 0;
1755
1756    while (1){
1757        i = rand() % args->ndocs;
1758        fdb_doc_create(&rdoc, args->doc[i]->key, args->doc[i]->keylen, NULL, 0, NULL, 0);
1759        status = fdb_get(db, rdoc);
1760
1761        TEST_CHK(status == FDB_RESULT_SUCCESS);
1762        TEST_CMP(rdoc->body, args->doc[i]->body, (IDX_DIGIT+1));
1763
1764        if (args->writer) {
1765            // if writer,
1766            // copy and parse the counter in body
1767            memcpy(cnt_str, (uint8_t *)rdoc->body + (IDX_DIGIT+1), IDX_DIGIT);
1768            cnt_int = atoi(cnt_str);
1769
1770            // increase and rephrase
1771            sprintf(cnt_str, "%0 " IDX_DIGIT_STR "d", ++cnt_int);
1772            memcpy((uint8_t *)rdoc->body + (IDX_DIGIT+1), cnt_str, IDX_DIGIT);
1773
1774            // update and commit
1775            status = fdb_set(db, rdoc);
1776            TEST_CHK(status == FDB_RESULT_SUCCESS);
1777
1778            if (args->nbatch > 0) {
1779                if (c % args->nbatch == 0) {
1780                    // commit for every NBATCH
1781                    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1782                    commit_count++;
1783                    fdb_file_info info;
1784                    fdb_get_file_info(dbfile, &info);
1785                    if (args->compact_term == (size_t)commit_count &&
1786                        args->compact_term > 0 &&
1787                        info.new_filename == NULL &&
1788                        args->tid == 0) {
1789                        // do compaction for every COMPACT_TERM batch
1790                        spin_lock(args->filename_count_lock);
1791                        *args->filename_count += 1;
1792                        filename_count = *args->filename_count;
1793                        spin_unlock(args->filename_count_lock);
1794
1795                        sprintf(temp, FILENAME"%d", filename_count);
1796
1797                        status = fdb_compact(dbfile, temp);
1798                        if (status != FDB_RESULT_SUCCESS) {
1799                            spin_lock(args->filename_count_lock);
1800                            *args->filename_count -= 1;
1801                            spin_unlock(args->filename_count_lock);
1802                        }
1803
1804                        commit_count = 0;
1805                    }
1806                }
1807            }
1808        }
1809        fdb_doc_free(rdoc);
1810        rdoc = NULL;
1811        c++;
1812
1813        gettimeofday(&ts_cur, NULL);
1814        ts_gap = _utime_gap(ts_begin, ts_cur);
1815        if ((size_t)ts_gap.tv_sec >= args->time_sec) break;
1816    }
1817
1818    DBG("Thread #%d (%s) %d ops / %d seconds\n",
1819        args->tid, (args->writer)?("writer"):("reader"), c, (int)args->time_sec);
1820    args->nops = c;
1821
1822    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1823
1824    fdb_kvs_close(db);
1825    fdb_close(dbfile);
1826    thread_exit(0);
1827    return NULL;
1828}
1829
1830void multi_thread_test(
1831    size_t ndocs, size_t wal_threshold, size_t time_sec,
1832    size_t nbatch, size_t compact_term, size_t nwriters, size_t nreaders)
1833{
1834    TEST_INIT();
1835
1836    size_t nwrites, nreads;
1837    int i, r;
1838    int n = nwriters + nreaders;;
1839    thread_t *tid = alca(thread_t, n);
1840    void **thread_ret = alca(void *, n);
1841    struct work_thread_args *args = alca(struct work_thread_args, n);
1842    struct timeval ts_begin, ts_cur, ts_gap;
1843    fdb_file_handle *dbfile;
1844    fdb_kvs_handle *db;
1845    fdb_doc **doc = alca(fdb_doc*, ndocs);
1846    fdb_status status;
1847    fdb_kvs_info kvs_info;
1848
1849    int filename_count = 1;
1850    int n_opened = 0;
1851    spin_t filename_count_lock;
1852    spin_init(&filename_count_lock);
1853
1854    char keybuf[1024], metabuf[1024], bodybuf[1024], temp[1024];
1855
1856    // remove previous dummy files
1857    r = system(SHELL_DEL" " FILENAME "* > errorlog.txt");
1858    (void)r;
1859
1860    fdb_config fconfig = fdb_get_default_config();
1861    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1862    fconfig.buffercache_size = 16777216;
1863    fconfig.wal_threshold = 1024;
1864    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1865    fconfig.compaction_threshold = 0;
1866
1867    memleak_start();
1868
1869    // initial population ===
1870    DBG("Initialize..\n");
1871
1872    // open db
1873    sprintf(temp, FILENAME"%d", filename_count);
1874    fdb_open(&dbfile, temp, &fconfig);
1875    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1876    status = fdb_set_log_callback(db, logCallbackFunc,
1877                                  (void *) "multi_thread_test");
1878    TEST_CHK(status == FDB_RESULT_SUCCESS);
1879
1880    gettimeofday(&ts_begin, NULL);
1881
1882    // insert documents
1883    for (i = 0; (size_t)i < ndocs; ++i){
1884        _set_random_string_smallabt(temp, KSIZE - (IDX_DIGIT+1));
1885        sprintf(keybuf, "k%0" IDX_DIGIT_STR "d%s", i, temp);
1886
1887        sprintf(metabuf, "m%0" IDX_DIGIT_STR "d", i);
1888
1889        _set_random_string_smallabt(temp, VSIZE-(IDX_DIGIT*2+1));
1890        sprintf(bodybuf, "b%0" IDX_DIGIT_STR "d%0" IDX_DIGIT_STR "d%s", i, 0, temp);
1891
1892        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1893            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1894        fdb_set(db, doc[i]);
1895    }
1896
1897    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1898
1899    gettimeofday(&ts_cur, NULL);
1900    ts_gap = _utime_gap(ts_begin, ts_cur);
1901    //DBG("%d.%09d seconds elapsed\n", (int)ts_gap.tv_sec, (int)ts_gap.tv_nsec);
1902
1903    fdb_kvs_close(db);
1904    fdb_close(dbfile);
1905    // end of population ===
1906
1907    // drop OS's page cache
1908    //r = system("free && sync && echo 3 > /proc/sys/vm/drop_caches && free");
1909
1910    // create workers
1911    for (i=0;i<n;++i){
1912        args[i].tid = i;
1913        args[i].nthreads = n;
1914        args[i].writer = (((size_t)i<nwriters)?(1):(0));
1915        args[i].ndocs = ndocs;
1916        args[i].doc = doc;
1917        args[i].time_sec = time_sec;
1918        args[i].nbatch = nbatch;
1919        args[i].compact_term = compact_term;
1920        args[i].n_opened = &n_opened;
1921        args[i].filename_count = &filename_count;
1922        args[i].filename_count_lock = &filename_count_lock;
1923        args[i].config = &fconfig;
1924        args[i].kvs_config = &kvs_config;
1925        thread_create(&tid[i], _worker_thread, &args[i]);
1926    }
1927
1928    printf("wait for %d seconds..\n", (int)time_sec);
1929
1930    // wait for thread termination
1931    for (i=0;i<n;++i){
1932        thread_join(tid[i], &thread_ret[i]);
1933    }
1934
1935    // free all documents
1936    for (i=0;(size_t)i<ndocs;++i){
1937        fdb_doc_free(doc[i]);
1938    }
1939
1940    nwrites = nreads = 0;
1941    for (i=0;i<n;++i){
1942        if (args[i].writer) {
1943            nwrites += args[i].nops;
1944        } else {
1945            nreads += args[i].nops;
1946        }
1947    }
1948    printf("read: %.1f ops/sec\n", (double)nreads/time_sec);
1949    printf("write: %.1f ops/sec\n", (double)nwrites/time_sec);
1950
1951    // check sequence number
1952    sprintf(temp, FILENAME"%d", filename_count);
1953    fdb_open(&dbfile, temp, &fconfig);
1954    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1955    fdb_get_kvs_info(db, &kvs_info);
1956    TEST_CHK(kvs_info.last_seqnum == ndocs+nwrites);
1957    fdb_kvs_close(db);
1958    fdb_close(dbfile);
1959
1960    // shutdown
1961    fdb_shutdown();
1962
1963    memleak_end();
1964
1965    TEST_RESULT("multi thread test");
1966}
1967
1968void *multi_thread_client_shutdown(void *args)
1969{
1970
1971    TEST_INIT();
1972
1973    int i, r;
1974    int nclients;
1975    fdb_file_handle *tdbfile;
1976    fdb_status status;
1977    fdb_config fconfig;
1978    fdb_kvs_config kvs_config;
1979    thread_t *tid;
1980    void **thread_ret;
1981
1982    if (args == NULL)
1983    { // parent
1984        memleak_start();
1985
1986        r = system(SHELL_DEL" dummy* > errorlog.txt");
1987        (void)r;
1988        nclients = 2;
1989        tid = alca(thread_t, nclients);
1990        thread_ret = alca(void *, nclients);
1991        for (i=0;i<nclients;++i){
1992            thread_create(&tid[i], multi_thread_client_shutdown, (void *)&i);
1993        }
1994        for (i=0;i<nclients;++i){
1995            thread_join(tid[i], &thread_ret[i]);
1996        }
1997
1998        memleak_end();
1999        TEST_RESULT("multi thread client shutdown");
2000        return NULL;
2001    }
2002
2003    // threads enter here //
2004
2005    fconfig = fdb_get_default_config();
2006    kvs_config = fdb_get_default_kvs_config();
2007    fconfig.wal_threshold = 1024;
2008    fconfig.compaction_threshold = 0;
2009
2010    // open/close db
2011    status = fdb_open(&tdbfile, "./dummy1", &fconfig);
2012    TEST_CHK(status == FDB_RESULT_SUCCESS);
2013    TEST_CHK(fdb_close(tdbfile) == FDB_RESULT_SUCCESS);
2014
2015    // shutdown
2016    fdb_shutdown();
2017    thread_exit(0);
2018    return NULL;
2019}
2020
2021void *multi_thread_kvs_client(void *args)
2022{
2023
2024    TEST_INIT();
2025
2026    int i, j, r;
2027    int n = 50;
2028    int nclients = 20;
2029    int *tid_args = alca(int, nclients);
2030    char dbstr[256];
2031    char keybuf[256], metabuf[256], bodybuf[256];
2032    fdb_file_handle *dbfile;
2033    fdb_kvs_handle *tdb;
2034    fdb_kvs_handle **db = alca(fdb_kvs_handle*, nclients);
2035    fdb_doc **doc = alca(fdb_doc*, n);
2036    fdb_doc *rdoc = NULL;
2037    fdb_status status;
2038    fdb_config fconfig;
2039    fdb_kvs_config kvs_config;
2040    fdb_seqnum_t seqnum;
2041    thread_t *tid;
2042    void **thread_ret;
2043
2044    if (args == NULL)
2045    { // parent
2046        memleak_start();
2047
2048        r = system(SHELL_DEL" dummy* > errorlog.txt");
2049        (void)r;
2050
2051        // init dbfile
2052        fconfig = fdb_get_default_config();
2053        fconfig.buffercache_size = 0;
2054        fconfig.wal_threshold = 1024;
2055        fconfig.compaction_threshold = 0;
2056
2057        status = fdb_open(&dbfile, "./dummy1", &fconfig);
2058        TEST_CHK(status == FDB_RESULT_SUCCESS);
2059
2060        tid = alca(thread_t, nclients);
2061        thread_ret = alca(void *, nclients);
2062        for (i=0;i<nclients;++i){
2063            sprintf(dbstr, "db%d", i);
2064            kvs_config = fdb_get_default_kvs_config();
2065            status = fdb_kvs_open(dbfile, &db[i], dbstr, &kvs_config);
2066            TEST_CHK(status == FDB_RESULT_SUCCESS);
2067            status = fdb_kvs_close(db[i]);
2068            TEST_CHK(status == FDB_RESULT_SUCCESS);
2069        }
2070        for (i=0;i<nclients;++i){
2071            tid_args[i] = i;
2072            thread_create(&tid[i], multi_thread_kvs_client,
2073                          (void *)&tid_args[i]);
2074        }
2075        for (i=0;i<nclients;++i){
2076            thread_join(tid[i], &thread_ret[i]);
2077        }
2078
2079        status = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2080        TEST_CHK(status == FDB_RESULT_SUCCESS);
2081
2082        // check threads updated kvs
2083        for (i=0; i<nclients; i++){
2084            sprintf(dbstr, "db%d", i);
2085            kvs_config = fdb_get_default_kvs_config();
2086            status = fdb_kvs_open(dbfile, &db[i], dbstr, &kvs_config);
2087            TEST_CHK(status == FDB_RESULT_SUCCESS);
2088
2089            // verify seqnum
2090            status = fdb_get_kvs_seqnum(db[i], &seqnum);
2091            TEST_CHK(status == FDB_RESULT_SUCCESS);
2092            TEST_CHK(seqnum == (fdb_seqnum_t)n);
2093
2094            for (j=0; j<n; j++){
2095                sprintf(keybuf, "key%d", j);
2096                sprintf(metabuf, "meta%d", j);
2097                sprintf(bodybuf, "body%d", j);
2098                fdb_doc_create(&rdoc, keybuf, strlen(keybuf),
2099                                      NULL, 0, NULL, 0);
2100                status = fdb_get(db[i], rdoc);
2101                TEST_CHK(status == FDB_RESULT_SUCCESS);
2102                TEST_CHK(!memcmp(rdoc->key, keybuf, strlen(keybuf)));
2103                TEST_CHK(!memcmp(rdoc->meta, metabuf, rdoc->metalen));
2104                TEST_CHK(!memcmp(rdoc->body, bodybuf, rdoc->bodylen));
2105                fdb_doc_free(rdoc);
2106            }
2107            status = fdb_kvs_close(db[i]);
2108            TEST_CHK(status == FDB_RESULT_SUCCESS);
2109        }
2110
2111        status = fdb_close(dbfile);
2112        TEST_CHK(status == FDB_RESULT_SUCCESS);
2113        fdb_shutdown();
2114        memleak_end();
2115        TEST_RESULT("multi thread kvs client");
2116        return NULL;
2117    }
2118
2119    // threads enter here //
2120
2121    // open fhandle
2122    fconfig = fdb_get_default_config();
2123    status = fdb_open(&dbfile, "./dummy1", &fconfig);
2124    TEST_CHK(status == FDB_RESULT_SUCCESS);
2125
2126    // get kvs ID from args
2127    memcpy(&i, args, sizeof(int));
2128    sprintf(dbstr, "db%d", i);
2129    kvs_config = fdb_get_default_kvs_config();
2130    status = fdb_kvs_open(dbfile, &tdb, dbstr, &kvs_config);
2131    TEST_CHK(status == FDB_RESULT_SUCCESS);
2132
2133    // insert documents
2134    for (i=0;i<n;++i){
2135        sprintf(keybuf, "key%d", i);
2136        sprintf(metabuf, "meta%d", i);
2137        sprintf(bodybuf, "body%d", i);
2138        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2139                                (void*)metabuf, strlen(metabuf),
2140                                (void*)bodybuf, strlen(bodybuf));
2141        status = fdb_set(tdb, doc[i]);
2142        TEST_CHK(status == FDB_RESULT_SUCCESS);
2143        fdb_doc_free(doc[i]);
2144    }
2145    status = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2146    TEST_CHK(status == FDB_RESULT_SUCCESS);
2147
2148    fdb_close(dbfile);
2149    return NULL;
2150}
2151
2152void incomplete_block_test()
2153{
2154    TEST_INIT();
2155
2156    memleak_start();
2157
2158    int i, r;
2159    int n = 2;
2160    fdb_file_handle *dbfile;
2161    fdb_kvs_handle *db;
2162    fdb_doc **doc = alca(fdb_doc*, n);
2163    fdb_doc *rdoc = NULL;
2164    fdb_status status;
2165
2166    char keybuf[256], metabuf[256], bodybuf[256];
2167
2168    // remove previous dummy files
2169    r = system(SHELL_DEL" dummy* > errorlog.txt");
2170    (void)r;
2171
2172    fdb_config fconfig = fdb_get_default_config();
2173    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2174    fconfig.buffercache_size = 0;
2175    fconfig.wal_threshold = 1024;
2176    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2177    fconfig.compaction_threshold = 0;
2178
2179    // open db
2180    fdb_open(&dbfile, "./dummy1", &fconfig);
2181    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2182    status = fdb_set_log_callback(db, logCallbackFunc,
2183                                  (void *) "incomplete_block_test");
2184    TEST_CHK(status == FDB_RESULT_SUCCESS);
2185
2186    // insert documents
2187    for (i=0;i<n;++i){
2188        sprintf(keybuf, "key%d", i);
2189        sprintf(metabuf, "meta%d", i);
2190        sprintf(bodybuf, "body%d", i);
2191        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2192            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2193        fdb_set(db, doc[i]);
2194    }
2195
2196    // retrieve documents
2197    for (i=0;i<n;++i){
2198        // search by key
2199        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
2200        status = fdb_get(db, rdoc);
2201
2202        // updated documents
2203        TEST_CHK(status == FDB_RESULT_SUCCESS);
2204        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2205        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2206
2207        // free result document
2208        fdb_doc_free(rdoc);
2209        rdoc = NULL;
2210    }
2211
2212    // close db file
2213    fdb_kvs_close(db);
2214    fdb_close(dbfile);
2215
2216    // free all documents
2217    for (i=0;i<n;++i){
2218        fdb_doc_free(doc[i]);
2219    }
2220
2221    // free all resources
2222    fdb_shutdown();
2223
2224    memleak_end();
2225
2226    TEST_RESULT("incomplete block test");
2227}
2228
2229
2230static int _cmp_double(void *key1, size_t keylen1, void *key2, size_t keylen2)
2231{
2232    double aa, bb;
2233
2234    if (!keylen1) {
2235        // key1 not set
2236        return -1;
2237    }
2238    if (!keylen2) {
2239        // key2 not set
2240        return 1;
2241    }
2242
2243    aa = *(double *)key1;
2244    bb = *(double *)key2;
2245
2246    if (aa<bb) {
2247        return -1;
2248    } else if (aa>bb) {
2249        return 1;
2250    } else {
2251        return 0;
2252    }
2253}
2254
2255void custom_compare_primitive_test()
2256{
2257    TEST_INIT();
2258
2259    memleak_start();
2260
2261    int i, r;
2262    int n = 10;
2263    fdb_file_handle *dbfile;
2264    fdb_kvs_handle *db;
2265    fdb_doc **doc = alca(fdb_doc*, n);
2266    fdb_doc *rdoc = NULL;
2267    fdb_status status;
2268    fdb_iterator *iterator;
2269
2270    char keybuf[256], bodybuf[256];
2271    double key_double, key_double_prev;
2272
2273    // remove previous dummy files
2274    r = system(SHELL_DEL" dummy* > errorlog.txt");
2275    (void)r;
2276
2277    fdb_config fconfig = fdb_get_default_config();
2278    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2279    fconfig.buffercache_size = 0;
2280    fconfig.wal_threshold = 1024;
2281    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2282    fconfig.compaction_threshold = 0;
2283    fconfig.multi_kv_instances = true;
2284
2285    kvs_config.custom_cmp = _cmp_double;
2286
2287    // open db with custom compare function for double key type
2288    fdb_open(&dbfile, "./dummy1", &fconfig);
2289    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2290    status = fdb_set_log_callback(db, logCallbackFunc,
2291                                  (void *) "custom_compare_primitive_test");
2292    TEST_CHK(status == FDB_RESULT_SUCCESS);
2293
2294    for (i=0;i<n;++i){
2295        key_double = 10000/(i*11.0);
2296        memcpy(keybuf, &key_double, sizeof(key_double));
2297        sprintf(bodybuf, "value: %d, %f", i, key_double);
2298        fdb_doc_create(&doc[i], (void*)keybuf, sizeof(key_double), NULL, 0,
2299            (void*)bodybuf, strlen(bodybuf)+1);
2300        fdb_set(db, doc[i]);
2301    }
2302
2303    // range scan (before flushing WAL)
2304    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2305    key_double_prev = -1;
2306    do {
2307        status = fdb_iterator_get(iterator, &rdoc);
2308        TEST_CHK(status == FDB_RESULT_SUCCESS);
2309        memcpy(&key_double, rdoc->key, rdoc->keylen);
2310        TEST_CHK(key_double > key_double_prev);
2311        key_double_prev = key_double;
2312        fdb_doc_free(rdoc);
2313        rdoc = NULL;
2314    } while(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2315    fdb_iterator_close(iterator);
2316
2317    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2318
2319    // range scan (after flushing WAL)
2320    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2321    key_double_prev = -1;
2322    do {
2323        status = fdb_iterator_get(iterator, &rdoc);
2324        TEST_CHK(status == FDB_RESULT_SUCCESS);
2325        memcpy(&key_double, rdoc->key, rdoc->keylen);
2326        TEST_CHK(key_double > key_double_prev);
2327        key_double_prev = key_double;
2328        fdb_doc_free(rdoc);
2329        rdoc = NULL;
2330    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2331    fdb_iterator_close(iterator);
2332
2333    // do compaction
2334    fdb_compact(dbfile, (char *) "./dummy2");
2335
2336    // range scan (after compaction)
2337    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2338    key_double_prev = -1;
2339    do {
2340        status = fdb_iterator_get(iterator, &rdoc);
2341        TEST_CHK(status == FDB_RESULT_SUCCESS);
2342        memcpy(&key_double, rdoc->key, rdoc->keylen);
2343        TEST_CHK(key_double > key_double_prev);
2344        key_double_prev = key_double;
2345        fdb_doc_free(rdoc);
2346        rdoc = NULL;
2347    } while(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2348    fdb_iterator_close(iterator);
2349
2350    // close db file
2351    fdb_kvs_close(db);
2352    fdb_close(dbfile);
2353
2354    // free all documents
2355    for (i=0;i<n;++i){
2356        fdb_doc_free(doc[i]);
2357    }
2358
2359    // free all resources
2360    fdb_shutdown();
2361
2362    memleak_end();
2363
2364    TEST_RESULT("custom compare function for primitive key test");
2365}
2366
2367static int _cmp_variable(void *key1, size_t keylen1, void *key2, size_t keylen2)
2368{
2369    if (keylen1 < 6 || keylen2 < 6) {
2370        return (keylen1 - keylen2);
2371    }
2372    // compare only 3rd~8th bytes (ignore the others)
2373    return memcmp((uint8_t*)key1+2, (uint8_t*)key2+2, 6);
2374}
2375
2376void custom_compare_variable_test()
2377{
2378    TEST_INIT();
2379
2380    memleak_start();
2381
2382    int i, j, r;
2383    int n = 1000;
2384    int count;
2385    fdb_file_handle *dbfile;
2386    fdb_kvs_handle *db, *db2;
2387    fdb_doc **doc = alca(fdb_doc*, n);
2388    fdb_doc *rdoc = NULL;
2389    fdb_status status;
2390    fdb_iterator *iterator;
2391
2392    size_t keylen = 16;
2393    size_t prev_keylen;
2394    char keybuf[256], bodybuf[256];
2395    char prev_key[256];
2396
2397    // remove previous dummy files
2398    r = system(SHELL_DEL" dummy* > errorlog.txt");
2399    (void)r;
2400
2401    fdb_config fconfig = fdb_get_default_config();
2402    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2403    fconfig.seqtree_opt = FDB_SEQTREE_USE;
2404    fconfig.buffercache_size = 0;
2405    fconfig.wal_threshold = 1024;
2406    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2407    fconfig.compaction_threshold = 0;
2408    fconfig.multi_kv_instances = true;
2409
2410    kvs_config.custom_cmp = _cmp_variable;
2411
2412    // open db with custom compare function for variable length key type
2413    //fdb_open_cmp_variable(&dbfile, "./dummy1", &fconfig);
2414    fdb_open(&dbfile, "./dummy1", &fconfig);
2415    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2416    status = fdb_set_log_callback(db, logCallbackFunc,
2417                                  (void *) "custom_compare_variable_test");
2418    TEST_CHK(status == FDB_RESULT_SUCCESS);
2419
2420    for (i=0;i<n;++i){
2421        for (j=0;j<2;++j){
2422            keybuf[j] = 'a' + rand()%('z'-'a');
2423        }
2424        sprintf(keybuf+2, "%06d", i);
2425        for (j=8;(size_t)j<keylen-1;++j){
2426            keybuf[j] = 'a' + rand()%('z'-'a');
2427        }
2428        keybuf[keylen-1] = 0;
2429        sprintf(bodybuf, "value: %d", i);
2430        fdb_doc_create(&doc[i], (void*)keybuf, keylen, NULL, 0,
2431            (void*)bodybuf, strlen(bodybuf)+1);
2432        fdb_set(db, doc[i]);
2433    }
2434
2435    // point query
2436    for (i=0;i<n;++i){
2437        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
2438        status = fdb_get(db, rdoc);
2439
2440        TEST_CHK(status == FDB_RESULT_SUCCESS);
2441        TEST_CHK(rdoc->bodylen == doc[i]->bodylen);
2442        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2443
2444        fdb_doc_free(rdoc);
2445        rdoc = NULL;
2446    }
2447
2448    // range scan (before flushing WAL)
2449    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2450    sprintf(prev_key, "%016d", 0);
2451    count = 0;
2452    prev_keylen = 16;
2453    do {
2454        status = fdb_iterator_get(iterator, &rdoc);
2455        TEST_CHK(status == FDB_RESULT_SUCCESS);
2456        TEST_CHK(_cmp_variable(prev_key, prev_keylen, rdoc->key, rdoc->keylen) <= 0);
2457        prev_keylen = rdoc->keylen;
2458        memcpy(prev_key, rdoc->key, rdoc->keylen);
2459        fdb_doc_free(rdoc);
2460        rdoc = NULL;
2461        count++;
2462    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2463    TEST_CHK(count == n);
2464    fdb_iterator_close(iterator);
2465
2466    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2467
2468    // range scan (after flushing WAL)
2469    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2470    sprintf(prev_key, "%016d", 0);
2471    count = 0;
2472    prev_keylen = 16;
2473    do {
2474        status = fdb_iterator_get(iterator, &rdoc);
2475        TEST_CHK(status == FDB_RESULT_SUCCESS);
2476        TEST_CHK(_cmp_variable(prev_key, prev_keylen, rdoc->key, rdoc->keylen)
2477                 <= 0);
2478        prev_keylen = rdoc->keylen;
2479        memcpy(prev_key, rdoc->key, rdoc->keylen);
2480        fdb_doc_free(rdoc);
2481        rdoc = NULL;
2482        count++;
2483    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2484    TEST_CHK(count == n);
2485    fdb_iterator_close(iterator);
2486
2487    // do compaction
2488    fdb_compact(dbfile, (char *) "./dummy2");
2489
2490    // range scan (after compaction)
2491    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2492    sprintf(prev_key, "%016d", 0);
2493    count = 0;
2494    prev_keylen = 16;
2495    do {
2496        status = fdb_iterator_get(iterator, &rdoc);
2497        TEST_CHK(status == FDB_RESULT_SUCCESS);
2498        TEST_CHK(_cmp_variable(prev_key, prev_keylen, rdoc->key, rdoc->keylen) <= 0);
2499        prev_keylen = rdoc->keylen;
2500        memcpy(prev_key, rdoc->key, rdoc->keylen);
2501        fdb_doc_free(rdoc);
2502        rdoc = NULL;
2503        count++;
2504    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2505    TEST_CHK(count == n);
2506    fdb_iterator_close(iterator);
2507
2508    // range scan by sequence
2509    fdb_iterator_sequence_init(db, &iterator, 0, 0, 0x0);
2510    count = 0;
2511    do { // forward
2512        status = fdb_iterator_get(iterator, &rdoc);
2513        TEST_CHK(status == FDB_RESULT_SUCCESS);
2514        fdb_doc_free(rdoc);
2515        rdoc = NULL;
2516        count++;
2517    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2518    TEST_CHK(count == n);
2519
2520    // Reverse direction
2521    for (; fdb_iterator_prev(iterator) != FDB_RESULT_ITERATOR_FAIL; --count) {
2522        status = fdb_iterator_get(iterator, &rdoc);
2523        TEST_CHK(status == FDB_RESULT_SUCCESS);
2524        fdb_doc_free(rdoc);
2525        rdoc = NULL;
2526    };
2527    TEST_CHK(count == 0);
2528    fdb_iterator_close(iterator);
2529
2530    // open another handle
2531    kvs_config.custom_cmp = NULL;
2532    fdb_kvs_open_default(dbfile, &db2, &kvs_config);
2533
2534    // point query
2535    for (i=0;i<n;++i){
2536        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
2537        status = fdb_get(db2, rdoc);
2538
2539        TEST_CHK(status == FDB_RESULT_SUCCESS);
2540        TEST_CHK(rdoc->bodylen == doc[i]->bodylen);
2541        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2542
2543        fdb_doc_free(rdoc);
2544        rdoc = NULL;
2545    }
2546
2547    // close db file
2548    fdb_kvs_close(db);
2549    fdb_kvs_close(db2);
2550    fdb_close(dbfile);
2551
2552    // free all documents
2553    for (i=0;i<n;++i){
2554        fdb_doc_free(doc[i]);
2555    }
2556
2557    // free all resources
2558    fdb_shutdown();
2559
2560    memleak_end();
2561
2562    TEST_RESULT("custom compare function for variable length key test");
2563}
2564
2565/*
2566 * custom compare test with commit and compact
2567 *    eqkeys:  boolean to toggle whether bytes in
2568 *             comparision range are equal
2569 */
2570void custom_compare_commit_compact(bool eqkeys)
2571{
2572    TEST_INIT();
2573
2574    memleak_start();
2575
2576    int i, j, r;
2577    int count;
2578    int n = 10;
2579    static const int len = 1024;
2580    char keybuf[len];
2581    static const char *achar = "a";
2582    fdb_doc *rdoc = NULL;
2583    fdb_status status;
2584    fdb_file_handle *dbfile;
2585    fdb_kvs_handle *db;
2586    fdb_iterator *iterator;
2587    fdb_config fconfig = fdb_get_default_config();
2588    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2589
2590    // remove previous dummy files
2591    r = system(SHELL_DEL" dummy* > errorlog.txt");
2592    (void)r;
2593
2594    fconfig.buffercache_size = 0;
2595    fconfig.wal_threshold = 1024;
2596    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2597    fconfig.compaction_threshold = 0;
2598    fconfig.multi_kv_instances = true;
2599
2600    kvs_config.custom_cmp = _cmp_variable;
2601
2602
2603
2604    r = system(SHELL_DEL" dummy* > errorlog.txt");
2605    (void)r;
2606
2607
2608    // open db
2609    status = fdb_open(&dbfile, "./dummy1", &fconfig);
2610    TEST_CHK(status == FDB_RESULT_SUCCESS);
2611    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
2612    TEST_CHK(status == FDB_RESULT_SUCCESS);
2613
2614
2615    for (i=0;i<n;++i){
2616        if(eqkeys){
2617            sprintf(keybuf, "%d", i);
2618            for (j=1;j<len;++j) {
2619                keybuf[j] = *achar;
2620            }
2621        } else {
2622            sprintf(keybuf, "000%d", i);
2623            for (j=4;j<len;++j) {
2624                keybuf[j] = *achar;
2625            }
2626        }
2627        keybuf[len-1] = '\0';
2628        // set kv
2629        status = fdb_set_kv(db, keybuf, strlen(keybuf), NULL, 0);
2630        TEST_CHK(status == FDB_RESULT_SUCCESS);
2631    }
2632
2633    // compact pre & post commit
2634    fdb_compact(dbfile, NULL);
2635    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2636    fdb_compact(dbfile, NULL);
2637
2638    // scan after flush
2639    count = 0;
2640    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2641    do {
2642        status = fdb_iterator_get(iterator, &rdoc);
2643        TEST_CHK(status == FDB_RESULT_SUCCESS);
2644        fdb_doc_free(rdoc);
2645        rdoc = NULL;
2646        count++;
2647    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2648
2649    if (eqkeys) {
2650        // since the custom cmp function compares only 3rd~8th bytes,
2651        // all keys are identified as the same key.
2652        TEST_CHK(count == 1);
2653    } else {
2654        TEST_CHK(count == n);
2655    }
2656
2657    fdb_iterator_close(iterator);
2658
2659    fdb_close(dbfile);
2660    fdb_shutdown();
2661
2662    memleak_end();
2663    TEST_RESULT("custom compare commit compact");
2664
2665}
2666
2667void custom_seqnum_test(bool multi_kv)
2668{
2669    TEST_INIT();
2670
2671    memleak_start();
2672
2673    int i, r;
2674    int n = 10;
2675    int num_kv = 4;
2676    fdb_file_handle *dbfile;
2677    fdb_kvs_handle **db = alca(fdb_kvs_handle*, num_kv);
2678    fdb_doc **doc = alca(fdb_doc*, n);
2679    fdb_doc real_doc;
2680    fdb_doc *rdoc = &real_doc;
2681    fdb_status status;
2682    fdb_iterator *iterator;
2683
2684    char keybuf[256], bodybuf[256];
2685    memset(&real_doc, 0, sizeof(fdb_doc));
2686    real_doc.key = &keybuf;
2687    real_doc.body = &bodybuf;
2688
2689    // remove previous dummy files
2690    r = system(SHELL_DEL" dummy* > errorlog.txt");
2691    (void)r;
2692
2693    fdb_config fconfig = fdb_get_default_config();
2694    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2695
2696    fconfig.buffercache_size = 0;
2697    fconfig.wal_threshold = 1024;
2698    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2699    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
2700    fconfig.compaction_threshold = 0;
2701    fconfig.multi_kv_instances = multi_kv;
2702    r = 0;
2703
2704    fdb_open(&dbfile, "./dummy1", &fconfig);
2705    if (multi_kv) {
2706        num_kv = 4;
2707        for (r = num_kv - 1; r >= 0; --r) {
2708            char tmp[16];
2709            sprintf(tmp, "kv%d", r);
2710            status = fdb_kvs_open(dbfile, &db[r], tmp, &kvs_config);
2711            TEST_CHK(status == FDB_RESULT_SUCCESS);
2712            status = fdb_set_log_callback(db[r], logCallbackFunc,
2713                                          (void *) "custom_seqnum_test");
2714            TEST_CHK(status == FDB_RESULT_SUCCESS);
2715        }
2716    } else {
2717        num_kv = 1;
2718        status = fdb_kvs_open_default(dbfile, &db[r], &kvs_config);
2719        TEST_CHK(status == FDB_RESULT_SUCCESS);
2720        status = fdb_set_log_callback(db[r], logCallbackFunc,
2721                (void *) "custom_seqnum_test");
2722        TEST_CHK(status == FDB_RESULT_SUCCESS);
2723    }
2724
2725    for (i = 0; i < n/2; ++i){
2726        sprintf(keybuf, "key%d", i);
2727        sprintf(bodybuf, "body%d", i);
2728        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf) + 1, NULL, 0,
2729            (void*)bodybuf, strlen(bodybuf)+1);
2730        for (r = num_kv - 1; r >= 0; --r) {
2731            status = fdb_set(db[r], doc[i]);
2732            TEST_CHK(status == FDB_RESULT_SUCCESS);
2733        }
2734    }
2735
2736    for (i = n/2; i < n; ++i){
2737        sprintf(keybuf, "key%d", i);
2738        sprintf(bodybuf, "body%d", i);
2739        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf) + 1, NULL, 0,
2740            (void*)bodybuf, strlen(bodybuf)+1);
2741        for (r = num_kv - 1; r >= 0; --r) {
2742            fdb_doc_set_seqnum(doc[i], (i+1)*2); // double seqnum instead of ++
2743            status = fdb_set(db[r], doc[i]);
2744            TEST_CHK(status == FDB_RESULT_SUCCESS);
2745        }
2746    }
2747
2748    for (r = num_kv - 1; r >= 0; --r) {
2749        // range scan (before flushing WAL)
2750        fdb_iterator_init(db[r], &iterator, NULL, 0, NULL, 0, 0x0);
2751        i = 0;
2752        do {
2753            status = fdb_iterator_get(iterator, &rdoc);
2754            TEST_CHK(status == FDB_RESULT_SUCCESS);
2755            TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2756            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2757            ++i;
2758            if (i <= n/2) {
2759                TEST_CHK(rdoc->seqnum == (fdb_seqnum_t)i);
2760            } else {
2761                TEST_CHK(rdoc->seqnum == (fdb_seqnum_t)i*2);
2762            }
2763        } while(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2764        fdb_iterator_close(iterator);
2765
2766        fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2767    }
2768
2769    // do compaction
2770    fdb_compact(dbfile, (char *) "./dummy2");
2771
2772    for (i = n/2; i < n; ++i){
2773        sprintf(keybuf, "key%d", i);
2774        for (r = num_kv - 1; r >= 0; --r) {
2775            rdoc->seqnum = (i + 1)*2;
2776            status = fdb_get_byseq(db[r], rdoc);
2777            TEST_CHK(status == FDB_RESULT_SUCCESS);
2778        }
2779    }
2780
2781    fdb_close(dbfile);
2782
2783    // free all documents
2784    for (i=0;i<n;++i){
2785        fdb_doc_free(doc[i]);
2786    }
2787
2788    // free all resources
2789    fdb_shutdown();
2790
2791    memleak_end();
2792
2793    sprintf(bodybuf,"Custom sequence number test %s", multi_kv ?
2794            "multiple kv instances" : "single kv instance");
2795    TEST_RESULT(bodybuf);
2796}
2797
2798void doc_compression_test()
2799{
2800    TEST_INIT();
2801
2802    memleak_start();
2803
2804    int i, r;
2805    int n = 10;
2806    int dummy_len = 32;
2807    fdb_file_handle *dbfile;
2808    fdb_kvs_handle *db;
2809    fdb_doc **doc = alca(fdb_doc*, n);
2810    fdb_doc *rdoc = NULL;
2811    fdb_status status;
2812
2813    char keybuf[256], metabuf[256], bodybuf[256], temp[256];
2814
2815    // remove previous dummy files
2816    r = system(SHELL_DEL" dummy* > errorlog.txt");
2817    (void)r;
2818
2819    fdb_config fconfig = fdb_get_default_config();
2820    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2821    fconfig.buffercache_size = 0;
2822    fconfig.wal_threshold = 1024;
2823    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2824    fconfig.compress_document_body = true;
2825    fconfig.compaction_threshold = 0;
2826    fconfig.purging_interval = 1;
2827
2828    // open db
2829    fdb_open(&dbfile, "./dummy1", &fconfig);
2830    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2831    status = fdb_set_log_callback(db, logCallbackFunc,
2832                                  (void *) "doc_compression_test");
2833    TEST_CHK(status == FDB_RESULT_SUCCESS);
2834
2835    // set dummy str
2836    memset(temp, 'a', dummy_len);
2837    temp[dummy_len]=0;
2838
2839    // insert documents
2840    for (i=0;i<n;++i){
2841        sprintf(keybuf, "key%d", i);
2842        sprintf(metabuf, "meta%d", i);
2843        sprintf(bodybuf, "body%d_%s", i, temp);
2844        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2845            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2846        fdb_set(db, doc[i]);
2847    }
2848
2849    // remove document #5
2850    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, doc[5]->meta, doc[5]->metalen, NULL, 0);
2851    status = fdb_del(db, rdoc);
2852    TEST_CHK(status == FDB_RESULT_SUCCESS);
2853    fdb_doc_free(rdoc);
2854    rdoc = NULL;
2855
2856    // commit
2857    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2858
2859    // close the db
2860    fdb_kvs_close(db);
2861    fdb_close(dbfile);
2862
2863    // reopen
2864    fdb_open(&dbfile, "./dummy1", &fconfig);
2865    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2866    status = fdb_set_log_callback(db, logCallbackFunc,
2867                                  (void *) "doc_compression_test");
2868    TEST_CHK(status == FDB_RESULT_SUCCESS);
2869
2870    // update dummy str
2871    dummy_len = 64;
2872    memset(temp, 'b', dummy_len);
2873    temp[dummy_len]=0;
2874
2875    // update document #0 and #1
2876    for (i=0;i<2;++i){
2877        sprintf(metabuf, "newmeta%d", i);
2878        sprintf(bodybuf, "newbody%d_%s", i, temp);
2879        fdb_doc_update(&doc[i], (void *)metabuf, strlen(metabuf),
2880            (void *)bodybuf, strlen(bodybuf));
2881        fdb_set(db, doc[i]);
2882    }
2883
2884    // commit
2885    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2886
2887    // retrieve documents
2888    for (i=0;i<n;++i){
2889        // search by key
2890        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
2891        status = fdb_get(db, rdoc);
2892
2893        if (i != 5) {
2894            // updated documents
2895            TEST_CHK(status == FDB_RESULT_SUCCESS);
2896            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2897            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2898        } else {
2899            // removed document
2900            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
2901        }
2902
2903        // free result document
2904        fdb_doc_free(rdoc);
2905        rdoc = NULL;
2906    }
2907
2908    // do compaction
2909    fdb_compact(dbfile, (char *) "./dummy2");
2910
2911    // retrieve documents after compaction
2912    for (i=0;i<n;++i){
2913        // search by key
2914        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
2915        status = fdb_get(db, rdoc);
2916
2917        if (i != 5) {
2918            // updated documents
2919            TEST_CHK(status == FDB_RESULT_SUCCESS);
2920            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2921            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2922        } else {
2923            // removed document
2924            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
2925        }
2926
2927        // free result document
2928        fdb_doc_free(rdoc);
2929        rdoc = NULL;
2930    }
2931
2932    // free all documents
2933    for (i=0;i<n;++i){
2934        fdb_doc_free(doc[i]);
2935    }
2936
2937    // close db file
2938    fdb_kvs_close(db);
2939    fdb_close(dbfile);
2940
2941    // free all resources
2942    fdb_shutdown();
2943
2944    memleak_end();
2945
2946    TEST_RESULT("document compression test");
2947}
2948
2949void read_doc_by_offset_test()
2950{
2951    TEST_INIT();
2952    memleak_start();
2953
2954    int i, r;
2955    int n = 100;
2956    fdb_file_handle *dbfile;
2957    fdb_kvs_handle *db;
2958    fdb_doc **doc = alca(fdb_doc*, n);
2959    fdb_doc *rdoc, *rdoc1;
2960    fdb_status status;
2961
2962    char keybuf[256], metabuf[256], bodybuf[256];
2963
2964    // remove previous dummy files
2965    r = system(SHELL_DEL" dummy* > errorlog.txt");
2966    (void)r;
2967
2968    fdb_config fconfig = fdb_get_default_config();
2969    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2970    fconfig.buffercache_size = 0;
2971    fconfig.wal_threshold = 1024;
2972    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2973    fconfig.purging_interval = 3600;
2974    fconfig.compaction_threshold = 0;
2975
2976    // open db
2977    fdb_open(&dbfile, "./dummy1", &fconfig);
2978    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2979    status = fdb_set_log_callback(db, logCallbackFunc,
2980                                  (void *) "read_doc_by_offset_test");
2981    TEST_CHK(status == FDB_RESULT_SUCCESS);
2982
2983    // insert documents
2984    for (i=0;i<n;++i){
2985        sprintf(keybuf, "key%d", i);
2986        sprintf(metabuf, "meta%d", i);
2987        sprintf(bodybuf, "body%d", i);
2988        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2989            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2990        fdb_set(db, doc[i]);
2991    }
2992
2993    // commit
2994    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2995
2996    // update documents from #0 to #49
2997    for (i=0;i<n/2;++i){
2998        sprintf(metabuf, "meta2%d", i);
2999        sprintf(bodybuf, "body2%d", i);
3000        fdb_doc_update(&doc[i], (void *)metabuf, strlen(metabuf),
3001            (void *)bodybuf, strlen(bodybuf));
3002        fdb_set(db, doc[i]);
3003    }
3004
3005    // remove document #50
3006    fdb_doc_create(&rdoc, doc[50]->key, doc[50]->keylen, doc[50]->meta,
3007                   doc[50]->metalen, NULL, 0);
3008    status = fdb_del(db, rdoc);
3009    TEST_CHK(status == FDB_RESULT_SUCCESS);
3010    fdb_doc_free(rdoc);
3011    rdoc = NULL;
3012
3013    // commit
3014    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
3015
3016    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, NULL, 0, NULL, 0);
3017    status = fdb_get_metaonly(db, rdoc);
3018    TEST_CHK(status == FDB_RESULT_SUCCESS);
3019    TEST_CHK(rdoc->deleted == false);
3020    TEST_CMP(rdoc->meta, doc[5]->meta, rdoc->metalen);
3021    // Fetch #5 doc using its offset.
3022    status = fdb_get_byoffset(db, rdoc);
3023    TEST_CHK(status == FDB_RESULT_SUCCESS);
3024    TEST_CHK(rdoc->deleted == false);
3025    TEST_CMP(rdoc->meta, doc[5]->meta, rdoc->metalen);
3026    TEST_CMP(rdoc->body, doc[5]->body, rdoc->bodylen);
3027
3028    // MB-13095
3029    fdb_doc_create(&rdoc1, NULL, 0, NULL, 0, NULL, 0);
3030    rdoc1->offset = rdoc->offset;
3031    status = fdb_get_byoffset(db, rdoc1);
3032    TEST_CHK(status == FDB_RESULT_SUCCESS);
3033    TEST_CMP(rdoc1->key, doc[5]->key, rdoc1->keylen);
3034    TEST_CMP(rdoc1->meta, doc[5]->meta, rdoc1->metalen);
3035    TEST_CMP(rdoc1->body, doc[5]->body, rdoc1->bodylen);
3036
3037    fdb_doc_free(rdoc);
3038    rdoc = NULL;
3039    fdb_doc_free(rdoc1);
3040
3041    // do compaction
3042    fdb_compact(dbfile, (char *) "./dummy2");
3043
3044    fdb_doc_create(&rdoc, doc[50]->key, doc[50]->keylen, NULL, 0, NULL, 0);
3045    status = fdb_get_metaonly(db, rdoc);
3046    TEST_CHK(status == FDB_RESULT_SUCCESS);
3047    TEST_CHK(rdoc->deleted == true);
3048    TEST_CMP(rdoc->meta, doc[50]->meta, rdoc->metalen);
3049    // Fetch #50 doc using its offset.
3050    status = fdb_get_byoffset(db, rdoc);
3051    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
3052    TEST_CHK(rdoc->deleted == true);
3053    fdb_doc_free(rdoc);
3054    rdoc = NULL;
3055
3056    // free all documents
3057    for (i=0;i<n;++i){
3058        fdb_doc_free(doc[i]);
3059    }
3060
3061    // close db file
3062    fdb_kvs_close(db);
3063    fdb_close(dbfile);
3064
3065    // free all resources
3066    fdb_shutdown();
3067
3068    memleak_end();
3069
3070    TEST_RESULT("read_doc_by_offset test");
3071}
3072
3073void purge_logically_deleted_doc_test()
3074{
3075    TEST_INIT();
3076
3077    memleak_start();
3078
3079    int i, r;
3080    int n = 10;
3081    fdb_file_handle *dbfile;
3082    fdb_kvs_handle *db;
3083    fdb_doc **doc = alca(fdb_doc*, n);
3084    fdb_doc *rdoc = NULL;
3085    fdb_status status;
3086
3087    char keybuf[256], metabuf[256], bodybuf[256];
3088
3089    // remove previous dummy files
3090    r = system(SHELL_DEL" dummy* fdb_test_config.json > errorlog.txt");
3091    (void)r;
3092
3093    fdb_config fconfig = fdb_get_default_config();
3094    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
3095    fconfig.buffercache_size = 0;
3096    fconfig.wal_threshold = 1024;
3097    fconfig.flags = FDB_OPEN_FLAG_CREATE;
3098    fconfig.purging_interval = 2;
3099    fconfig.compaction_threshold = 0;
3100
3101    // open db
3102    fdb_open(&dbfile, "./dummy1", &fconfig);
3103    fdb_kvs_open_default(dbfile, &db, &kvs_config);
3104    status = fdb_set_log_callback(db, logCallbackFunc,
3105                                  (void *) "purge_logically_deleted_doc_test");
3106    TEST_CHK(status == FDB_RESULT_SUCCESS);
3107
3108    // insert documents
3109    for (i=0;i<n;++i){
3110        sprintf(keybuf, "key%d", i);
3111        sprintf(metabuf, "meta%d", i);
3112        sprintf(bodybuf, "body%d", i);
3113        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
3114            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
3115        fdb_set(db, doc[i]);
3116    }
3117
3118    // remove document #5
3119    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, doc[5]->meta, doc[5]->metalen, NULL, 0);
3120    status = fdb_del(db, rdoc);
3121    TEST_CHK(status == FDB_RESULT_SUCCESS);
3122    fdb_doc_free(rdoc);
3123    rdoc = NULL;
3124
3125    // commit
3126    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
3127
3128    // do compaction
3129    fdb_compact(dbfile, (char *) "./dummy2");
3130
3131    // retrieve documents after compaction
3132    for (i=0;i<n;++i){
3133        // search by key
3134        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
3135        status = fdb_get(db, rdoc);
3136
3137        if (i != 5) {
3138            // updated documents
3139            TEST_CHK(status == FDB_RESULT_SUCCESS);
3140            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
3141            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
3142        } else {
3143            // removed document
3144            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
3145        }
3146        // free result document
3147        fdb_doc_free(rdoc);
3148        rdoc = NULL;
3149
3150        // retrieve metadata
3151        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
3152        status = fdb_get_metaonly(db, rdoc);
3153        if (i != 5) {
3154            TEST_CHK(status == FDB_RESULT_SUCCESS);
3155        } else {
3156            // If the above compaction takes longer than two secs (e.g., slow disk),
3157            // then, fdb_get_metaonly will return KEY_NOT_FOUND error.
3158            TEST_CHK(status == FDB_RESULT_SUCCESS ||
3159                     status == FDB_RESULT_KEY_NOT_FOUND);
3160        }
3161        fdb_doc_free(rdoc);
3162        rdoc = NULL;
3163    }
3164
3165    printf("wait for 3 seconds..\n");
3166    sleep(3);
3167
3168    // do one more compaction
3169    fdb_compact(dbfile, (char *) "./dummy3");
3170
3171    // retrieve documents after compaction
3172    for (i=0;i<n;++i){
3173        // search by key
3174        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
3175        status = fdb_get(db, rdoc);
3176
3177        if (i != 5) {
3178            // updated documents
3179            TEST_CHK(status == FDB_RESULT_SUCCESS);
3180            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
3181            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
3182        } else {
3183            // removed document
3184            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
3185        }
3186        // free result document
3187        fdb_doc_free(rdoc);
3188        rdoc = NULL;
3189
3190        // retrieve metadata
3191        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
3192        status = fdb_get_metaonly(db, rdoc);
3193        if (i != 5) {
3194            TEST_CHK(status == FDB_RESULT_SUCCESS);
3195        } else {
3196            // logically deletec document must be purged during the compaction
3197            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
3198        }
3199        fdb_doc_free(rdoc);
3200        rdoc = NULL;
3201    }
3202
3203    // close db file
3204    fdb_kvs_close(db);
3205    fdb_close(dbfile);
3206
3207    // free all documents
3208    for (i=0;i<n;++i){
3209        fdb_doc_free(doc[i]);
3210    }
3211
3212    // free all resources
3213    fdb_shutdown();
3214
3215    memleak_end();
3216
3217    TEST_RESULT("purge logically deleted doc test");
3218}
3219
3220void api_wrapper_test()
3221{
3222    TEST_INIT();
3223
3224    memleak_start();
3225
3226    int i, r;
3227    int n = 10;
3228    size_t valuelen;
3229    void *value;
3230    fdb_file_handle *dbfile;
3231    fdb_kvs_handle *db;
3232    fdb_status status;
3233
3234    char keybuf[256], bodybuf[256], temp[256];
3235
3236    // remove previous dummy files
3237    r = system(SHELL_DEL" dummy* > errorlog.txt");
3238    (void)r;
3239
3240    fdb_config fconfig = fdb_get_default_config();
3241    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
3242    fconfig.buffercache_size = 0;
3243    fconfig.wal_threshold = 1024;
3244    fconfig.flags = FDB_OPEN_FLAG_CREATE;
3245    fconfig.purging_interval = 0;
3246    fconfig.compaction_threshold = 0;
3247
3248    // open db
3249    fdb_open(&dbfile, "./dummy1", &fconfig);
3250    fdb_kvs_open_default(dbfile, &db, &kvs_config);
3251    status = fdb_set_log_callback(db, logCallbackFunc,
3252                                  (void *) "api_wrapper_test");
3253    TEST_CHK(status == FDB_RESULT_SUCCESS);
3254
3255    // error check
3256    status = fdb_set_kv(db, NULL, 0, NULL, 0);
3257    TEST_CHK(status == FDB_RESULT_INVALID_ARGS);
3258
3259    // insert key-value pairs
3260    for (i=0;i<n;++i){
3261        sprintf(keybuf, "key%d", i);
3262        sprintf(bodybuf, "body%d", i);
3263        status = fdb_set_kv(db, keybuf, strlen(keybuf), bodybuf, strlen(bodybuf));
3264        TEST_CHK(status == FDB_RESULT_SUCCESS);
3265    }
3266
3267    // remove key5
3268    sprintf(keybuf, "key%d", 5);
3269    status = fdb_del_kv(db, keybuf, strlen(keybuf));
3270    TEST_CHK(status == FDB_RESULT_SUCCESS);
3271
3272    // error check
3273    status = fdb_del_kv(db, NULL, 0);
3274    TEST_CHK(status == FDB_RESULT_INVALID_ARGS);
3275
3276    // retrieve key-value pairs
3277    for (i=0;i<n;++i){
3278        sprintf(keybuf, "key%d", i);
3279        status = fdb_get_kv(db, keybuf, strlen(keybuf), &value, &valuelen);
3280
3281        if (i != 5) {
3282            // updated documents
3283            TEST_CHK(status == FDB_RESULT_SUCCESS);
3284            sprintf(temp, "body%d", i);
3285            TEST_CMP(value, temp, valuelen);
3286            fdb_free_block(value);
3287        } else {
3288            // removed document
3289            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
3290        }
3291    }
3292
3293    // error check
3294    status = fdb_get_kv(db, NULL, 0, &value, &valuelen);
3295    TEST_CHK(status == FDB_RESULT_INVALID_ARGS);
3296
3297    status = fdb_get_kv(db, keybuf, strlen(keybuf), NULL, NULL);
3298    TEST_CHK(status == FDB_RESULT_INVALID_ARGS);
3299
3300    // close db file
3301    fdb_kvs_close(db);
3302    fdb_close(dbfile);
3303
3304    // free all resources
3305    fdb_shutdown();
3306
3307    memleak_end();
3308
3309    TEST_RESULT("API wrapper test");
3310}
3311
3312
3313void flush_before_commit_test()
3314{
3315    TEST_INIT();
3316
3317    memleak_start();
3318
3319    int i, r;
3320    int n = 30;
3321    fdb_file_handle *dbfile, *dbfile_txn;
3322    fdb_kvs_handle *db, *db_txn;
3323    fdb_doc **doc = alca(fdb_doc*, n);
3324    fdb_status status;
3325
3326    char keybuf[256], metabuf[256], bodybuf[256];
3327
3328    // remove previous dummy files
3329    r = system(SHELL_DEL" dummy* > errorlog.txt");
3330    (void)r;
3331
3332    fdb_config fconfig = fdb_get_default_config();
3333    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
3334    fconfig.buffercache_size = 0;
3335    fconfig.wal_threshold = 5;
3336    fconfig.flags = FDB_OPEN_FLAG_CREATE;
3337    fconfig.purging_interval = 0;
3338    fconfig.compaction_threshold = 0;
3339    fconfig.wal_flush_before_commit = true;
3340
3341    // open db
3342    fdb_open(&dbfile, "dummy1", &fconfig);
3343    fdb_open(&dbfile_txn, "dummy1", &fconfig);
3344    fdb_kvs_open_default(dbfile, &db, &kvs_config);
3345    fdb_kvs_open_default(dbfile_txn, &db_txn, &kvs_config);
3346    status = fdb_set_log_callback(db_txn, logCallbackFunc,
3347                                  (void *) "flush_before_commit_test");
3348    TEST_CHK(status == FDB_RESULT_SUCCESS);
3349
3350    // create docs
3351    for (i=0;i<n;++i){
3352        sprintf(keybuf, "key%d", i);
3353        sprintf(metabuf, "meta%d", i);
3354        sprintf(bodybuf, "body%d", i);
3355        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
3356                                (void*)metabuf, strlen(metabuf),
3357                                (void*)bodybuf, strlen(bodybuf));
3358    }
3359
3360    // non-transactional commit first, transactional commit next
3361    fdb_begin_transaction(dbfile_txn, FDB_ISOLATION_READ_COMMITTED);
3362    for (i=0;i<2;++i){
3363        fdb_set(db, doc[i]);
3364    }
3365    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
3366    for (i=0;i<2;++i){
3367        fdb_set(db_txn, doc[i]);
3368    }
3369    fdb_end_transaction(dbfile_txn, FDB_COMMIT_NORMAL);
3370    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
3371
3372    // transactional commit first, non-transactional commit next
3373    fdb_begin_transaction(dbfile_txn, FDB_ISOLATION_READ_COMMITTED);
3374    for (i=0;i<2;++i){
3375        fdb_set(db_txn, doc[i]);
3376    }
3377    fdb_end_transaction(dbfile_txn, FDB_COMMIT_NORMAL);
3378    for (i=0;i<2;++i){
3379        fdb_set(db, doc[i]);
3380    }
3381    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
3382    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
3383
3384    // concurrent update (non-txn commit first, txn commit next)
3385    fdb_begin_transaction(dbfile_txn, FDB_ISOLATION_READ_COMMITTED);
3386    for (i=0;i<2;++i){
3387        fdb_set(db_txn, doc[i]);
3388    }
3389    for (i=0;i<2;++i){
3390        fdb_set(db, doc[i]);
3391    }
3392    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
3393    fdb_end_transaction(dbfile_txn, FDB_COMMIT_NORMAL);
3394    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
3395
3396    // concurrent update (txn commit first, non-txn commit next)
3397    fdb_begin_transaction(dbfile_txn, FDB_ISOLATION_READ_COMMITTED);
3398    for (i=0;i<2;++i){
3399        fdb_set(db, doc[i]);
3400    }
3401    for (i=0;i<2;++i){
3402        fdb_set(db_txn, doc[i]);
3403    }
3404    fdb_end_transaction(dbfile_txn, FDB_COMMIT_NORMAL);
3405    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
3406    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
3407
3408    // begin transaction
3409    fdb_begin_transaction(dbfile_txn, FDB_ISOLATION_READ_COMMITTED);
3410
3411    // insert docs using transaction
3412    for (i=0;i<10;++i){
3413        fdb_set(db_txn, doc[i]);
3414    }
3415
3416    // insert docs without transaction
3417    for (i=10;i<20;++i){
3418        fdb_set(db, doc[i]);
3419    }
3420
3421    // do compaction
3422    fdb_compact(dbfile, "dummy2");
3423
3424    for (i=20;i<25;++i){
3425        fdb_set(db_txn, doc[i]);
3426    }
3427    // end transaction
3428    fdb_end_transaction(dbfile_txn, FDB_COMMIT_NORMAL);
3429
3430    for (i=25;i<30;++i){
3431        fdb_set(db, doc[i]);
3432    }
3433
3434    // close db file
3435    fdb_close(dbfile);
3436    fdb_close(dbfile_txn);
3437
3438    // free all documents
3439    for (i=0;i<n;++i){
3440        fdb_doc_free(doc[i]);
3441    }
3442
3443    // free all resources
3444    fdb_shutdown();
3445
3446    memleak_end();
3447
3448    TEST_RESULT("flush before commit test");
3449}
3450
3451void flush_before_commit_multi_writers_test()
3452{
3453    TEST_INIT();
3454
3455    memleak_start();
3456
3457    int i, r;
3458    int n = 10;
3459    fdb_file_handle *dbfile1, *dbfile2;
3460    fdb_kvs_handle *db1, *db2;
3461    fdb_doc **doc = alca(fdb_doc*, n);
3462    fdb_doc *rdoc = NULL;
3463    fdb_status status;
3464    fdb_config fconfig;
3465    fdb_kvs_config kvs_config;
3466
3467    char keybuf[256], metabuf[256], bodybuf[256];
3468
3469    // remove previous dummy files
3470    r = system(SHELL_DEL" dummy* > errorlog.txt");
3471    (void)r;
3472
3473    fconfig = fdb_get_default_config();
3474    fconfig.buffercache_size = 0;
3475    fconfig.wal_threshold = 8;
3476    fconfig.flags = FDB_OPEN_FLAG_CREATE;
3477    fconfig.purging_interval = 0;
3478    fconfig.compaction_threshold = 0;
3479    fconfig.wal_flush_before_commit = true;
3480
3481    kvs_config = fdb_get_default_kvs_config();
3482
3483    // open db
3484    fdb_open(&dbfile1, "dummy1", &fconfig);
3485    fdb_kvs_open(dbfile1, &db1, NULL, &kvs_config);
3486
3487    // create & insert docs
3488    for (i=0;i<n;++i){
3489        sprintf(keybuf, "key%d", i);
3490        sprintf(metabuf, "meta%d", i);
3491        sprintf(bodybuf, "body%d", i);
3492        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
3493                                (void*)metabuf, strlen(metabuf),
3494                                (void*)bodybuf, strlen(bodybuf));
3495        fdb_set(db1, doc[i]);
3496    }
3497    fdb_commit(dbfile1, FDB_COMMIT_MANUAL_WAL_FLUSH);
3498
3499    // open second writer
3500    fdb_open(&dbfile2, "dummy1", &fconfig);
3501    fdb_kvs_open(dbfile2, &db2, NULL, &kvs_config);
3502
3503    for (i=0;i<n/2;++i){
3504        sprintf(metabuf, "meta2%d", i);
3505        sprintf(bodybuf, "body2%d(db2)", i);
3506        fdb_doc_update(&doc[i], (void *)metabuf, strlen(metabuf),
3507            (void *)bodybuf, strlen(bodybuf));
3508        fdb_set(db2, doc[i]);
3509    }
3510    for (i=n/2;i<n;++i){
3511        sprintf(metabuf, "meta2%d", i);
3512        sprintf(bodybuf, "body2%d(db1)", i);
3513        fdb_doc_update(&doc[i], (void *)metabuf, strlen(metabuf),
3514            (void *)bodybuf, strlen(bodybuf));
3515        fdb_set(db1, doc[i]);
3516    }
3517
3518    // retrieve before commit
3519    for (i=0;i<n;++i){
3520        sprintf(keybuf, "key%d", i);
3521        sprintf(metabuf, "meta2%d", i);
3522        if (i < n/2) {
3523            sprintf(bodybuf, "body2%d(db2)", i);
3524        } else {
3525            sprintf(bodybuf, "body2%d(db1)", i);
3526        }
3527        // retrieve through db1
3528        fdb_doc_create(&rdoc, (void*)keybuf, strlen(keybuf),
3529                                NULL, 0, NULL, 0);
3530        status = fdb_get(db1, rdoc);
3531        TEST_CHK(status == FDB_RESULT_SUCCESS);
3532        TEST_CMP(metabuf, rdoc->meta, rdoc->metalen);
3533        TEST_CMP(bodybuf, rdoc->body, rdoc->bodylen);
3534        fdb_doc_free(rdoc);
3535        rdoc = NULL;
3536
3537        // retrieve through db2
3538        fdb_doc_create(&