1/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/*
3 *     Copyright 2010 Couchbase, Inc
4 *
5 *   Licensed under the Apache License, Version 2.0 (the "License");
6 *   you may not use this file except in compliance with the License.
7 *   You may obtain a copy of the License at
8 *
9 *       http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *   Unless required by applicable law or agreed to in writing, software
12 *   distributed under the License is distributed on an "AS IS" BASIS,
13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *   See the License for the specific language governing permissions and
15 *   limitations under the License.
16 */
17
18#include <stdio.h>
19#include <stdlib.h>
20#include <string.h>
21#include <stdint.h>
22#include <time.h>
23#if !defined(WIN32) && !defined(_WIN32)
24#include <unistd.h>
25#endif
26
27#include "libforestdb/forestdb.h"
28#include "test.h"
29#include "internal_types.h"
30#include "functional_util.h"
31
32void basic_test()
33{
34    TEST_INIT();
35
36    memleak_start();
37
38    int i, r;
39    int n = 10;
40    fdb_file_handle *dbfile, *dbfile_rdonly;
41    fdb_kvs_handle *db;
42    fdb_kvs_handle *db_rdonly;
43    fdb_doc **doc = alca(fdb_doc*, n);
44    fdb_doc *rdoc = NULL;
45    fdb_status status;
46
47    char keybuf[256], metabuf[256], bodybuf[256];
48
49    // remove previous dummy test files
50    r = system(SHELL_DEL" dummy* > errorlog.txt");
51    (void)r;
52
53    // Get the ForestDB version
54    const char *version = fdb_get_lib_version();
55    TEST_CHK(version != NULL && strlen(version) > 0);
56
57    fdb_config fconfig = fdb_get_default_config();
58    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
59    fconfig.wal_threshold = 1024;
60    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
61    fconfig.compaction_threshold = 0;
62    fconfig.purging_interval = 1;
63
64    // Read-Write mode test without a create flag.
65    fconfig.flags = 0;
66    status = fdb_open(&dbfile, "./dummy1", &fconfig);
67    TEST_CHK(status == FDB_RESULT_NO_SUCH_FILE);
68    TEST_CHK(!strcmp(fdb_error_msg(status), "no such file"));
69
70    // Read-Only mode test: Must not create new file.
71    fconfig.flags = FDB_OPEN_FLAG_RDONLY;
72    status = fdb_open(&dbfile, "./dummy1", &fconfig);
73    TEST_CHK(status == FDB_RESULT_NO_SUCH_FILE);
74
75    // Read-Only and Create mode: Must not create a new file.
76    fconfig.flags = FDB_OPEN_FLAG_RDONLY | FDB_OPEN_FLAG_CREATE;
77    status = fdb_open(&dbfile, "./dummy1", &fconfig);
78    TEST_CHK(status == FDB_RESULT_INVALID_CONFIG);
79    TEST_CHK(!strcmp(fdb_error_msg(status), "invalid configuration"));
80
81    // open and close db with a create flag.
82    fconfig.flags = FDB_OPEN_FLAG_CREATE;
83    status = fdb_open(&dbfile, "./dummy1", &fconfig);
84    TEST_CHK(status == FDB_RESULT_SUCCESS);
85    TEST_CHK(!strcmp(fdb_error_msg(status), "success"));
86    const char *file_version = fdb_get_file_version(dbfile);
87    TEST_CHK(file_version != NULL && strlen(file_version) > 0);
88    fdb_close(dbfile);
89
90    // reopen db
91    r = system(SHELL_DEL" dummy* > errorlog.txt");
92    (void)r;
93    fdb_open(&dbfile, "./dummy1",&fconfig);
94    fdb_kvs_open_default(dbfile, &db, &kvs_config);
95    status = fdb_set_log_callback(db, logCallbackFunc, (void *) "basic_test");
96    TEST_CHK(status == FDB_RESULT_SUCCESS);
97
98    // insert documents
99    for (i=0;i<n;++i){
100        sprintf(keybuf, "key%d", i);
101        sprintf(metabuf, "meta%d", i);
102        sprintf(bodybuf, "body%d", i);
103        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
104            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
105        fdb_set(db, doc[i]);
106    }
107
108    // remove document #5
109    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, doc[5]->meta,
110                   doc[5]->metalen, NULL, 0);
111    status = fdb_del(db, rdoc);
112    TEST_CHK(status == FDB_RESULT_SUCCESS);
113    fdb_doc_free(rdoc);
114    rdoc = NULL;
115
116    // commit
117    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
118
119    // check the file info
120    fdb_file_info info;
121    fdb_get_file_info(dbfile, &info);
122    TEST_CHK(info.doc_count == 9);
123    TEST_CHK(info.deleted_count == 1);
124    TEST_CHK(info.space_used > 0);
125    TEST_CHK(info.num_kv_stores == 1);
126
127    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, NULL, 0, NULL, 0);
128    status = fdb_get_metaonly(db, rdoc);
129    TEST_CHK(status == FDB_RESULT_SUCCESS);
130    TEST_CHK(rdoc->deleted == true);
131    TEST_CMP(rdoc->meta, doc[5]->meta, rdoc->metalen);
132    fdb_doc_free(rdoc);
133    rdoc = NULL;
134
135    // close the db
136    fdb_kvs_close(db);
137    fdb_close(dbfile);
138
139    // reopen
140    fdb_open(&dbfile, "./dummy1", &fconfig);
141    fdb_kvs_open_default(dbfile, &db, &kvs_config);
142    status = fdb_set_log_callback(db, logCallbackFunc, (void *) "basic_test");
143    TEST_CHK(status == FDB_RESULT_SUCCESS);
144
145    // update document #0 and #1
146    for (i=0;i<2;++i){
147        sprintf(metabuf, "meta2%d", i);
148        sprintf(bodybuf, "body2%d", i);
149        fdb_doc_update(&doc[i], (void *)metabuf, strlen(metabuf),
150            (void *)bodybuf, strlen(bodybuf));
151        fdb_set(db, doc[i]);
152    }
153
154    // commit
155    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
156
157    // retrieve documents
158    for (i=0;i<n;++i){
159        // search by key
160        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
161        status = fdb_get(db, rdoc);
162
163        if (i != 5) {
164            // updated documents
165            TEST_CHK(status == FDB_RESULT_SUCCESS);
166            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
167            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
168        } else {
169            // removed document
170            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
171            TEST_CHK(!strcmp(fdb_error_msg(status), "key not found"));
172        }
173
174        // free result document
175        fdb_doc_free(rdoc);
176        rdoc = NULL;
177    }
178
179    // do compaction
180    fdb_compact(dbfile, (char *) "./dummy2");
181
182    // retrieve documents after compaction
183    for (i=0;i<n;++i){
184        // search by key
185        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
186        status = fdb_get(db, rdoc);
187
188        if (i != 5) {
189            // updated documents
190            TEST_CHK(status == FDB_RESULT_SUCCESS);
191            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
192            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
193        } else {
194            // removed document
195            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
196        }
197
198        // free result document
199        fdb_doc_free(rdoc);
200        rdoc = NULL;
201    }
202
203    // retrieve documents by sequence number
204    for (i=0; i < n+3; ++i){
205        // search by seq
206        fdb_doc_create(&rdoc, NULL, 0, NULL, 0, NULL, 0);
207        rdoc->seqnum = i + 1;
208        status = fdb_get_byseq(db, rdoc);
209        if ( (i>=2 && i<=4) || (i>=6 && i<=9) || (i>=11 && i<=12)) {
210            // updated documents
211            TEST_CHK(status == FDB_RESULT_SUCCESS);
212        } else {
213            // removed document
214            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
215        }
216
217        // free result document
218        fdb_doc_free(rdoc);
219        rdoc = NULL;
220    }
221
222    // update document #5 with an empty doc body.
223    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, doc[5]->meta,
224                   doc[5]->metalen, NULL, 0);
225    status = fdb_set(db, rdoc);
226    TEST_CHK(status == FDB_RESULT_SUCCESS);
227    fdb_doc_free(rdoc);
228    rdoc = NULL;
229    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
230
231    // Check document #5 with respect to metadata and doc body.
232    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, NULL, 0, NULL, 0);
233    status = fdb_get(db, rdoc);
234    TEST_CHK(status == FDB_RESULT_SUCCESS);
235    TEST_CHK(memcmp(rdoc->meta, doc[5]->meta, rdoc->metalen) == 0);
236    TEST_CHK(rdoc->body == NULL);
237    TEST_CHK(rdoc->bodylen == 0);
238    fdb_doc_free(rdoc);
239    rdoc = NULL;
240
241    // Read-Only mode test: Open succeeds if file exists, but disallow writes
242    fconfig.flags = FDB_OPEN_FLAG_RDONLY;
243    status = fdb_open(&dbfile_rdonly, "./dummy2", &fconfig);
244    TEST_CHK(status == FDB_RESULT_SUCCESS);
245    status = fdb_kvs_open_default(dbfile_rdonly, &db_rdonly, &kvs_config);
246    TEST_CHK(status == FDB_RESULT_SUCCESS);
247
248    fdb_doc_create(&rdoc, doc[0]->key, doc[0]->keylen, NULL, 0, NULL, 0);
249    status = fdb_get(db_rdonly, rdoc);
250    TEST_CHK(status == FDB_RESULT_SUCCESS);
251
252    status = fdb_set_log_callback(db_rdonly, logCallbackFunc,
253                                  (void *) "basic_test");
254    TEST_CHK(status == FDB_RESULT_SUCCESS);
255
256    status = fdb_set(db_rdonly, doc[i]);
257    TEST_CHK(status == FDB_RESULT_RONLY_VIOLATION);
258    TEST_CHK(!strcmp(fdb_error_msg(status), "database is read-only"));
259
260    status = fdb_commit(dbfile_rdonly, FDB_COMMIT_NORMAL);
261    TEST_CHK(status == FDB_RESULT_RONLY_VIOLATION);
262
263    fdb_doc_free(rdoc);
264    rdoc = NULL;
265    fdb_kvs_close(db_rdonly);
266    fdb_close(dbfile_rdonly);
267
268    // free all documents
269    for (i=0;i<n;++i){
270        fdb_doc_free(doc[i]);
271    }
272
273    // do one more compaction
274    fdb_compact(dbfile, (char *) "./dummy3");
275
276    // close db file
277    fdb_kvs_close(db);
278    fdb_close(dbfile);
279
280    // free all resources
281    fdb_shutdown();
282
283    memleak_end();
284
285    TEST_RESULT("basic test");
286}
287
288void init_test()
289{
290    TEST_INIT();
291    memleak_start();
292
293    int r;
294    fdb_status status;
295    fdb_file_handle *dbfile;
296    fdb_kvs_handle *db;
297    fdb_config fconfig = fdb_get_default_config();
298    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
299
300    r = system(SHELL_DEL" dummy* > errorlog.txt");
301    (void)r;
302
303    status = fdb_init(&fconfig);
304    TEST_CHK(status == FDB_RESULT_SUCCESS);
305
306    status = fdb_open(&dbfile, "./dummy1", &fconfig);
307    TEST_CHK(status == FDB_RESULT_SUCCESS);
308    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
309    TEST_CHK(status == FDB_RESULT_SUCCESS);
310
311    status = fdb_close(dbfile);
312    TEST_CHK(status == FDB_RESULT_SUCCESS);
313    status = fdb_shutdown();
314    TEST_CHK(status == FDB_RESULT_SUCCESS);
315
316    memleak_end();
317    TEST_RESULT("init test");
318}
319
320void set_get_max_keylen()
321{
322    TEST_INIT();
323    memleak_start();
324
325    int r;
326    static const int len = FDB_MAX_KEYLEN;
327    char keybuf[len];
328    void *rvalue;
329    size_t rvalue_len;
330    static const char *achar = "a";
331
332    fdb_status status;
333    fdb_file_handle *dbfile;
334    fdb_kvs_handle *db;
335    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
336    fdb_config fconfig = fdb_get_default_config();
337    fconfig.chunksize = 16;
338
339
340    r = system(SHELL_DEL" dummy* > errorlog.txt");
341    (void)r;
342
343    for (int i = 0; i < len; ++i) {
344        keybuf[i] = *achar;
345    }
346    keybuf[len-1] = '\0';
347
348    // open db
349    status = fdb_open(&dbfile, "./dummy1", &fconfig);
350    TEST_CHK(status == FDB_RESULT_SUCCESS);
351    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
352    TEST_CHK(status == FDB_RESULT_SUCCESS);
353
354    // set kv
355    status = fdb_set_kv(db, keybuf, strlen(keybuf), NULL, 0);
356    TEST_CHK(status == FDB_RESULT_SUCCESS);
357
358    // get NULL pointer
359    status = fdb_get(db, NULL);
360    TEST_CHK(status == FDB_RESULT_INVALID_ARGS);
361
362    // get kv
363    status = fdb_get_kv(db, keybuf, strlen(keybuf), &rvalue, &rvalue_len);
364    TEST_CHK(status == FDB_RESULT_SUCCESS);
365
366    fdb_close(dbfile);
367    fdb_shutdown();
368
369    memleak_end();
370    TEST_RESULT("set get max keylen");
371}
372
373void config_test()
374{
375    TEST_INIT();
376
377    memleak_start();
378
379    fdb_file_handle *dbfile;
380    fdb_kvs_handle *db;
381    fdb_status status;
382    fdb_config fconfig;
383    fdb_kvs_config kvs_config;
384    int nfiles = 4;
385    int i;
386    size_t bcache_space_used;
387    char fname[256];
388
389    // remove previous dummy test files
390    int r = system(SHELL_DEL" dummy* > errorlog.txt");
391    (void)r;
392
393    bcache_space_used = fdb_get_buffer_cache_used();
394    TEST_CHK(bcache_space_used == 0);
395
396    fconfig = fdb_get_default_config();
397    fconfig.buffercache_size= (uint64_t) -1;
398    status = fdb_open(&dbfile, "./dummy1", &fconfig);
399    TEST_CHK(status == FDB_RESULT_TOO_BIG_BUFFER_CACHE);
400
401    fconfig = fdb_get_default_config();
402    fconfig.max_writer_lock_prob = 120;
403    status = fdb_open(&dbfile, "./dummy1", &fconfig);
404    TEST_CHK(status == FDB_RESULT_INVALID_CONFIG);
405
406    fconfig = fdb_get_default_config();
407    kvs_config = fdb_get_default_kvs_config();
408    for (i = nfiles; i; --i) {
409        sprintf(fname, "dummy%d", i);
410        status = fdb_open(&dbfile, fname, &fconfig);
411        TEST_CHK(status == FDB_RESULT_SUCCESS);
412        status = fdb_kvs_open(dbfile, &db, "justonekv", &kvs_config);
413        TEST_CHK(status == FDB_RESULT_SUCCESS);
414
415        bcache_space_used = fdb_get_buffer_cache_used();
416
417        fdb_file_info finfo;
418        status = fdb_get_file_info(dbfile, &finfo);
419        TEST_CHK(status == FDB_RESULT_SUCCESS);
420        // Since V3 magic number, 7 blocks are used:
421        // 4 superblocks + KV name header + Stale-tree root node + DB header
422        TEST_CHK(finfo.file_size == fconfig.blocksize * 7);
423        // Buffercache must only have KV name header + stale-tree root
424        TEST_CHK(bcache_space_used == fconfig.blocksize * 2);
425
426        status = fdb_close(dbfile);
427        TEST_CHK(status == FDB_RESULT_SUCCESS);
428    }
429
430    status = fdb_open(&dbfile, fname, &fconfig);
431    TEST_CHK(status == FDB_RESULT_SUCCESS);
432    status = fdb_kvs_open(dbfile, &db, "justonekv", &kvs_config);
433    TEST_CHK(status == FDB_RESULT_SUCCESS);
434    status = fdb_set_kv(db, (void*)"key", 3, (void*)"body", 5);
435    TEST_CHK(status == FDB_RESULT_SUCCESS);
436
437    bcache_space_used = fdb_get_buffer_cache_used();
438
439    // Since V3 magic number, 8 blocks are used:
440    // 7 blocks created eariler + document block for KV pair
441    TEST_CHK(bcache_space_used == fconfig.blocksize * 8);
442
443    fdb_close(dbfile);
444
445    fdb_shutdown();
446
447    memleak_end();
448    TEST_RESULT("forestdb config test");
449}
450
451void delete_reopen_test()
452{
453    TEST_INIT();
454    memleak_start();
455
456    int r;
457    fdb_file_handle *fh;
458    fdb_kvs_handle *db;
459    fdb_status status;
460    fdb_config fconfig;
461
462    r = system(SHELL_DEL " dummy* > errorlog.txt");
463    (void)r;
464
465    fconfig = fdb_get_default_config();
466    fconfig.buffercache_size = 0;
467    fconfig.num_compactor_threads = 1;
468    status = fdb_open(&fh, "./dummy3", &fconfig);
469    TEST_CHK(status == FDB_RESULT_SUCCESS);
470
471    status = fdb_kvs_open_default(fh, &db, NULL);
472    TEST_CHK(status == FDB_RESULT_SUCCESS);
473
474    status = fdb_begin_transaction(fh, FDB_ISOLATION_READ_COMMITTED);
475    TEST_CHK(status == FDB_RESULT_SUCCESS);
476
477    status = fdb_set_kv(db, (void *) "foo", 3, (void *)"value", 5);
478    TEST_CHK(status == FDB_RESULT_SUCCESS);
479
480    status = fdb_end_transaction(fh, FDB_COMMIT_NORMAL);
481    TEST_CHK(status == FDB_RESULT_SUCCESS);
482
483    void *value;
484    size_t valueSize;
485    status = fdb_get_kv(db, (void*)"foo", 3, &value, &valueSize);
486    TEST_CHK(status == FDB_RESULT_SUCCESS);
487
488    TEST_CHK(valueSize == 5);
489    TEST_CMP(value, "value", 5);
490    fdb_free_block(value);
491
492    status = fdb_begin_transaction(fh, FDB_ISOLATION_READ_COMMITTED);
493    TEST_CHK(status == FDB_RESULT_SUCCESS);
494
495    status = fdb_del_kv(db, "foo", 3);
496    TEST_CHK(status == FDB_RESULT_SUCCESS);
497
498    status = fdb_end_transaction(fh, FDB_COMMIT_NORMAL);
499    TEST_CHK(status == FDB_RESULT_SUCCESS);
500
501    status = fdb_get_kv(db, "foo", 3, &value, &valueSize);
502    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
503
504    status = fdb_close(fh);
505    TEST_CHK(status == FDB_RESULT_SUCCESS);
506
507    // Reopen:
508    status = fdb_open(&fh, "./dummy3", &fconfig);
509    TEST_CHK(status == FDB_RESULT_SUCCESS);
510
511    status = fdb_kvs_open_default(fh, &db, NULL);
512    TEST_CHK(status == FDB_RESULT_SUCCESS);
513
514    status = fdb_get_kv(db, "foo", 3, &value, &valueSize);
515    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
516
517    status = fdb_close(fh);
518    TEST_CHK(status == FDB_RESULT_SUCCESS);
519
520    fdb_shutdown();
521
522    memleak_end();
523    TEST_RESULT("end trans delete & reopen passed");
524}
525
526void deleted_doc_get_api_test()
527{
528    TEST_INIT();
529    memleak_start();
530
531    int r;
532    fdb_file_handle *dbfile;
533    fdb_kvs_handle *db;
534    fdb_doc _doc;
535    fdb_doc *doc = &_doc;
536    fdb_doc *rdoc;
537    fdb_status status;
538    fdb_config fconfig;
539    fdb_kvs_config kvs_config;
540    char keybuf[256], bodybuf[256];
541
542    r = system(SHELL_DEL " dummy* > errorlog.txt");
543    (void)r;
544
545    memset(doc, 0, sizeof(fdb_doc));
546    doc->key = &keybuf[0];
547    doc->body = &bodybuf[0];
548    doc->seqnum = SEQNUM_NOT_USED;
549
550    // open dbfile
551    fconfig = fdb_get_default_config();
552    fconfig.purging_interval = 1;
553    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
554    kvs_config = fdb_get_default_kvs_config();
555    status = fdb_open(&dbfile, "./dummy1", &fconfig);
556    TEST_CHK(status == FDB_RESULT_SUCCESS);
557    status = fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
558    TEST_CHK(status == FDB_RESULT_SUCCESS);
559
560    sprintf(keybuf, "key");
561    sprintf(bodybuf, "body");
562    doc->keylen = strlen(keybuf);
563    doc->bodylen = strlen(bodybuf);
564    status = fdb_set(db, doc);
565    TEST_CHK(status == FDB_RESULT_SUCCESS);
566
567    // Commit the doc so it goes into main index
568    status = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
569    TEST_CHK(status == FDB_RESULT_SUCCESS);
570
571    // Delete the doc
572    status = fdb_del(db, doc);
573    TEST_CHK(status == FDB_RESULT_SUCCESS);
574
575    // Commit the doc with wal flush so the delete is appended into the file
576    status = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
577    TEST_CHK(status == FDB_RESULT_SUCCESS);
578
579
580    fdb_doc_create(&rdoc, keybuf, doc->keylen, NULL, 0, NULL, 0);
581
582    // Deleted document should be accessible via fdb_get_metaonly()
583    status = fdb_get_metaonly(db, rdoc);
584    TEST_CHK(status == FDB_RESULT_SUCCESS);
585    TEST_CHK(rdoc->deleted);
586    rdoc->deleted = false;
587
588    // Deleted document should be accessible via fdb_get_metaonly_byseq()
589    status = fdb_get_metaonly_byseq(db, rdoc);
590    TEST_CHK(status == FDB_RESULT_SUCCESS);
591    TEST_CHK(rdoc->deleted);
592    rdoc->deleted = false;
593
594    // Deleted document should be accessible via fdb_get_byoffset()
595    // But the return code must be FDB_RESULT_KEY_NOT_FOUND!
596    status = fdb_get_byoffset(db, rdoc);
597    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
598    TEST_CHK(rdoc->deleted);
599    rdoc->deleted = false;
600
601    // Deleted document should NOT be accessible via fdb_get()
602    status = fdb_get(db, rdoc);
603    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
604    TEST_CHK(!rdoc->deleted);
605    rdoc->deleted = false;
606
607    status = fdb_get_byseq(db, rdoc);
608    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
609    TEST_CHK(!rdoc->deleted);
610
611    fdb_doc_free(rdoc);
612    // close without commit
613    status = fdb_kvs_close(db);
614    TEST_CHK(status == FDB_RESULT_SUCCESS);
615    status = fdb_close(dbfile);
616    TEST_CHK(status == FDB_RESULT_SUCCESS);
617    fdb_shutdown();
618    memleak_end();
619    TEST_RESULT("deleted doc get api test");
620}
621
622void deleted_doc_stat_test()
623{
624    TEST_INIT();
625    memleak_start();
626
627    int r;
628    fdb_file_handle *dbfile;
629    fdb_kvs_handle *db;
630    fdb_doc _doc;
631    fdb_doc *doc = &_doc;
632    fdb_doc *rdoc;
633    fdb_status status;
634    fdb_config fconfig;
635    fdb_file_info info;
636    fdb_kvs_config kvs_config;
637    char keybuf[256], bodybuf[256];
638
639    r = system(SHELL_DEL " dummy* > errorlog.txt");
640    (void)r;
641
642    memset(doc, 0, sizeof(fdb_doc));
643    doc->key = &keybuf[0];
644    doc->body = &bodybuf[0];
645    doc->seqnum = SEQNUM_NOT_USED;
646
647    // open dbfile
648    fconfig = fdb_get_default_config();
649    fconfig.purging_interval = 0;
650    kvs_config = fdb_get_default_kvs_config();
651    status = fdb_open(&dbfile, "./dummy1", &fconfig);
652    TEST_CHK(status == FDB_RESULT_SUCCESS);
653    status = fdb_kvs_open(dbfile, &db, "main", &kvs_config);
654    TEST_CHK(status == FDB_RESULT_SUCCESS);
655
656    sprintf(keybuf, "K"); // This is necessary to set keysize to 2 bytes so
657    sprintf(bodybuf, "body"); // it matches KV_header doc's keysize of 10
658    doc->keylen = strlen(keybuf) + 1; // in multi-kv mode and hits MB-16491
659    doc->bodylen = strlen(bodybuf) + 1;
660    status = fdb_set(db, doc);
661    TEST_CHK(status == FDB_RESULT_SUCCESS);
662
663    // Delete the doc
664    status = fdb_del(db, doc);
665    TEST_CHK(status == FDB_RESULT_SUCCESS);
666
667    // Fetch the doc back
668    fdb_doc_create(&rdoc, doc->key, doc->keylen, NULL, 0, NULL, 0);
669    status = fdb_get(db, rdoc);
670    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
671    fdb_doc_free(rdoc);
672
673    // check the file info
674    fdb_get_file_info(dbfile, &info);
675    TEST_CHK(info.doc_count == 0);
676
677    status = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
678    TEST_CHK(status == FDB_RESULT_SUCCESS);
679    // check the file info again after commit..
680    fdb_get_file_info(dbfile, &info);
681    TEST_CHK(info.doc_count == 0);
682
683    status = fdb_kvs_close(db);
684    TEST_CHK(status == FDB_RESULT_SUCCESS);
685    status = fdb_close(dbfile);
686    TEST_CHK(status == FDB_RESULT_SUCCESS);
687    fdb_shutdown();
688    memleak_end();
689    TEST_RESULT("deleted doc stat test");
690}
691
692// MB-16312
693void complete_delete_test()
694{
695    TEST_INIT();
696
697    int i, r, n = 1000;
698    fdb_file_handle *dbfile;
699    fdb_kvs_handle *db;
700    fdb_iterator *fit;
701    fdb_config config;
702    fdb_kvs_config kvs_config;
703    fdb_status s; (void)s;
704    char path[256];
705    char keybuf[256], valuebuf[256];
706
707    memleak_start();
708
709    sprintf(path, "./dummy1");
710
711    r = system(SHELL_DEL " dummy* > errorlog.txt");
712    (void)r;
713
714    config = fdb_get_default_config();
715    config.buffercache_size = 0;
716    kvs_config = fdb_get_default_kvs_config();
717
718    fdb_open(&dbfile, path, &config);
719    s = fdb_kvs_open(dbfile, &db, "db1", &kvs_config);
720    TEST_CHK(s == FDB_RESULT_SUCCESS);
721
722    for (i=0;i<n;++i){
723        sprintf(keybuf, "key%05d", i);
724        sprintf(valuebuf, "value%05d", i);
725        s = fdb_set_kv(db, keybuf, strlen(keybuf)+1, valuebuf, strlen(valuebuf)+1);
726        TEST_CHK(s == FDB_RESULT_SUCCESS);
727    }
728    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
729
730    for (i=0;i<n;++i){
731        sprintf(keybuf, "key%05d", i);
732        sprintf(valuebuf, "value%05d", i);
733        s = fdb_del_kv(db, keybuf, strlen(keybuf)+1);
734        TEST_CHK(s == FDB_RESULT_SUCCESS);
735    }
736    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
737
738    s = fdb_iterator_init(db, &fit, NULL, 0, NULL, 0, FDB_ITR_NO_DELETES);
739    TEST_CHK(s == FDB_RESULT_SUCCESS);
740    s = fdb_iterator_close(fit);
741    TEST_CHK(s == FDB_RESULT_SUCCESS);
742
743    fdb_close(dbfile);
744    fdb_shutdown();
745
746    memleak_end();
747    TEST_RESULT("complete delete");
748}
749
750void large_batch_write_no_commit_test()
751{
752    TEST_INIT();
753    memleak_start();
754
755    int i, r;
756    int n = 500000;
757    fdb_file_handle *dbfile;
758    fdb_kvs_handle *db;
759    fdb_doc **doc = (fdb_doc **) malloc(sizeof(fdb_doc *) * n);
760    fdb_status status;
761    fdb_config fconfig;
762    fdb_kvs_config kvs_config;
763    char keybuf[256], metabuf[256], bodybuf[256];
764
765    r = system(SHELL_DEL " dummy* > errorlog.txt");
766    (void)r;
767
768    // open dbfile
769    fconfig = fdb_get_default_config();
770    kvs_config = fdb_get_default_kvs_config();
771    status = fdb_open(&dbfile, "./dummy1", &fconfig);
772    TEST_CHK(status == FDB_RESULT_SUCCESS);
773    status = fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
774    TEST_CHK(status == FDB_RESULT_SUCCESS);
775
776    // Write 500K docs to eject and flush some dirty pages into disk.
777    for (i=0;i<n;++i){
778        sprintf(keybuf, "key%128d", i);
779        sprintf(metabuf, "meta%128d", i);
780        sprintf(bodybuf, "body%128d", i);
781        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
782            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
783        fdb_set(db, doc[i]);
784        fdb_doc_free(doc[i]);
785    }
786
787    // close without commit
788    status = fdb_kvs_close(db);
789    TEST_CHK(status == FDB_RESULT_SUCCESS);
790    status = fdb_close(dbfile);
791    TEST_CHK(status == FDB_RESULT_SUCCESS);
792
793    status = fdb_open(&dbfile, "./dummy1", &fconfig);
794    TEST_CHK(status == FDB_RESULT_NO_DB_HEADERS ||
795             status == FDB_RESULT_SUCCESS); // No dirty pages are flushed into disk.
796    if (status == FDB_RESULT_SUCCESS) {
797        status = fdb_close(dbfile);
798        TEST_CHK(status == FDB_RESULT_SUCCESS);
799    }
800
801    free(doc);
802    fdb_shutdown();
803    memleak_end();
804    TEST_RESULT("large batch write test with no commits");
805}
806
807void set_get_meta_test()
808{
809    TEST_INIT();
810    memleak_start();
811
812    int r;
813    char keybuf[256];
814    fdb_file_handle *dbfile;
815    fdb_kvs_handle *db;
816    fdb_doc *rdoc;
817    fdb_status status;
818    fdb_config fconfig = fdb_get_default_config();
819    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
820    fconfig.wal_threshold = 1024;
821    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
822    fconfig.purging_interval = 1;
823    fconfig.flags = FDB_OPEN_FLAG_CREATE;
824
825    // remove previous dummy files
826    r = system(SHELL_DEL" dummy* > errorlog.txt");
827    (void)r;
828
829    // open db
830    fdb_open(&dbfile, "./dummy1", &fconfig);
831    fdb_kvs_open(dbfile, &db, "db1", &kvs_config);
832
833    sprintf(keybuf, "key%d", 0);
834    fdb_doc_create(&rdoc, keybuf, strlen(keybuf), NULL, 0, NULL, 0);
835    fdb_set(db, rdoc);
836    status = fdb_get(db, rdoc);
837    assert(status == FDB_RESULT_SUCCESS);
838    status = fdb_get_byoffset(db, rdoc);
839    TEST_CHK(status == FDB_RESULT_SUCCESS);
840    status = fdb_get_metaonly(db, rdoc);
841    assert(status == FDB_RESULT_SUCCESS);
842    status = fdb_get_metaonly_byseq(db, rdoc);
843    TEST_CHK(status == FDB_RESULT_SUCCESS);
844
845    fdb_del(db, rdoc);
846    status = fdb_get(db, rdoc);
847    assert(status == FDB_RESULT_KEY_NOT_FOUND);
848    assert(rdoc->deleted == true);
849
850    status = fdb_get_metaonly(db, rdoc);
851    assert(status == FDB_RESULT_SUCCESS);
852    assert(rdoc->deleted == true);
853
854    status = fdb_get_metaonly_byseq(db, rdoc);
855    TEST_CHK(status == FDB_RESULT_SUCCESS);
856    assert(rdoc->deleted == true);
857
858    status = fdb_get_byoffset(db, rdoc);
859    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
860    assert(rdoc->deleted == true);
861
862
863    fdb_doc_free(rdoc);
864    fdb_kvs_close(db);
865    fdb_close(dbfile);
866    fdb_shutdown();
867
868    memleak_end();
869    TEST_RESULT("set get meta test");
870}
871
872void long_filename_test()
873{
874    TEST_INIT();
875    memleak_start();
876
877    int i, j, r;
878    int n=15, m=1000;
879    char keyword[] = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
880    char filename[4096], cmd[4096], temp[4096];
881    fdb_file_handle *dbfile;
882    fdb_kvs_handle *db;
883    fdb_config config;
884    fdb_kvs_config kvs_config;
885    fdb_status s;
886    size_t rvalue_len;
887    char key[256], value[256];
888    void *rvalue;
889
890    config = fdb_get_default_config();
891    kvs_config = fdb_get_default_kvs_config();
892    sprintf(temp, SHELL_DMT"%s", keyword);
893
894    // filename longer than 1024 bytes
895    sprintf(filename, "%s", keyword);
896    while (strlen(filename) < 1024) {
897        strcat(filename, keyword);
898    }
899    s = fdb_open(&dbfile, filename, &config);
900    TEST_CHK(s == FDB_RESULT_TOO_LONG_FILENAME);
901
902    // make nested directories for long path
903    // but shorter than 1024 bytes (windows: 256 bytes)
904    sprintf(cmd, SHELL_RMDIR" %s", keyword);
905    r = system(cmd);
906    (void)r;
907    for (i=0;i<n;++i) {
908        sprintf(cmd, SHELL_MKDIR" %s", keyword);
909        for (j=0;j<i;++j){
910            strcat(cmd, temp);
911        }
912        if (strlen(cmd) > SHELL_MAX_PATHLEN) break;
913        r = system(cmd);
914        (void)r;
915    }
916
917    // create DB file
918    sprintf(filename, "%s", keyword);
919    for (j=0;j<i-1;++j){
920        strcat(filename, temp);
921    }
922    strcat(filename, SHELL_DMT"dbfile");
923    s = fdb_open(&dbfile, filename, &config);
924    TEST_CHK(s == FDB_RESULT_SUCCESS);
925    s = fdb_kvs_open_default(dbfile, &db, &kvs_config);
926    TEST_CHK(s == FDB_RESULT_SUCCESS);
927
928    // === write ===
929    for (i=0;i<m;++i){
930        sprintf(key, "key%08d", i);
931        sprintf(value, "value%08d", i);
932        s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
933        TEST_CHK(s == FDB_RESULT_SUCCESS);
934    }
935    s = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
936    TEST_CHK(s == FDB_RESULT_SUCCESS);
937
938    // === read ===
939    for (i=0;i<m;++i){
940        sprintf(key, "key%08d", i);
941        s = fdb_get_kv(db, key, strlen(key)+1, &rvalue, &rvalue_len);
942        TEST_CHK(s == FDB_RESULT_SUCCESS);
943        fdb_free_block(rvalue);
944    }
945
946    s = fdb_kvs_close(db);
947    TEST_CHK(s == FDB_RESULT_SUCCESS);
948    s = fdb_close(dbfile);
949    TEST_CHK(s == FDB_RESULT_SUCCESS);
950    s = fdb_shutdown();
951    TEST_CHK(s == FDB_RESULT_SUCCESS);
952
953    sprintf(cmd, SHELL_RMDIR" %s", keyword);
954    r = system(cmd);
955    (void)r;
956
957    memleak_end();
958    TEST_RESULT("long filename test");
959}
960
961void error_to_str_test()
962{
963    TEST_INIT();
964    memleak_start();
965    int i;
966    const char *err_msg;
967
968    for (i = FDB_RESULT_SUCCESS; i >= FDB_RESULT_LAST; --i) {
969        err_msg = fdb_error_msg((fdb_status)i);
970        // Verify that all error codes have corresponding error messages
971        TEST_CHK(strcmp(err_msg, "unknown error"));
972    }
973
974    err_msg = fdb_error_msg((fdb_status)i);
975    // Verify that the last error code has been checked
976    TEST_CHK(!strcmp(err_msg, "unknown error"));
977
978    memleak_end();
979    TEST_RESULT("error to string message test");
980}
981
982void seq_tree_exception_test()
983{
984    TEST_INIT();
985
986    memleak_start();
987
988    int i, r;
989    int n = 10;
990    fdb_file_handle *dbfile;
991    fdb_kvs_handle *db;
992    fdb_doc **doc = alca(fdb_doc*, n);
993    fdb_doc *rdoc = NULL;
994    fdb_status status;
995    fdb_iterator *it;
996
997    char keybuf[256], metabuf[256], bodybuf[256];
998
999    // remove previous dummy files
1000    r = system(SHELL_DEL" dummy* > errorlog.txt");
1001    (void)r;
1002
1003    fdb_config fconfig = fdb_get_default_config();
1004    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1005    fconfig.seqtree_opt = FDB_SEQTREE_NOT_USE;
1006
1007    // open db
1008    fdb_open(&dbfile, "./dummy1", &fconfig);
1009    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1010    status = fdb_set_log_callback(db, logCallbackFunc,
1011                                  (void *) "seq_tree_exception_test");
1012    TEST_CHK(status == FDB_RESULT_SUCCESS);
1013
1014    // insert documents
1015    for (i=0;i<n;++i){
1016        sprintf(keybuf, "key%d", i);
1017        sprintf(metabuf, "meta%d", i);
1018        sprintf(bodybuf, "body%d", i);
1019        fdb_doc_create(&doc[i],
1020                       (void *)keybuf,  strlen(keybuf),
1021                       (void *)metabuf, strlen(metabuf),
1022                       (void *)bodybuf, strlen(bodybuf));
1023        fdb_set(db, doc[i]);
1024    }
1025
1026    // commit
1027    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1028
1029    status = fdb_compact(dbfile, NULL);
1030    TEST_CHK(status == FDB_RESULT_SUCCESS);
1031
1032    // close the db
1033    fdb_kvs_close(db);
1034    fdb_close(dbfile);
1035
1036    // reopen with seq tree option
1037    fconfig.seqtree_opt = FDB_SEQTREE_USE;
1038    status = fdb_open(&dbfile, "./dummy1", &fconfig);
1039    // must succeed
1040    TEST_CHK(status == FDB_RESULT_SUCCESS);
1041    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
1042    TEST_CHK(status == FDB_RESULT_SUCCESS);
1043    status = fdb_set_log_callback(db, logCallbackFunc,
1044                                  (void *) "seq_tree_exception_test");
1045    TEST_CHK(status == FDB_RESULT_SUCCESS);
1046
1047    // search by seq
1048    fdb_doc_create(&rdoc, NULL, 0, NULL, 0, NULL, 0);
1049    rdoc->seqnum = 1;
1050    status = fdb_get_byseq(db, rdoc);
1051    // must fail
1052    TEST_CHK(status != FDB_RESULT_SUCCESS);
1053
1054    // search meta by seq
1055    status = fdb_get_metaonly_byseq(db, rdoc);
1056    // must fail
1057    TEST_CHK(status != FDB_RESULT_SUCCESS);
1058
1059    // init iterator by seq
1060    status = fdb_iterator_sequence_init(db , &it, 0, 0, FDB_ITR_NONE);
1061    // must fail
1062    TEST_CHK(status != FDB_RESULT_SUCCESS);
1063
1064    // close db file
1065    fdb_kvs_close(db);
1066    fdb_close(dbfile);
1067
1068    // free all documents
1069    free(rdoc);
1070    for (i=0;i<n;++i){
1071        fdb_doc_free(doc[i]);
1072    }
1073
1074    // remove previous dummy files
1075    r = system(SHELL_DEL" dummy* > errorlog.txt");
1076    (void)r;
1077
1078    // open db
1079    fconfig.seqtree_opt = FDB_SEQTREE_USE;
1080    fdb_open(&dbfile, "./dummy1", &fconfig);
1081    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1082    status = fdb_set_log_callback(db, logCallbackFunc,
1083                                  (void *) "seq_tree_exception_test");
1084    TEST_CHK(status == FDB_RESULT_SUCCESS);
1085
1086    // insert documents
1087    for (i=0;i<n;++i){
1088        sprintf(keybuf, "key%d", i);
1089        sprintf(metabuf, "meta%d", i);
1090        sprintf(bodybuf, "body%d", i);
1091        fdb_doc_create(&doc[i],
1092                       (void *)keybuf,  strlen(keybuf),
1093                       (void *)metabuf, strlen(metabuf),
1094                       (void *)bodybuf, strlen(bodybuf));
1095        fdb_set(db, doc[i]);
1096    }
1097
1098    // commit
1099    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1100
1101    // close the db
1102    fdb_kvs_close(db);
1103    fdb_close(dbfile);
1104
1105    // reopen with an option disabling seq tree
1106    fconfig.seqtree_opt = FDB_SEQTREE_NOT_USE;
1107    status = fdb_open(&dbfile, "./dummy1", &fconfig);
1108    // must succeed
1109    TEST_CHK(status == FDB_RESULT_SUCCESS);
1110    status = fdb_close(dbfile);
1111    TEST_CHK(status == FDB_RESULT_SUCCESS);
1112
1113    // free all documents
1114    for (i=0;i<n;++i){
1115        fdb_doc_free(doc[i]);
1116    }
1117
1118    // free all resources
1119    fdb_shutdown();
1120
1121    memleak_end();
1122
1123    TEST_RESULT("sequence tree exception test");
1124}
1125
1126void wal_commit_test()
1127{
1128    TEST_INIT();
1129
1130    memleak_start();
1131
1132    int i, r;
1133    int n = 10;
1134    fdb_file_handle *dbfile;
1135    fdb_kvs_handle *db;
1136    fdb_doc **doc = alca(fdb_doc*, n);
1137    fdb_doc *rdoc = NULL;
1138    fdb_status status;
1139
1140    char keybuf[256], metabuf[256], bodybuf[256];
1141
1142    // remove previous dummy files
1143    r = system(SHELL_DEL" dummy* > errorlog.txt");
1144    (void)r;
1145
1146    fdb_config fconfig = fdb_get_default_config();
1147    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1148    fconfig.buffercache_size = 0;
1149    fconfig.wal_threshold = 1024;
1150    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1151    fconfig.compaction_threshold = 0;
1152
1153    // open db
1154    fdb_open(&dbfile, "./dummy1", &fconfig);
1155    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1156    status = fdb_set_log_callback(db, logCallbackFunc, (void *) "wal_commit_test");
1157    TEST_CHK(status == FDB_RESULT_SUCCESS);
1158
1159    // insert half documents
1160    for (i=0;i<n/2;++i){
1161        sprintf(keybuf, "key%d", i);
1162        sprintf(metabuf, "meta%d", i);
1163        sprintf(bodybuf, "body%d", i);
1164        fdb_doc_create(&doc[i], (void *)keybuf, strlen(keybuf),
1165            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1166        fdb_set(db, doc[i]);
1167    }
1168
1169    // commit
1170    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1171
1172    // insert the other half documents
1173    for (i=n/2;i<n;++i){
1174        sprintf(keybuf, "key%d", i);
1175        sprintf(metabuf, "meta%d", i);
1176        sprintf(bodybuf, "body%d", i);
1177        fdb_doc_create(&doc[i], (void *)keybuf, strlen(keybuf),
1178            (void *)metabuf, strlen(metabuf), (void *)bodybuf, strlen(bodybuf));
1179        fdb_set(db, doc[i]);
1180    }
1181
1182    // close the db
1183    fdb_kvs_close(db);
1184    fdb_close(dbfile);
1185
1186    // reopen
1187    fdb_open(&dbfile, "./dummy1", &fconfig);
1188    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1189    status = fdb_set_log_callback(db, logCallbackFunc, (void *) "wal_commit_test");
1190    TEST_CHK(status == FDB_RESULT_SUCCESS);
1191
1192    // retrieve documents
1193    for (i=0;i<n;++i){
1194        // search by key
1195        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
1196        status = fdb_get(db, rdoc);
1197
1198        if (i < n/2) {
1199            // committed documents
1200            TEST_CHK(status == FDB_RESULT_SUCCESS);
1201            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
1202            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
1203        } else {
1204            // not committed document
1205            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
1206        }
1207
1208        // free result document
1209        fdb_doc_free(rdoc);
1210        rdoc = NULL;
1211    }
1212
1213    // free all documents
1214    for (i=0;i<n;++i){
1215        fdb_doc_free(doc[i]);
1216    }
1217
1218    // close db file
1219    fdb_kvs_close(db);
1220    fdb_close(dbfile);
1221
1222    // free all resources
1223    fdb_shutdown();
1224
1225    memleak_end();
1226
1227    TEST_RESULT("WAL commit test");
1228}
1229
1230void db_close_and_remove()
1231{
1232
1233    TEST_INIT();
1234    memleak_start();
1235
1236    int i, r;
1237    int n = 10;
1238    fdb_file_handle *dbfile;
1239    fdb_kvs_handle *db;
1240    fdb_doc **doc = alca(fdb_doc *, n);
1241    fdb_status status;
1242    fdb_config fconfig;
1243    fdb_kvs_config kvs_config;
1244    char keybuf[256], metabuf[256], bodybuf[256];
1245
1246    r = system(SHELL_DEL " dummy* > errorlog.txt");
1247    (void)r;
1248
1249    // open dbfile
1250    fconfig = fdb_get_default_config();
1251    kvs_config = fdb_get_default_kvs_config();
1252    fconfig.cleanup_cache_onclose = false;
1253    fdb_open(&dbfile, "./dummy1", &fconfig);
1254    fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
1255
1256    // write to db
1257    for (i=0;i<n;++i){
1258        sprintf(keybuf, "key%d", i);
1259        sprintf(metabuf, "meta%d", i);
1260        sprintf(bodybuf, "body%d", i);
1261        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1262            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1263        fdb_set(db, doc[i]);
1264        fdb_doc_free(doc[i]);
1265    }
1266    status = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1267    TEST_CHK(status == FDB_RESULT_SUCCESS);
1268
1269    // close
1270    fdb_kvs_close(db);
1271    fdb_close(dbfile);
1272
1273    // remove dbfile
1274    r = system(SHELL_DEL " dummy* > errorlog.txt");
1275    (void)r;
1276
1277    // re-open read-only
1278    fconfig.flags = FDB_OPEN_FLAG_RDONLY;
1279    status = fdb_open(&dbfile, "./dummy1", &fconfig);
1280    TEST_CHK(status == FDB_RESULT_NO_SUCH_FILE);
1281
1282    fdb_shutdown();
1283    memleak_end();
1284    TEST_RESULT("db close and remove");
1285}
1286
1287void db_drop_test()
1288{
1289    TEST_INIT();
1290
1291    memleak_start();
1292
1293    int i, r;
1294    int n = 3;
1295    fdb_file_handle *dbfile;
1296    fdb_kvs_handle *db;
1297    fdb_doc **doc = alca(fdb_doc *, n);
1298    fdb_doc *rdoc = NULL;
1299    fdb_status status;
1300
1301    char keybuf[256], metabuf[256], bodybuf[256];
1302
1303    // remove previous dummy files
1304    r = system(SHELL_DEL " dummy* > errorlog.txt");
1305    (void)r;
1306
1307    fdb_config fconfig = fdb_get_default_config();
1308    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1309    fconfig.buffercache_size = 16777216;
1310    fconfig.wal_threshold = 1024;
1311    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1312    fconfig.compaction_threshold = 0;
1313
1314    // open db
1315    fdb_open(&dbfile, "./dummy1", &fconfig);
1316    fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
1317    status = fdb_set_log_callback(db, logCallbackFunc,
1318                                  (void *) "db_drop_test");
1319    TEST_CHK(status == FDB_RESULT_SUCCESS);
1320
1321    // insert first two documents
1322    for (i=0;i<2;++i){
1323        sprintf(keybuf, "key%d", i);
1324        sprintf(metabuf, "meta%d", i);
1325        sprintf(bodybuf, "body%d", i);
1326        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1327            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1328        fdb_set(db, doc[i]);
1329    }
1330
1331    // commit
1332    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1333    fdb_kvs_close(db);
1334    fdb_close(dbfile);
1335
1336    // Remove the database file manually.
1337    r = system(SHELL_DEL " dummy1 > errorlog.txt");
1338    (void)r;
1339
1340    // Open the empty db with the same name.
1341    fdb_open(&dbfile, "./dummy1", &fconfig);
1342    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1343    status = fdb_set_log_callback(db, logCallbackFunc,
1344                                  (void *) "db_drop_test");
1345    TEST_CHK(status == FDB_RESULT_SUCCESS);
1346
1347    // now insert a new doc.
1348    sprintf(keybuf, "key%d", 0);
1349    sprintf(metabuf, "meta%d", 0);
1350    sprintf(bodybuf, "body%d", 0);
1351    fdb_doc_free(doc[0]);
1352    fdb_doc_create(&doc[0], (void*)keybuf, strlen(keybuf),
1353        (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1354    fdb_set(db, doc[0]);
1355
1356    // commit
1357    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1358
1359    // search by key
1360    fdb_doc_create(&rdoc, doc[0]->key, doc[0]->keylen, NULL, 0, NULL, 0);
1361    status = fdb_get(db, rdoc);
1362    TEST_CHK(status == FDB_RESULT_SUCCESS);
1363    // Make sure that a doc seqnum starts with one.
1364    TEST_CHK(rdoc->seqnum == 1);
1365
1366    fdb_kvs_close(db);
1367    fdb_close(dbfile);
1368
1369    // free all documents
1370    fdb_doc_free(rdoc);
1371    rdoc = NULL;
1372    for (i=0;i<2;++i){
1373        fdb_doc_free(doc[i]);
1374    }
1375
1376    // free all resources
1377    fdb_shutdown();
1378
1379    memleak_end();
1380
1381    TEST_RESULT("Database drop test");
1382}
1383
1384void db_destroy_test()
1385{
1386    TEST_INIT();
1387
1388    memleak_start();
1389
1390    int i, r;
1391    int n = 30;
1392    fdb_file_handle *dbfile, *dbfile2;
1393    fdb_kvs_handle *db, *db2;
1394    fdb_doc **doc = alca(fdb_doc *, n);
1395    fdb_doc *rdoc = NULL;
1396    fdb_status status;
1397    fdb_config fconfig;
1398    fdb_kvs_config kvs_config;
1399
1400    char keybuf[256], metabuf[256], bodybuf[256];
1401
1402    // remove previous dummy files
1403    r = system(SHELL_DEL " dummy* > errorlog.txt");
1404    (void)r;
1405
1406    fconfig = fdb_get_default_config();
1407    kvs_config = fdb_get_default_kvs_config();
1408    fconfig.buffercache_size = 16777216;
1409    fconfig.wal_threshold = 1024;
1410    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1411    fconfig.compaction_threshold = 0;
1412
1413    // open db
1414    fdb_open(&dbfile, "./dummy1", &fconfig);
1415    fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
1416    status = fdb_set_log_callback(db, logCallbackFunc,
1417                                  (void *) "db_destroy_test");
1418    TEST_CHK(status == FDB_RESULT_SUCCESS);
1419
1420    // insert 30 documents
1421    for (i=0;i<n;++i){
1422        sprintf(keybuf, "key%d", i);
1423        sprintf(metabuf, "meta%d", i);
1424        sprintf(bodybuf, "body%d", i);
1425        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1426            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1427        fdb_set(db, doc[i]);
1428    }
1429
1430    // commit
1431    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1432
1433    // Open the empty db with the same name.
1434    fdb_open(&dbfile2, "./dummy2", &fconfig);
1435    fdb_kvs_open(dbfile2, &db2, NULL, &kvs_config);
1436    status = fdb_set_log_callback(db2, logCallbackFunc,
1437                                  (void *) "db_destroy_test");
1438    TEST_CHK(status == FDB_RESULT_SUCCESS);
1439    // insert 30 documents
1440    for (i=0;i<n;++i){
1441        fdb_set(db2, doc[i]);
1442    }
1443
1444    // commit
1445    fdb_commit(dbfile2, FDB_COMMIT_NORMAL);
1446
1447    // Only close db not db2 and try to destroy
1448    fdb_close(dbfile);
1449
1450    status = fdb_destroy("./dummy2", &fconfig);
1451    TEST_CHK(status == FDB_RESULT_FILE_IS_BUSY);
1452
1453    //Now close the open db file
1454    fdb_close(dbfile2);
1455
1456    status = fdb_destroy("./dummy1", &fconfig);
1457    TEST_CHK(status == FDB_RESULT_SUCCESS);
1458
1459    // Open the same db with the same names.
1460    fdb_open(&dbfile, "./dummy1", &fconfig);
1461    fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
1462    status = fdb_set_log_callback(db, logCallbackFunc,
1463                                  (void *) "db_destroy_test");
1464    TEST_CHK(status == FDB_RESULT_SUCCESS);
1465
1466    // search by key
1467    fdb_doc_create(&rdoc, doc[0]->key, doc[0]->keylen, NULL, 0, NULL, 0);
1468    status = fdb_get(db, rdoc);
1469    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
1470    fdb_close(dbfile);
1471
1472    // free all documents
1473    fdb_doc_free(rdoc);
1474    rdoc = NULL;
1475    for (i=0;i<n;++i){
1476        fdb_doc_free(doc[i]);
1477    }
1478
1479    // free all resources
1480    fdb_shutdown();
1481
1482    memleak_end();
1483
1484    TEST_RESULT("Database destroy test");
1485}
1486
1487// Test for MB-16348
1488void db_destroy_test_full_path()
1489{
1490    TEST_INIT();
1491
1492    memleak_start();
1493    randomize();
1494
1495    int r;
1496    fdb_file_handle *dbfile;
1497    fdb_config config;
1498    fdb_status s;
1499    char path[256];
1500    char cmd[256];
1501
1502    sprintf(path, "/tmp/fdb_destroy_test_%d", random(10000));
1503
1504    sprintf(cmd, "rm -rf %s*", path);
1505    r = system(cmd); (void)r;
1506
1507    config = fdb_get_default_config();
1508    config.compaction_mode = FDB_COMPACTION_AUTO;
1509
1510    fdb_open(&dbfile, path, &config);
1511    fdb_close(dbfile);
1512
1513    s = fdb_destroy(path, &config);
1514    TEST_CHK(s == FDB_RESULT_SUCCESS);
1515
1516    fdb_shutdown();
1517
1518    memleak_end();
1519
1520    TEST_RESULT("Database destroy (full path) test");
1521}
1522
1523void operational_stats_test(bool multi_kv)
1524{
1525    TEST_INIT();
1526
1527    memleak_start();
1528
1529    int i, r;
1530    int n = 10;
1531    int num_kv = 4;
1532    fdb_file_handle *dbfile;
1533    fdb_kvs_handle **db = alca(fdb_kvs_handle*, num_kv);
1534    fdb_doc **doc = alca(fdb_doc*, n);
1535    fdb_doc real_doc;
1536    fdb_doc *rdoc = &real_doc;
1537    fdb_status status;
1538    fdb_iterator *iterator;
1539    fdb_kvs_ops_info info, rinfo;
1540
1541    char keybuf[256], bodybuf[256];
1542    memset(&info, 0, sizeof(fdb_kvs_ops_info));
1543    memset(&real_doc, 0, sizeof(fdb_doc));
1544    real_doc.key = &keybuf;
1545    real_doc.body = &bodybuf;
1546
1547    // remove previous dummy files
1548    r = system(SHELL_DEL" dummy* > errorlog.txt");
1549    (void)r;
1550
1551    fdb_config fconfig = fdb_get_default_config();
1552    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1553
1554    fconfig.buffercache_size = 0;
1555    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
1556    fconfig.wal_threshold = 1024;
1557    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1558    fconfig.compaction_threshold = 0;
1559    fconfig.multi_kv_instances = multi_kv;
1560    r = 0;
1561
1562    fdb_open(&dbfile, "./dummy1", &fconfig);
1563    if (multi_kv) {
1564        num_kv = 4;
1565        for (r = num_kv - 1; r >= 0; --r) {
1566            char tmp[16];
1567            sprintf(tmp, "kv%d", r);
1568            status = fdb_kvs_open(dbfile, &db[r], tmp, &kvs_config);
1569            TEST_CHK(status == FDB_RESULT_SUCCESS);
1570            status = fdb_set_log_callback(db[r], logCallbackFunc,
1571                                          (void *) "operational_stats_test");
1572            TEST_CHK(status == FDB_RESULT_SUCCESS);
1573        }
1574    } else {
1575        num_kv = 1;
1576        status = fdb_kvs_open_default(dbfile, &db[r], &kvs_config);
1577        TEST_CHK(status == FDB_RESULT_SUCCESS);
1578        status = fdb_set_log_callback(db[r], logCallbackFunc,
1579                (void *) "operational_stats_test");
1580        TEST_CHK(status == FDB_RESULT_SUCCESS);
1581    }
1582
1583    for (i = 0; i < n; ++i){
1584        sprintf(keybuf, "key%d", i);
1585        sprintf(bodybuf, "body%d", i);
1586        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf) + 1, NULL, 0,
1587            (void*)bodybuf, strlen(bodybuf)+1);
1588        for (r = num_kv - 1; r >= 0; --r) {
1589            status = fdb_set(db[r], doc[i]);
1590            TEST_CHK(status == FDB_RESULT_SUCCESS);
1591            status = fdb_get_kvs_ops_info(db[r], &rinfo);
1592            TEST_CHK(status == FDB_RESULT_SUCCESS);
1593            info.num_sets = i + 1;
1594            TEST_CMP(&rinfo, &info, sizeof(fdb_kvs_ops_info));
1595        }
1596    }
1597
1598    for (r = num_kv - 1; r >= 0; --r) {
1599        // range scan (before flushing WAL)
1600        fdb_iterator_init(db[r], &iterator, NULL, 0, NULL, 0, 0x0);
1601        i = 0;
1602        do {
1603            status = fdb_iterator_get(iterator, &rdoc);
1604            TEST_CHK(status == FDB_RESULT_SUCCESS);
1605            TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
1606            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
1607            status = fdb_get_kvs_ops_info(db[r], &rinfo);
1608            TEST_CHK(status == FDB_RESULT_SUCCESS);
1609            ++info.num_iterator_gets;
1610            ++info.num_iterator_moves;
1611            TEST_CMP(&rinfo, &info, sizeof(fdb_kvs_ops_info));
1612            ++i;
1613        } while(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
1614        ++info.num_iterator_moves; // account for the last move that failed
1615        fdb_iterator_close(iterator);
1616
1617        fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1618        ++info.num_commits;
1619
1620        status = fdb_get_kvs_ops_info(db[r], &rinfo);
1621        TEST_CHK(status == FDB_RESULT_SUCCESS);
1622        TEST_CMP(&rinfo, &info, sizeof(fdb_kvs_ops_info));
1623
1624        if (r) {
1625            info.num_iterator_gets = 0;
1626            info.num_iterator_moves = 0;
1627        }
1628    }
1629
1630    ++info.num_compacts;
1631    // do compaction
1632    fdb_compact(dbfile, (char *) "./dummy2");
1633
1634    status = fdb_get_kvs_ops_info(db[0], &rinfo);
1635    TEST_CHK(status == FDB_RESULT_SUCCESS);
1636    TEST_CMP(&rinfo, &info, sizeof(fdb_kvs_ops_info));
1637
1638    for (i = 0; i < n; ++i){
1639        sprintf(keybuf, "key%d", i);
1640        for (r = num_kv - 1; r >= 0; --r) {
1641            if (i % 2 == 0) {
1642                if (i % 4 == 0) {
1643                    status = fdb_get_metaonly(db[r], rdoc);
1644                } else {
1645                    rdoc->seqnum = i + 1;
1646                    status = fdb_get_byseq(db[r], rdoc);
1647                }
1648            } else {
1649                status = fdb_get(db[r], rdoc);
1650            }
1651            TEST_CHK(status == FDB_RESULT_SUCCESS);
1652            status = fdb_get_kvs_ops_info(db[r], &rinfo);
1653            TEST_CHK(status == FDB_RESULT_SUCCESS);
1654            info.num_gets = i + 1;
1655            TEST_CMP(&rinfo, &info, sizeof(fdb_kvs_ops_info));
1656        }
1657    }
1658    // also get latency stats..
1659    for (int i = 0; i < FDB_LATENCY_NUM_STATS; ++i) {
1660        fdb_latency_stat stat;
1661        memset(&stat, 0, sizeof(fdb_latency_stat));
1662        status = fdb_get_latency_stats(dbfile, &stat, i);
1663        TEST_CHK(status == FDB_RESULT_SUCCESS);
1664        fprintf(stderr, "%d:\t%u\t%u\t%u\t%" _F64 "\n", i,
1665                stat.lat_max, stat.lat_avg, stat.lat_max, stat.lat_count);
1666    }
1667
1668    fdb_close(dbfile);
1669
1670    // free all documents
1671    for (i=0;i<n;++i){
1672        fdb_doc_free(doc[i]);
1673    }
1674
1675    // free all resources
1676    fdb_shutdown();
1677
1678    memleak_end();
1679
1680    sprintf(bodybuf,"Operational stats test %s", multi_kv ?
1681            "multiple kv instances" : "single kv instance");
1682    TEST_RESULT(bodybuf);
1683}
1684
1685struct work_thread_args{
1686    int tid;
1687    size_t nthreads;
1688    size_t ndocs;
1689    size_t writer;
1690    fdb_doc **doc;
1691    size_t time_sec;
1692    size_t nbatch;
1693    size_t compact_term;
1694    int *n_opened;
1695    int *filename_count;
1696    spin_t *filename_count_lock;
1697    size_t nops;
1698    fdb_config *config;
1699    fdb_kvs_config *kvs_config;
1700};
1701
1702//#define FILENAME "./hdd/dummy"
1703#define FILENAME "dummy"
1704
1705#define KSIZE (100)
1706#define VSIZE (100)
1707#define IDX_DIGIT (7)
1708#define IDX_DIGIT_STR "7"
1709
1710void *_worker_thread(void *voidargs)
1711{
1712    TEST_INIT();
1713
1714    struct work_thread_args *args = (struct work_thread_args *)voidargs;
1715    int i, c, commit_count, filename_count;
1716    struct timeval ts_begin, ts_cur, ts_gap;
1717    fdb_file_handle *dbfile;
1718    fdb_kvs_handle *db;
1719    fdb_status status;
1720    fdb_doc *rdoc = NULL;
1721    char temp[1024];
1722
1723    char cnt_str[IDX_DIGIT+1];
1724    int cnt_int;
1725
1726    filename_count = *args->filename_count;
1727    sprintf(temp, FILENAME"%d", filename_count);
1728    fdb_open(&dbfile, temp, args->config);
1729    fdb_kvs_open_default(dbfile, &db, args->kvs_config);
1730    status = fdb_set_log_callback(db, logCallbackFunc,
1731                                  (void *) "worker_thread");
1732    TEST_CHK(status == FDB_RESULT_SUCCESS);
1733
1734    // wait until all other threads open the DB file.
1735    // (to avoid performing compaction before opening the file)
1736    spin_lock(args->filename_count_lock);
1737    *args->n_opened += 1;
1738    spin_unlock(args->filename_count_lock);
1739    do {
1740        spin_lock(args->filename_count_lock);
1741        if ((size_t)(*args->n_opened) == args->nthreads) {
1742            // all threads open the DB file
1743            spin_unlock(args->filename_count_lock);
1744            break;
1745        }
1746        spin_unlock(args->filename_count_lock);
1747        // sleep 1 sec
1748        sleep(1);
1749    } while (1);
1750
1751    gettimeofday(&ts_begin, NULL);
1752
1753    c = cnt_int = commit_count = 0;
1754    cnt_str[IDX_DIGIT] = 0;
1755
1756    while (1){
1757        i = rand() % args->ndocs;
1758        fdb_doc_create(&rdoc, args->doc[i]->key, args->doc[i]->keylen, NULL, 0, NULL, 0);
1759        status = fdb_get(db, rdoc);
1760
1761        TEST_CHK(status == FDB_RESULT_SUCCESS);
1762        TEST_CMP(rdoc->body, args->doc[i]->body, (IDX_DIGIT+1));
1763
1764        if (args->writer) {
1765            // if writer,
1766            // copy and parse the counter in body
1767            memcpy(cnt_str, (uint8_t *)rdoc->body + (IDX_DIGIT+1), IDX_DIGIT);
1768            cnt_int = atoi(cnt_str);
1769
1770            // increase and rephrase
1771            sprintf(cnt_str, "%0 " IDX_DIGIT_STR "d", ++cnt_int);
1772            memcpy((uint8_t *)rdoc->body + (IDX_DIGIT+1), cnt_str, IDX_DIGIT);
1773
1774            // update and commit
1775            status = fdb_set(db, rdoc);
1776            TEST_CHK(status == FDB_RESULT_SUCCESS);
1777
1778            if (args->nbatch > 0) {
1779                if (c % args->nbatch == 0) {
1780                    // commit for every NBATCH
1781                    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1782                    commit_count++;
1783                    fdb_file_info info;
1784                    fdb_get_file_info(dbfile, &info);
1785                    if (args->compact_term == (size_t)commit_count &&
1786                        args->compact_term > 0 &&
1787                        info.new_filename == NULL &&
1788                        args->tid == 0) {
1789                        // do compaction for every COMPACT_TERM batch
1790                        spin_lock(args->filename_count_lock);
1791                        *args->filename_count += 1;
1792                        filename_count = *args->filename_count;
1793                        spin_unlock(args->filename_count_lock);
1794
1795                        sprintf(temp, FILENAME"%d", filename_count);
1796
1797                        status = fdb_compact(dbfile, temp);
1798                        if (status != FDB_RESULT_SUCCESS) {
1799                            spin_lock(args->filename_count_lock);
1800                            *args->filename_count -= 1;
1801                            spin_unlock(args->filename_count_lock);
1802                        }
1803
1804                        commit_count = 0;
1805                    }
1806                }
1807            }
1808        }
1809        fdb_doc_free(rdoc);
1810        rdoc = NULL;
1811        c++;
1812
1813        gettimeofday(&ts_cur, NULL);
1814        ts_gap = _utime_gap(ts_begin, ts_cur);
1815        if ((size_t)ts_gap.tv_sec >= args->time_sec) break;
1816    }
1817
1818    DBG("Thread #%d (%s) %d ops / %d seconds\n",
1819        args->tid, (args->writer)?("writer"):("reader"), c, (int)args->time_sec);
1820    args->nops = c;
1821
1822    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1823
1824    fdb_kvs_close(db);
1825    fdb_close(dbfile);
1826    thread_exit(0);
1827    return NULL;
1828}
1829
1830void multi_thread_test(
1831    size_t ndocs, size_t wal_threshold, size_t time_sec,
1832    size_t nbatch, size_t compact_term, size_t nwriters, size_t nreaders)
1833{
1834    TEST_INIT();
1835
1836    size_t nwrites, nreads;
1837    int i, r;
1838    int n = nwriters + nreaders;;
1839    thread_t *tid = alca(thread_t, n);
1840    void **thread_ret = alca(void *, n);
1841    struct work_thread_args *args = alca(struct work_thread_args, n);
1842    struct timeval ts_begin, ts_cur, ts_gap;
1843    fdb_file_handle *dbfile;
1844    fdb_kvs_handle *db;
1845    fdb_doc **doc = alca(fdb_doc*, ndocs);
1846    fdb_status status;
1847    fdb_kvs_info kvs_info;
1848
1849    int filename_count = 1;
1850    int n_opened = 0;
1851    spin_t filename_count_lock;
1852    spin_init(&filename_count_lock);
1853
1854    char keybuf[1024], metabuf[1024], bodybuf[1024], temp[1024];
1855
1856    // remove previous dummy files
1857    r = system(SHELL_DEL" " FILENAME "* > errorlog.txt");
1858    (void)r;
1859
1860    fdb_config fconfig = fdb_get_default_config();
1861    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1862    fconfig.buffercache_size = 16777216;
1863    fconfig.wal_threshold = 1024;
1864    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1865    fconfig.compaction_threshold = 0;
1866
1867    memleak_start();
1868
1869    // initial population ===
1870    DBG("Initialize..\n");
1871
1872    // open db
1873    sprintf(temp, FILENAME"%d", filename_count);
1874    fdb_open(&dbfile, temp, &fconfig);
1875    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1876    status = fdb_set_log_callback(db, logCallbackFunc,
1877                                  (void *) "multi_thread_test");
1878    TEST_CHK(status == FDB_RESULT_SUCCESS);
1879
1880    gettimeofday(&ts_begin, NULL);
1881
1882    // insert documents
1883    for (i = 0; (size_t)i < ndocs; ++i){
1884        _set_random_string_smallabt(temp, KSIZE - (IDX_DIGIT+1));
1885        sprintf(keybuf, "k%0" IDX_DIGIT_STR "d%s", i, temp);
1886
1887        sprintf(metabuf, "m%0" IDX_DIGIT_STR "d", i);
1888
1889        _set_random_string_smallabt(temp, VSIZE-(IDX_DIGIT*2+1));
1890        sprintf(bodybuf, "b%0" IDX_DIGIT_STR "d%0" IDX_DIGIT_STR "d%s", i, 0, temp);
1891
1892        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1893            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1894        fdb_set(db, doc[i]);
1895    }
1896
1897    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1898
1899    gettimeofday(&ts_cur, NULL);
1900    ts_gap = _utime_gap(ts_begin, ts_cur);
1901    //DBG("%d.%09d seconds elapsed\n", (int)ts_gap.tv_sec, (int)ts_gap.tv_nsec);
1902
1903    fdb_kvs_close(db);
1904    fdb_close(dbfile);
1905    // end of population ===
1906
1907    // drop OS's page cache
1908    //r = system("free && sync && echo 3 > /proc/sys/vm/drop_caches && free");
1909
1910    // create workers
1911    for (i=0;i<n;++i){
1912        args[i].tid = i;
1913        args[i].nthreads = n;
1914        args[i].writer = (((size_t)i<nwriters)?(1):(0));
1915        args[i].ndocs = ndocs;
1916        args[i].doc = doc;
1917        args[i].time_sec = time_sec;
1918        args[i].nbatch = nbatch;
1919        args[i].compact_term = compact_term;
1920        args[i].n_opened = &n_opened;
1921        args[i].filename_count = &filename_count;
1922        args[i].filename_count_lock = &filename_count_lock;
1923        args[i].config = &fconfig;
1924        args[i].kvs_config = &kvs_config;
1925        thread_create(&tid[i], _worker_thread, &args[i]);
1926    }
1927
1928    printf("wait for %d seconds..\n", (int)time_sec);
1929
1930    // wait for thread termination
1931    for (i=0;i<n;++i){
1932        thread_join(tid[i], &thread_ret[i]);
1933    }
1934
1935    // free all documents
1936    for (i=0;(size_t)i<ndocs;++i){
1937        fdb_doc_free(doc[i]);
1938    }
1939
1940    nwrites = nreads = 0;
1941    for (i=0;i<n;++i){
1942        if (args[i].writer) {
1943            nwrites += args[i].nops;
1944        } else {
1945            nreads += args[i].nops;
1946        }
1947    }
1948    printf("read: %.1f ops/sec\n", (double)nreads/time_sec);
1949    printf("write: %.1f ops/sec\n", (double)nwrites/time_sec);
1950
1951    // check sequence number
1952    sprintf(temp, FILENAME"%d", filename_count);
1953    fdb_open(&dbfile, temp, &fconfig);
1954    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1955    fdb_get_kvs_info(db, &kvs_info);
1956    TEST_CHK(kvs_info.last_seqnum == ndocs+nwrites);
1957    fdb_kvs_close(db);
1958    fdb_close(dbfile);
1959
1960    // shutdown
1961    fdb_shutdown();
1962
1963    memleak_end();
1964
1965    TEST_RESULT("multi thread test");
1966}
1967
1968void *multi_thread_client_shutdown(void *args)
1969{
1970
1971    TEST_INIT();
1972
1973    int i, r;
1974    int nclients;
1975    fdb_file_handle *tdbfile;
1976    fdb_status status;
1977    fdb_config fconfig;
1978    fdb_kvs_config kvs_config;
1979    thread_t *tid;
1980    void **thread_ret;
1981
1982    if (args == NULL)
1983    { // parent
1984        memleak_start();
1985
1986        r = system(SHELL_DEL" dummy* > errorlog.txt");
1987        (void)r;
1988        nclients = 2;
1989        tid = alca(thread_t, nclients);
1990        thread_ret = alca(void *, nclients);
1991        for (i=0;i<nclients;++i){
1992            thread_create(&tid[i], multi_thread_client_shutdown, (void *)&i);
1993        }
1994        for (i=0;i<nclients;++i){
1995            thread_join(tid[i], &thread_ret[i]);
1996        }
1997
1998        memleak_end();
1999        TEST_RESULT("multi thread client shutdown");
2000        return NULL;
2001    }
2002
2003    // threads enter here //
2004
2005    fconfig = fdb_get_default_config();
2006    kvs_config = fdb_get_default_kvs_config();
2007    fconfig.wal_threshold = 1024;
2008    fconfig.compaction_threshold = 0;
2009
2010    // open/close db
2011    status = fdb_open(&tdbfile, "./dummy1", &fconfig);
2012    TEST_CHK(status == FDB_RESULT_SUCCESS);
2013    TEST_CHK(fdb_close(tdbfile) == FDB_RESULT_SUCCESS);
2014
2015    // shutdown
2016    fdb_shutdown();
2017    thread_exit(0);
2018    return NULL;
2019}
2020
2021void *multi_thread_kvs_client(void *args)
2022{
2023
2024    TEST_INIT();
2025
2026    int i, j, r;
2027    int n = 50;
2028    int nclients = 20;
2029    int *tid_args = alca(int, nclients);
2030    char dbstr[256];
2031    char keybuf[256], metabuf[256], bodybuf[256];
2032    fdb_file_handle *dbfile;
2033    fdb_kvs_handle *tdb;
2034    fdb_kvs_handle **db = alca(fdb_kvs_handle*, nclients);
2035    fdb_doc **doc = alca(fdb_doc*, n);
2036    fdb_doc *rdoc = NULL;
2037    fdb_status status;
2038    fdb_config fconfig;
2039    fdb_kvs_config kvs_config;
2040    fdb_seqnum_t seqnum;
2041    thread_t *tid;
2042    void **thread_ret;
2043
2044    if (args == NULL)
2045    { // parent
2046        memleak_start();
2047
2048        r = system(SHELL_DEL" dummy* > errorlog.txt");
2049        (void)r;
2050
2051        // init dbfile
2052        fconfig = fdb_get_default_config();
2053        fconfig.buffercache_size = 0;
2054        fconfig.wal_threshold = 1024;
2055        fconfig.compaction_threshold = 0;
2056
2057        status = fdb_open(&dbfile, "./dummy1", &fconfig);
2058        TEST_CHK(status == FDB_RESULT_SUCCESS);
2059
2060        tid = alca(thread_t, nclients);
2061        thread_ret = alca(void *, nclients);
2062        for (i=0;i<nclients;++i){
2063            sprintf(dbstr, "db%d", i);
2064            kvs_config = fdb_get_default_kvs_config();
2065            status = fdb_kvs_open(dbfile, &db[i], dbstr, &kvs_config);
2066            TEST_CHK(status == FDB_RESULT_SUCCESS);
2067            status = fdb_kvs_close(db[i]);
2068            TEST_CHK(status == FDB_RESULT_SUCCESS);
2069        }
2070        for (i=0;i<nclients;++i){
2071            tid_args[i] = i;
2072            thread_create(&tid[i], multi_thread_kvs_client,
2073                          (void *)&tid_args[i]);
2074        }
2075        for (i=0;i<nclients;++i){
2076            thread_join(tid[i], &thread_ret[i]);
2077        }
2078
2079        status = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2080        TEST_CHK(status == FDB_RESULT_SUCCESS);
2081
2082        // check threads updated kvs
2083        for (i=0; i<nclients; i++){
2084            sprintf(dbstr, "db%d", i);
2085            kvs_config = fdb_get_default_kvs_config();
2086            status = fdb_kvs_open(dbfile, &db[i], dbstr, &kvs_config);
2087            TEST_CHK(status == FDB_RESULT_SUCCESS);
2088
2089            // verify seqnum
2090            status = fdb_get_kvs_seqnum(db[i], &seqnum);
2091            TEST_CHK(status == FDB_RESULT_SUCCESS);
2092            TEST_CHK(seqnum == (fdb_seqnum_t)n);
2093
2094            for (j=0; j<n; j++){
2095                sprintf(keybuf, "key%d", j);
2096                sprintf(metabuf, "meta%d", j);
2097                sprintf(bodybuf, "body%d", j);
2098                fdb_doc_create(&rdoc, keybuf, strlen(keybuf),
2099                                      NULL, 0, NULL, 0);
2100                status = fdb_get(db[i], rdoc);
2101                TEST_CHK(status == FDB_RESULT_SUCCESS);
2102                TEST_CHK(!memcmp(rdoc->key, keybuf, strlen(keybuf)));
2103                TEST_CHK(!memcmp(rdoc->meta, metabuf, rdoc->metalen));
2104                TEST_CHK(!memcmp(rdoc->body, bodybuf, rdoc->bodylen));
2105                fdb_doc_free(rdoc);
2106            }
2107            status = fdb_kvs_close(db[i]);
2108            TEST_CHK(status == FDB_RESULT_SUCCESS);
2109        }
2110
2111        status = fdb_close(dbfile);
2112        TEST_CHK(status == FDB_RESULT_SUCCESS);
2113        fdb_shutdown();
2114        memleak_end();
2115        TEST_RESULT("multi thread kvs client");
2116        return NULL;
2117    }
2118
2119    // threads enter here //
2120
2121    // open fhandle
2122    fconfig = fdb_get_default_config();
2123    status = fdb_open(&dbfile, "./dummy1", &fconfig);
2124    TEST_CHK(status == FDB_RESULT_SUCCESS);
2125
2126    // get kvs ID from args
2127    memcpy(&i, args, sizeof(int));
2128    sprintf(dbstr, "db%d", i);
2129    kvs_config = fdb_get_default_kvs_config();
2130    status = fdb_kvs_open(dbfile, &tdb, dbstr, &kvs_config);
2131    TEST_CHK(status == FDB_RESULT_SUCCESS);
2132
2133    // insert documents
2134    for (i=0;i<n;++i){
2135        sprintf(keybuf, "key%d", i);
2136        sprintf(metabuf, "meta%d", i);
2137        sprintf(bodybuf, "body%d", i);
2138        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2139                                (void*)metabuf, strlen(metabuf),
2140                                (void*)bodybuf, strlen(bodybuf));
2141        status = fdb_set(tdb, doc[i]);
2142        TEST_CHK(status == FDB_RESULT_SUCCESS);
2143        fdb_doc_free(doc[i]);
2144    }
2145    status = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2146    TEST_CHK(status == FDB_RESULT_SUCCESS);
2147
2148    fdb_close(dbfile);
2149    return NULL;
2150}
2151
2152void *multi_thread_fhandle_share(void *args)
2153{
2154    TEST_INIT();
2155    fdb_status status;
2156    int n = 2000;
2157    int i, r;
2158    char tmpbuf[32];
2159    typedef struct {
2160        fdb_file_handle *dbfile;
2161        fdb_kvs_handle *def;
2162        fdb_kvs_handle *main;
2163        fdb_kvs_handle *back;
2164        bool isWriter;
2165        std::atomic<bool> shutdown;
2166    } thread_data_t;
2167
2168    if (args == NULL) { // MAIN THREAD..
2169        int nthreads = 2; // Half of these are reader and half are writers
2170        int nwriters = nthreads / 2;
2171        thread_t *tid = (thread_t *)malloc(nthreads * sizeof(thread_t *));
2172        thread_data_t *tdata = (thread_data_t *) malloc(nthreads
2173                                               * sizeof(thread_data_t));
2174        void **thread_ret = (void **)malloc(nthreads * sizeof (void *));
2175        fdb_kvs_config kvs_config;
2176        fdb_config fconfig;
2177
2178        r = system(SHELL_DEL" func_test* > errorlog.txt");
2179        (void)r;
2180
2181        // Shared File Handle data...
2182        fconfig = fdb_get_default_config();
2183        fconfig.buffercache_size = 0;
2184        fconfig.compaction_threshold = 0;
2185        fconfig.num_compactor_threads = 1;
2186        kvs_config = fdb_get_default_kvs_config();
2187        for (i=0; i < nwriters; ++i) {
2188            // Let Readers share same file handle as writers..
2189            fdb_file_handle *dbfile;
2190            sprintf(tmpbuf, "./func_test_pt.%d", i);
2191            status = fdb_open(&dbfile, tmpbuf, &fconfig);
2192            TEST_CHK(status == FDB_RESULT_SUCCESS);
2193            tdata[i].dbfile = dbfile;
2194            int ridx = i+nwriters; // reader index
2195            tdata[ridx].dbfile = dbfile;
2196            tdata[i].isWriter = true;
2197            // Open separate KVS Handles for Readers..
2198            tdata[ridx].isWriter = false; // Set for readers
2199            status = fdb_kvs_open_default(dbfile, &tdata[ridx].def, &kvs_config);
2200            TEST_CHK(status == FDB_RESULT_SUCCESS);
2201            status = fdb_kvs_open(dbfile, &tdata[ridx].main, "main", &kvs_config);
2202            TEST_CHK(status == FDB_RESULT_SUCCESS);
2203            status = fdb_kvs_open(dbfile, &tdata[ridx].back, "back", &kvs_config);
2204            TEST_CHK(status == FDB_RESULT_SUCCESS);
2205            // Open Separate KVS Handle for Writers..
2206            status = fdb_kvs_open_default(dbfile, &tdata[i].def, &kvs_config);
2207            TEST_CHK(status == FDB_RESULT_SUCCESS);
2208            status = fdb_kvs_open(dbfile, &tdata[i].main, "main", &kvs_config);
2209            TEST_CHK(status == FDB_RESULT_SUCCESS);
2210            status = fdb_kvs_open(dbfile, &tdata[i].back, "back", &kvs_config);
2211            TEST_CHK(status == FDB_RESULT_SUCCESS);
2212        }
2213        printf("Creating %d writers+readers over %d docs..\n", nwriters, n);
2214        for (i=nthreads - 1;i>=0;--i){
2215            tdata[i].shutdown = false;
2216            thread_create(&tid[i], multi_thread_fhandle_share,
2217                          reinterpret_cast<void *>(&tdata[i]));
2218        }
2219        for (i=0; i < nwriters; ++i) { // first wait for writers..
2220            thread_join(tid[i], &thread_ret[i]);
2221            printf("Writer %d done\n", i);
2222            tdata[i+nwriters].shutdown = true; // tell reader to shutdown
2223        }
2224        for (;i<nthreads;++i){ // now wait for readers..
2225            thread_join(tid[i], &thread_ret[i]);
2226        }
2227
2228        for (i=0; i<nwriters;++i) {
2229            status = fdb_close(tdata[i].dbfile);
2230            TEST_CHK(status == FDB_RESULT_SUCCESS);
2231        }
2232
2233        free(tid);
2234        free(tdata);
2235        free(thread_ret);
2236        fdb_shutdown();
2237        TEST_RESULT("multi thread file handle share test");
2238        return NULL;
2239    }
2240    // threads enter here ----
2241    thread_data_t *tdata = reinterpret_cast<thread_data_t *>(args);
2242    if (tdata->isWriter) { // Writer Threads Run this...
2243        for (i=0; i < n; ++i) {
2244            sprintf(tmpbuf, "key%03d", i);
2245            status = fdb_set_kv(tdata->main, &tmpbuf, 7, nullptr, 0);
2246            TEST_CHK(status == FDB_RESULT_SUCCESS);
2247            status = fdb_set_kv(tdata->back, &tmpbuf, 7, nullptr, 0);
2248            TEST_CHK(status == FDB_RESULT_SUCCESS);
2249            status = fdb_set_kv(tdata->def, &tmpbuf, 7, nullptr, 0);
2250            TEST_CHK(status == FDB_RESULT_SUCCESS);
2251            if (n % 100 == 0) {
2252                status = fdb_commit(tdata->dbfile,
2253                                    FDB_COMMIT_MANUAL_WAL_FLUSH);
2254                TEST_CHK(status != FDB_RESULT_HANDLE_BUSY);
2255            }
2256        }
2257        return NULL;
2258    } // else  Reader Threads Run this ...
2259    while (!tdata->shutdown) {
2260        for (i=0; i < n; ++i) {
2261            void *value = nullptr;
2262            size_t valuelen;
2263            sprintf(tmpbuf, "key%03d", i);
2264            status = fdb_get_kv(tdata->main, &tmpbuf, 7, &value, &valuelen);
2265            TEST_CHK(status != FDB_RESULT_HANDLE_BUSY);
2266            status = fdb_get_kv(tdata->back, &tmpbuf, 7, &value, &valuelen);
2267            TEST_CHK(status != FDB_RESULT_HANDLE_BUSY);
2268            status = fdb_get_kv(tdata->def, &tmpbuf, 7, &value, &valuelen);
2269            TEST_CHK(status != FDB_RESULT_HANDLE_BUSY);
2270        }
2271    }
2272
2273    return NULL;
2274}
2275
2276void incomplete_block_test()
2277{
2278    TEST_INIT();
2279
2280    memleak_start();
2281
2282    int i, r;
2283    int n = 2;
2284    fdb_file_handle *dbfile;
2285    fdb_kvs_handle *db;
2286    fdb_doc **doc = alca(fdb_doc*, n);
2287    fdb_doc *rdoc = NULL;
2288    fdb_status status;
2289
2290    char keybuf[256], metabuf[256], bodybuf[256];
2291
2292    // remove previous dummy files
2293    r = system(SHELL_DEL" dummy* > errorlog.txt");
2294    (void)r;
2295
2296    fdb_config fconfig = fdb_get_default_config();
2297    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2298    fconfig.buffercache_size = 0;
2299    fconfig.wal_threshold = 1024;
2300    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2301    fconfig.compaction_threshold = 0;
2302
2303    // open db
2304    fdb_open(&dbfile, "./dummy1", &fconfig);
2305    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2306    status = fdb_set_log_callback(db, logCallbackFunc,
2307                                  (void *) "incomplete_block_test");
2308    TEST_CHK(status == FDB_RESULT_SUCCESS);
2309
2310    // insert documents
2311    for (i=0;i<n;++i){
2312        sprintf(keybuf, "key%d", i);
2313        sprintf(metabuf, "meta%d", i);
2314        sprintf(bodybuf, "body%d", i);
2315        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2316            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2317        fdb_set(db, doc[i]);
2318    }
2319
2320    // retrieve documents
2321    for (i=0;i<n;++i){
2322        // search by key
2323        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
2324        status = fdb_get(db, rdoc);
2325
2326        // updated documents
2327        TEST_CHK(status == FDB_RESULT_SUCCESS);
2328        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2329        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2330
2331        // free result document
2332        fdb_doc_free(rdoc);
2333        rdoc = NULL;
2334    }
2335
2336    // close db file
2337    fdb_kvs_close(db);
2338    fdb_close(dbfile);
2339
2340    // free all documents
2341    for (i=0;i<n;++i){
2342        fdb_doc_free(doc[i]);
2343    }
2344
2345    // free all resources
2346    fdb_shutdown();
2347
2348    memleak_end();
2349
2350    TEST_RESULT("incomplete block test");
2351}
2352
2353
2354static int _cmp_double(void *key1, size_t keylen1, void *key2, size_t keylen2)
2355{
2356    double aa, bb;
2357
2358    if (!keylen1) {
2359        // key1 not set
2360        return -1;
2361    }
2362    if (!keylen2) {
2363        // key2 not set
2364        return 1;
2365    }
2366
2367    aa = *(double *)key1;
2368    bb = *(double *)key2;
2369
2370    if (aa<bb) {
2371        return -1;
2372    } else if (aa>bb) {
2373        return 1;
2374    } else {
2375        return 0;
2376    }
2377}
2378
2379void custom_compare_primitive_test()
2380{
2381    TEST_INIT();
2382
2383    memleak_start();
2384
2385    int i, r;
2386    int n = 10;
2387    fdb_file_handle *dbfile;
2388    fdb_kvs_handle *db;
2389    fdb_doc **doc = alca(fdb_doc*, n);
2390    fdb_doc *rdoc = NULL;
2391    fdb_status status;
2392    fdb_iterator *iterator;
2393
2394    char keybuf[256], bodybuf[256];
2395    double key_double, key_double_prev;
2396
2397    // remove previous dummy files
2398    r = system(SHELL_DEL" dummy* > errorlog.txt");
2399    (void)r;
2400
2401    fdb_config fconfig = fdb_get_default_config();
2402    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2403    fconfig.buffercache_size = 0;
2404    fconfig.wal_threshold = 1024;
2405    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2406    fconfig.compaction_threshold = 0;
2407    fconfig.multi_kv_instances = true;
2408
2409    kvs_config.custom_cmp = _cmp_double;
2410
2411    // open db with custom compare function for double key type
2412    fdb_open(&dbfile, "./dummy1", &fconfig);
2413    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2414    status = fdb_set_log_callback(db, logCallbackFunc,
2415                                  (void *) "custom_compare_primitive_test");
2416    TEST_CHK(status == FDB_RESULT_SUCCESS);
2417
2418    for (i=0;i<n;++i){
2419        key_double = 10000/(i*11.0);
2420        memcpy(keybuf, &key_double, sizeof(key_double));
2421        sprintf(bodybuf, "value: %d, %f", i, key_double);
2422        fdb_doc_create(&doc[i], (void*)keybuf, sizeof(key_double), NULL, 0,
2423            (void*)bodybuf, strlen(bodybuf)+1);
2424        fdb_set(db, doc[i]);
2425    }
2426
2427    // range scan (before flushing WAL)
2428    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2429    key_double_prev = -1;
2430    do {
2431        status = fdb_iterator_get(iterator, &rdoc);
2432        TEST_CHK(status == FDB_RESULT_SUCCESS);
2433        memcpy(&key_double, rdoc->key, rdoc->keylen);
2434        TEST_CHK(key_double > key_double_prev);
2435        key_double_prev = key_double;
2436        fdb_doc_free(rdoc);
2437        rdoc = NULL;
2438    } while(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2439    fdb_iterator_close(iterator);
2440
2441    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2442
2443    // range scan (after flushing WAL)
2444    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2445    key_double_prev = -1;
2446    do {
2447        status = fdb_iterator_get(iterator, &rdoc);
2448        TEST_CHK(status == FDB_RESULT_SUCCESS);
2449        memcpy(&key_double, rdoc->key, rdoc->keylen);
2450        TEST_CHK(key_double > key_double_prev);
2451        key_double_prev = key_double;
2452        fdb_doc_free(rdoc);
2453        rdoc = NULL;
2454    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2455    fdb_iterator_close(iterator);
2456
2457    // do compaction
2458    fdb_compact(dbfile, (char *) "./dummy2");
2459
2460    // range scan (after compaction)
2461    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2462    key_double_prev = -1;
2463    do {
2464        status = fdb_iterator_get(iterator, &rdoc);
2465        TEST_CHK(status == FDB_RESULT_SUCCESS);
2466        memcpy(&key_double, rdoc->key, rdoc->keylen);
2467        TEST_CHK(key_double > key_double_prev);
2468        key_double_prev = key_double;
2469        fdb_doc_free(rdoc);
2470        rdoc = NULL;
2471    } while(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2472    fdb_iterator_close(iterator);
2473
2474    // close db file
2475    fdb_kvs_close(db);
2476    fdb_close(dbfile);
2477
2478    // free all documents
2479    for (i=0;i<n;++i){
2480        fdb_doc_free(doc[i]);
2481    }
2482
2483    // free all resources
2484    fdb_shutdown();
2485
2486    memleak_end();
2487
2488    TEST_RESULT("custom compare function for primitive key test");
2489}
2490
2491static int _cmp_variable(void *key1, size_t keylen1, void *key2, size_t keylen2)
2492{
2493    if (keylen1 < 6 || keylen2 < 6) {
2494        return (keylen1 - keylen2);
2495    }
2496    // compare only 3rd~8th bytes (ignore the others)
2497    return memcmp((uint8_t*)key1+2, (uint8_t*)key2+2, 6);
2498}
2499
2500void custom_compare_variable_test()
2501{
2502    TEST_INIT();
2503
2504    memleak_start();
2505
2506    int i, j, r;
2507    int n = 1000;
2508    int count;
2509    fdb_file_handle *dbfile;
2510    fdb_kvs_handle *db, *db2;
2511    fdb_doc **doc = alca(fdb_doc*, n);
2512    fdb_doc *rdoc = NULL;
2513    fdb_status status;
2514    fdb_iterator *iterator;
2515
2516    size_t keylen = 16;
2517    size_t prev_keylen;
2518    char keybuf[256], bodybuf[256];
2519    char prev_key[256];
2520
2521    // remove previous dummy files
2522    r = system(SHELL_DEL" dummy* > errorlog.txt");
2523    (void)r;
2524
2525    fdb_config fconfig = fdb_get_default_config();
2526    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2527    fconfig.seqtree_opt = FDB_SEQTREE_USE;
2528    fconfig.buffercache_size = 0;
2529    fconfig.wal_threshold = 1024;
2530    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2531    fconfig.compaction_threshold = 0;
2532    fconfig.multi_kv_instances = true;
2533
2534    kvs_config.custom_cmp = _cmp_variable;
2535
2536    // open db with custom compare function for variable length key type
2537    //fdb_open_cmp_variable(&dbfile, "./dummy1", &fconfig);
2538    fdb_open(&dbfile, "./dummy1", &fconfig);
2539    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2540    status = fdb_set_log_callback(db, logCallbackFunc,
2541                                  (void *) "custom_compare_variable_test");
2542    TEST_CHK(status == FDB_RESULT_SUCCESS);
2543
2544    for (i=0;i<n;++i){
2545        for (j=0;j<2;++j){
2546            keybuf[j] = 'a' + rand()%('z'-'a');
2547        }
2548        sprintf(keybuf+2, "%06d", i);
2549        for (j=8;(size_t)j<keylen-1;++j){
2550            keybuf[j] = 'a' + rand()%('z'-'a');
2551        }
2552        keybuf[keylen-1] = 0;
2553        sprintf(bodybuf, "value: %d", i);
2554        fdb_doc_create(&doc[i], (void*)keybuf, keylen, NULL, 0,
2555            (void*)bodybuf, strlen(bodybuf)+1);
2556        fdb_set(db, doc[i]);
2557    }
2558
2559    // point query
2560    for (i=0;i<n;++i){
2561        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
2562        status = fdb_get(db, rdoc);
2563
2564        TEST_CHK(status == FDB_RESULT_SUCCESS);
2565        TEST_CHK(rdoc->bodylen == doc[i]->bodylen);
2566        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2567
2568        fdb_doc_free(rdoc);
2569        rdoc = NULL;
2570    }
2571
2572    // range scan (before flushing WAL)
2573    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2574    sprintf(prev_key, "%016d", 0);
2575    count = 0;
2576    prev_keylen = 16;
2577    do {
2578        status = fdb_iterator_get(iterator, &rdoc);
2579        TEST_CHK(status == FDB_RESULT_SUCCESS);
2580        TEST_CHK(_cmp_variable(prev_key, prev_keylen, rdoc->key, rdoc->keylen) <= 0);
2581        prev_keylen = rdoc->keylen;
2582        memcpy(prev_key, rdoc->key, rdoc->keylen);
2583        fdb_doc_free(rdoc);
2584        rdoc = NULL;
2585        count++;
2586    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2587    TEST_CHK(count == n);
2588    fdb_iterator_close(iterator);
2589
2590    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2591
2592    // range scan (after flushing WAL)
2593    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2594    sprintf(prev_key, "%016d", 0);
2595    count = 0;
2596    prev_keylen = 16;
2597    do {
2598        status = fdb_iterator_get(iterator, &rdoc);
2599        TEST_CHK(status == FDB_RESULT_SUCCESS);
2600        TEST_CHK(_cmp_variable(prev_key, prev_keylen, rdoc->key, rdoc->keylen)
2601                 <= 0);
2602        prev_keylen = rdoc->keylen;
2603        memcpy(prev_key, rdoc->key, rdoc->keylen);
2604        fdb_doc_free(rdoc);
2605        rdoc = NULL;
2606        count++;
2607    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2608    TEST_CHK(count == n);
2609    fdb_iterator_close(iterator);
2610
2611    // do compaction
2612    fdb_compact(dbfile, (char *) "./dummy2");
2613
2614    // range scan (after compaction)
2615    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2616    sprintf(prev_key, "%016d", 0);
2617    count = 0;
2618    prev_keylen = 16;
2619    do {
2620        status = fdb_iterator_get(iterator, &rdoc);
2621        TEST_CHK(status == FDB_RESULT_SUCCESS);
2622        TEST_CHK(_cmp_variable(prev_key, prev_keylen, rdoc->key, rdoc->keylen) <= 0);
2623        prev_keylen = rdoc->keylen;
2624        memcpy(prev_key, rdoc->key, rdoc->keylen);
2625        fdb_doc_free(rdoc);
2626        rdoc = NULL;
2627        count++;
2628    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2629    TEST_CHK(count == n);
2630    fdb_iterator_close(iterator);
2631
2632    // range scan by sequence
2633    fdb_iterator_sequence_init(db, &iterator, 0, 0, 0x0);
2634    count = 0;
2635    do { // forward
2636        status = fdb_iterator_get(iterator, &rdoc);
2637        TEST_CHK(status == FDB_RESULT_SUCCESS);
2638        fdb_doc_free(rdoc);
2639        rdoc = NULL;
2640        count++;
2641    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2642    TEST_CHK(count == n);
2643
2644    // Reverse direction
2645    for (; fdb_iterator_prev(iterator) != FDB_RESULT_ITERATOR_FAIL; --count) {
2646        status = fdb_iterator_get(iterator, &rdoc);
2647        TEST_CHK(status == FDB_RESULT_SUCCESS);
2648        fdb_doc_free(rdoc);
2649        rdoc = NULL;
2650    };
2651    TEST_CHK(count == 0);
2652    fdb_iterator_close(iterator);
2653
2654    // open another handle
2655    kvs_config.custom_cmp = NULL;
2656    fdb_kvs_open_default(dbfile, &db2, &kvs_config);
2657
2658    // point query
2659    for (i=0;i<n;++i){
2660        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
2661        status = fdb_get(db2, rdoc);
2662
2663        TEST_CHK(status == FDB_RESULT_SUCCESS);
2664        TEST_CHK(rdoc->bodylen == doc[i]->bodylen);
2665        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2666
2667        fdb_doc_free(rdoc);
2668        rdoc = NULL;
2669    }
2670
2671    // close db file
2672    fdb_kvs_close(db);
2673    fdb_kvs_close(db2);
2674    fdb_close(dbfile);
2675
2676    // free all documents
2677    for (i=0;i<n;++i){
2678        fdb_doc_free(doc[i]);
2679    }
2680
2681    // free all resources
2682    fdb_shutdown();
2683
2684    memleak_end();
2685
2686    TEST_RESULT("custom compare function for variable length key test");
2687}
2688
2689/*
2690 * custom compare test with commit and compact
2691 *    eqkeys:  boolean to toggle whether bytes in
2692 *             comparision range are equal
2693 */
2694void custom_compare_commit_compact(bool eqkeys)
2695{
2696    TEST_INIT();
2697
2698    memleak_start();
2699
2700    int i, j, r;
2701    int count;
2702    int n = 10;
2703    static const int len = 1024;
2704    char keybuf[len];
2705    static const char *achar = "a";
2706    fdb_doc *rdoc = NULL;
2707    fdb_status status;
2708    fdb_file_handle *dbfile;
2709    fdb_kvs_handle *db;
2710    fdb_iterator *iterator;
2711    fdb_config fconfig = fdb_get_default_config();
2712    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2713
2714    // remove previous dummy files
2715    r = system(SHELL_DEL" dummy* > errorlog.txt");
2716    (void)r;
2717
2718    fconfig.buffercache_size = 0;
2719    fconfig.wal_threshold = 1024;
2720    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2721    fconfig.compaction_threshold = 0;
2722    fconfig.multi_kv_instances = true;
2723
2724    kvs_config.custom_cmp = _cmp_variable;
2725
2726
2727
2728    r = system(SHELL_DEL" dummy* > errorlog.txt");
2729    (void)r;
2730
2731
2732    // open db
2733    status = fdb_open(&dbfile, "./dummy1", &fconfig);
2734    TEST_CHK(status == FDB_RESULT_SUCCESS);
2735    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
2736    TEST_CHK(status == FDB_RESULT_SUCCESS);
2737
2738
2739    for (i=0;i<n;++i){
2740        if(eqkeys){
2741            sprintf(keybuf, "%d", i);
2742            for (j=1;j<len;++j) {
2743                keybuf[j] = *achar;
2744            }
2745        } else {
2746            sprintf(keybuf, "000%d", i);
2747            for (j=4;j<len;++j) {
2748                keybuf[j] = *achar;
2749            }
2750        }
2751        keybuf[len-1] = '\0';
2752        // set kv
2753        status = fdb_set_kv(db, keybuf, strlen(keybuf), NULL, 0);
2754        TEST_CHK(status == FDB_RESULT_SUCCESS);
2755    }
2756
2757    // compact pre & post commit
2758    fdb_compact(dbfile, NULL);
2759    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2760    fdb_compact(dbfile, NULL);
2761
2762    // scan after flush
2763    count = 0;
2764    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2765    do {
2766        status = fdb_iterator_get(iterator, &rdoc);
2767        TEST_CHK(status == FDB_RESULT_SUCCESS);
2768        fdb_doc_free(rdoc);
2769        rdoc = NULL;
2770        count++;
2771    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2772
2773    if (eqkeys) {
2774        // since the custom cmp function compares only 3rd~8th bytes,
2775        // all keys are identified as the same key.
2776        TEST_CHK(count == 1);
2777    } else {
2778        TEST_CHK(count == n);
2779    }
2780
2781    fdb_iterator_close(iterator);
2782
2783    fdb_close(dbfile);
2784    fdb_shutdown();
2785
2786    memleak_end();
2787    TEST_RESULT("custom compare commit compact");
2788
2789}
2790
2791void custom_seqnum_test(bool multi_kv)
2792{
2793    TEST_INIT();
2794
2795    memleak_start();
2796
2797    int i, r;
2798    int n = 10;
2799    int num_kv = 4;
2800    fdb_file_handle *dbfile;
2801    fdb_kvs_handle **db = alca(fdb_kvs_handle*, num_kv);
2802    fdb_doc **doc = alca(fdb_doc*, n);
2803    fdb_doc real_doc;
2804    fdb_doc *rdoc = &real_doc;
2805    fdb_status status;
2806    fdb_iterator *iterator;
2807
2808    char keybuf[256], bodybuf[256];
2809    memset(&real_doc, 0, sizeof(fdb_doc));
2810    real_doc.key = &keybuf;
2811    real_doc.body = &bodybuf;
2812
2813    // remove previous dummy files
2814    r = system(SHELL_DEL" dummy* > errorlog.txt");
2815    (void)r;
2816
2817    fdb_config fconfig = fdb_get_default_config();
2818    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2819
2820    fconfig.buffercache_size = 0;
2821    fconfig.wal_threshold = 1024;
2822    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2823    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
2824    fconfig.compaction_threshold = 0;
2825    fconfig.multi_kv_instances = multi_kv;
2826    r = 0;
2827
2828    fdb_open(&dbfile, "./dummy1", &fconfig);
2829    if (multi_kv) {
2830        num_kv = 4;
2831        for (r = num_kv - 1; r >= 0; --r) {
2832            char tmp[16];
2833            sprintf(tmp, "kv%d", r);
2834            status = fdb_kvs_open(dbfile, &db[r], tmp, &kvs_config);
2835            TEST_CHK(status == FDB_RESULT_SUCCESS);
2836            status = fdb_set_log_callback(db[r], logCallbackFunc,
2837                                          (void *) "custom_seqnum_test");
2838            TEST_CHK(status == FDB_RESULT_SUCCESS);
2839        }
2840    } else {
2841        num_kv = 1;
2842        status = fdb_kvs_open_default(dbfile, &db[r], &kvs_config);
2843        TEST_CHK(status == FDB_RESULT_SUCCESS);
2844        status = fdb_set_log_callback(db[r], logCallbackFunc,
2845                (void *) "custom_seqnum_test");
2846        TEST_CHK(status == FDB_RESULT_SUCCESS);
2847    }
2848
2849    for (i = 0; i < n/2; ++i){
2850        sprintf(keybuf, "key%d", i);
2851        sprintf(bodybuf, "body%d", i);
2852        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf) + 1, NULL, 0,
2853            (void*)bodybuf, strlen(bodybuf)+1);
2854        for (r = num_kv - 1; r >= 0; --r) {
2855            status = fdb_set(db[r], doc[i]);
2856            TEST_CHK(status == FDB_RESULT_SUCCESS);
2857        }
2858    }
2859
2860    for (i = n/2; i < n; ++i){
2861        sprintf(keybuf, "key%d", i);
2862        sprintf(bodybuf, "body%d", i);
2863        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf) + 1, NULL, 0,
2864            (void*)bodybuf, strlen(bodybuf)+1);
2865        for (r = num_kv - 1; r >= 0; --r) {
2866            fdb_doc_set_seqnum(doc[i], (i+1)*2); // double seqnum instead of ++
2867            status = fdb_set(db[r], doc[i]);
2868            TEST_CHK(status == FDB_RESULT_SUCCESS);
2869        }
2870    }
2871
2872    for (r = num_kv - 1; r >= 0; --r) {
2873        // range scan (before flushing WAL)
2874        fdb_iterator_init(db[r], &iterator, NULL, 0, NULL, 0, 0x0);
2875        i = 0;
2876        do {
2877            status = fdb_iterator_get(iterator, &rdoc);
2878            TEST_CHK(status == FDB_RESULT_SUCCESS);
2879            TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2880            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2881            ++i;
2882            if (i <= n/2) {
2883                TEST_CHK(rdoc->seqnum == (fdb_seqnum_t)i);
2884            } else {
2885                TEST_CHK(rdoc->seqnum == (fdb_seqnum_t)i*2);
2886            }
2887        } while(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2888        fdb_iterator_close(iterator);
2889
2890        fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2891    }
2892
2893    // do compaction
2894    fdb_compact(dbfile, (char *) "./dummy2");
2895
2896    for (i = n/2; i < n; ++i){
2897        sprintf(keybuf, "key%d", i);
2898        for (r = num_kv - 1; r >= 0; --r) {
2899            rdoc->seqnum = (i + 1)*2;
2900            status = fdb_get_byseq(db[r], rdoc);
2901            TEST_CHK(status == FDB_RESULT_SUCCESS);
2902        }
2903    }
2904
2905    fdb_close(dbfile);
2906
2907    // free all documents
2908    for (i=0;i<n;++i){
2909        fdb_doc_free(doc[i]);
2910    }
2911
2912    // free all resources
2913    fdb_shutdown();
2914
2915    memleak_end();
2916
2917    sprintf(bodybuf,"Custom sequence number test %s", multi_kv ?
2918            "multiple kv instances" : "single kv instance");
2919    TEST_RESULT(bodybuf);
2920}
2921
2922void doc_compression_test()
2923{
2924    TEST_INIT();
2925
2926    memleak_start();
2927
2928    int i, r;
2929    int n = 10;
2930    int dummy_len = 32;
2931    fdb_file_handle *dbfile;
2932    fdb_kvs_handle *db;
2933    fdb_doc **doc = alca(fdb_doc*, n);
2934    fdb_doc *rdoc = NULL;
2935    fdb_status status;
2936
2937    char keybuf[256], metabuf[256], bodybuf[256], temp[256];
2938
2939    // remove previous dummy files
2940    r = system(SHELL_DEL" dummy* > errorlog.txt");
2941    (void)r;
2942
2943    fdb_config fconfig = fdb_get_default_config();
2944    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2945    fconfig.buffercache_size = 0;
2946    fconfig.wal_threshold = 1024;
2947    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2948    fconfig.compress_document_body = true;
2949    fconfig.compaction_threshold = 0;
2950    fconfig.purging_interval = 1;
2951
2952    // open db
2953    fdb_open(&dbfile, "./dummy1", &fconfig);
2954    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2955    status = fdb_set_log_callback(db, logCallbackFunc,
2956                                  (void *) "doc_compression_test");
2957    TEST_CHK(status == FDB_RESULT_SUCCESS);
2958
2959    // set dummy str
2960    memset(temp, 'a', dummy_len);
2961    temp[dummy_len]=0;
2962
2963    // insert documents
2964    for (i=0;i<n;++i){
2965        sprintf(keybuf, "key%d", i);
2966        sprintf(metabuf, "meta%d", i);
2967        sprintf(bodybuf, "body%d_%s", i, temp);
2968        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2969            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2970        fdb_set(db, doc[i]);
2971    }
2972
2973    // remove document #5
2974    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, doc[5]->meta, doc[5]->metalen, NULL, 0);
2975    status = fdb_del(db, rdoc);
2976    TEST_CHK(status == FDB_RESULT_SUCCESS);
2977    fdb_doc_free(rdoc);
2978    rdoc = NULL;
2979
2980    // commit
2981    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2982
2983    // close the db
2984    fdb_kvs_close(db);
2985    fdb_close(dbfile);
2986
2987    // reopen
2988    fdb_open(&dbfile, "./dummy1", &fconfig);
2989    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2990    status = fdb_set_log_callback(db, logCallbackFunc,
2991                                  (void *) "doc_compression_test");
2992    TEST_CHK(status == FDB_RESULT_SUCCESS);
2993
2994    // update dummy str
2995    dummy_len = 64;
2996    memset(temp, 'b', dummy_len);
2997    temp[dummy_len]=0;
2998
2999    // update document #0 and #1
3000    for (i=0;i<2;++i){
3001        sprintf(metabuf, "newmeta%d", i);
3002        sprintf(bodybuf, "newbody%d_%s", i, temp);
3003        fdb_doc_update(&doc[i], (void *)metabuf, strlen(metabuf),
3004            (void *)bodybuf, strlen(bodybuf));
3005        fdb_set(db, doc[i]);
3006    }
3007
3008    // commit
3009    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
3010
3011    // retrieve documents
3012    for (i=0;i<n;++i){
3013        // search by key
3014        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
3015        status = fdb_get(db, rdoc);
3016
3017        if (i != 5) {
3018            // updated documents
3019            TEST_CHK(status == FDB_RESULT_SUCCESS);
3020            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
3021            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
3022        } else {
3023            // removed document
3024            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
3025        }
3026
3027        // free result document
3028        fdb_doc_free(rdoc);
3029        rdoc = NULL;
3030    }
3031
3032    // do compaction
3033    fdb_compact(dbfile, (char *) "./dummy2");
3034
3035    // retrieve documents after compaction
3036    for (i=0;i<n;++i){
3037        // search by key
3038        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
3039        status = fdb_get(db, rdoc);
3040
3041        if (i != 5) {
3042            // updated documents
3043            TEST_CHK(status == FDB_RESULT_SUCCESS);
3044            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
3045            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
3046        } else {
3047            // removed document
3048            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
3049        }
3050
3051        // free result document
3052        fdb_doc_free(rdoc);
3053        rdoc = NULL;
3054    }
3055
3056    // free all documents
3057    for (i=0;i<n;++i){
3058        fdb_doc_free(doc[i]);
3059    }
3060
3061    // close db file
3062    fdb_kvs_close(db);
3063    fdb_close(dbfile);
3064
3065    // free all resources
3066    fdb_shutdown();
3067
3068    memleak_end();
3069
3070    TEST_RESULT("document compression test");
3071}
3072
3073void read_doc_by_offset_test()
3074{
3075    TEST_INIT();
3076    memleak_start();
3077
3078    int i, r;
3079    int n = 100;
3080    fdb_file_handle *dbfile;
3081    fdb_kvs_handle *db;
3082    fdb_doc **doc = alca(fdb_doc*, n);
3083    fdb_doc *rdoc, *rdoc1;
3084    fdb_status status;
3085
3086    char keybuf[256], metabuf[256], bodybuf[256];
3087
3088    // remove previous dummy files
3089    r = system(SHELL_DEL" dummy* > errorlog.txt");
3090    (void)r;
3091
3092    fdb_config fconfig = fdb_get_default_config();
3093    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
3094    fconfig.buffercache_size = 0;
3095    fconfig.wal_threshold = 1024;
3096    fconfig.flags = FDB_OPEN_FLAG_CREATE;
3097    fconfig.purging_interval = 3600;
3098    fconfig.compaction_threshold = 0;
3099
3100    // open db
3101    fdb_open(&dbfile, "./dummy1", &fconfig);
3102    fdb_kvs_open_default(dbfile, &db, &kvs_config);
3103    status = fdb_set_log_callback(db, logCallbackFunc,
3104                                  (void *) "read_doc_by_offset_test");
3105    TEST_CHK(status == FDB_RESULT_SUCCESS);
3106
3107    // insert documents
3108    for (i=0;i<n;++i){
3109        sprintf(keybuf, "key%d", i);
3110        sprintf(metabuf, "meta%d", i);
3111        sprintf(bodybuf, "body%d", i);
3112        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
3113            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
3114        fdb_set(db, doc[i]);
3115    }
3116
3117    // commit
3118    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
3119
3120    // update documents from #0 to #49
3121    for (i=0;i<n/2;++i){
3122        sprintf(metabuf, "meta2%d", i);
3123        sprintf(bodybuf, "body2%d", i);
3124        fdb_doc_update(&doc[i], (void *)metabuf, strlen(metabuf),
3125            (void *)bodybuf, strlen(bodybuf));
3126        fdb_set(db, doc[i]);
3127    }
3128
3129    // remove document #50
3130    fdb_doc_create(&rdoc, doc[50]->key, doc[50]->keylen, doc[50]->meta,
3131                   doc[50]->metalen, NULL, 0);
3132    status = fdb_del(db, rdoc);
3133    TEST_CHK(status == FDB_RESULT_SUCCESS);
3134    fdb_doc_free(rdoc);
3135    rdoc = NULL;
3136
3137    // commit
3138    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
3139
3140    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, NULL, 0, NULL, 0);
3141    status = fdb_get_metaonly(db, rdoc);
3142    TEST_CHK(status == FDB_RESULT_SUCCESS);
3143    TEST_CHK(rdoc->deleted == false);
3144    TEST_CMP(rdoc->meta, doc[5]->meta, rdoc->metalen);
3145    // Fetch #5 doc using its offset.
3146    status = fdb_get_byoffset(db, rdoc);
3147    TEST_CHK(status == FDB_RESULT_SUCCESS);
3148    TEST_CHK(rdoc->deleted == false);
3149    TEST_CMP(rdoc->meta, doc[5]->meta, rdoc->metalen);
3150    TEST_CMP(rdoc->body, doc[5]->body, rdoc->bodylen);
3151
3152    // MB-13095
3153    fdb_doc_create(&rdoc1, NULL, 0, NULL, 0, NULL, 0);
3154    rdoc1->offset = rdoc->offset;
3155    status = fdb_get_byoffset(db, rdoc1);
3156    TEST_CHK(status == FDB_RESULT_SUCCESS);
3157    TEST_CMP(rdoc1->key, doc[5]->key, rdoc1->keylen);
3158    TEST_CMP(rdoc1->meta, doc[5]->meta, rdoc1->metalen);
3159    TEST_CMP(rdoc1->body, doc[5]->body, rdoc1->bodylen);
3160
3161    fdb_doc_free(rdoc);
3162    rdoc = NULL;
3163    fdb_doc_free(rdoc1);
3164
3165    // do compaction
3166    fdb_compact(dbfile, (char *) "./dummy2");
3167
3168    fdb_doc_create(&rdoc, doc[50]->key, doc[50]->keylen, NULL, 0, NULL, 0);
3169    status = fdb_get_metaonly(db, rdoc);
3170    TEST_CHK(status == FDB_RESULT_SUCCESS);
3171    TEST_CHK(rdoc->deleted == true);
3172    TEST_CMP(rdoc->meta, doc[50]->meta, rdoc->metalen);
3173    // Fetch #50 doc using its offset.
3174    status = fdb_get_byoffset(db, rdoc);
3175    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
3176    TEST_CHK(rdoc->deleted == true);
3177    fdb_doc_free(rdoc);
3178    rdoc = NULL;
3179
3180    // free all documents
3181    for (i=0;i<n;++i){
3182        fdb_doc_free(doc[i]);
3183    }
3184
3185    // close db file
3186    fdb_kvs_close(db);
3187    fdb_close(dbfile);
3188
3189    // free all resources
3190    fdb_shutdown();
3191
3192    memleak_end();
3193
3194    TEST_RESULT("read_doc_by_offset test");
3195}
3196
3197void purge_logically_deleted_doc_test()
3198{
3199    TEST_INIT();
3200
3201    memleak_start();
3202
3203    int i, r;
3204    int n = 10;
3205    fdb_file_handle *dbfile;
3206    fdb_kvs_handle *db;
3207    fdb_doc **doc = alca(fdb_doc*, n);
3208    fdb_doc *rdoc = NULL;
3209    fdb_status status;
3210
3211    char keybuf[256], metabuf[256], bodybuf[256];
3212
3213    // remove previous dummy files
3214    r = system(SHELL_DEL" dummy* fdb_test_config.json > errorlog.txt");
3215    (void)r;
3216
3217    fdb_config fconfig = fdb_get_default_config();
3218    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
3219    fconfig.buffercache_size = 0;
3220    fconfig.wal_threshold = 1024;
3221    fconfig.flags = FDB_OPEN_FLAG_CREATE;
3222    fconfig.purging_interval = 2;
3223    fconfig.compaction_threshold = 0;
3224
3225    // open db
3226    fdb_open(&dbfile, "./dummy1", &fconfig);
3227    fdb_kvs_open_default(dbfile, &db, &kvs_config);
3228    status = fdb_set_log_callback(db, logCallbackFunc,
3229                                  (void *) "purge_logically_deleted_doc_test");
3230    TEST_CHK(status == FDB_RESULT_SUCCESS);
3231
3232    // insert documents
3233    for (i=0;i<n;++i){
3234        sprintf(keybuf, "key%d", i);
3235        sprintf(metabuf, "meta%d", i);
3236        sprintf(bodybuf, "body%d", i);
3237        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
3238            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
3239        fdb_set(db, doc[i]);
3240    }
3241
3242    // remove document #5
3243    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, doc[5]->meta, doc[5]->metalen, NULL, 0);
3244    status = fdb_del(db, rdoc);
3245    TEST_CHK(status == FDB_RESULT_SUCCESS);
3246    fdb_doc_free(rdoc);
3247    rdoc = NULL;
3248
3249    // commit
3250    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
3251
3252    // do compaction
3253    fdb_compact(dbfile, (char *) "./dummy2");
3254
3255    // retrieve documents after compaction
3256    for (i=0;i<n;++i){
3257        // search by key
3258        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
3259        status = fdb_get(db, rdoc);
3260
3261        if (i != 5) {
3262            // updated documents
3263            TEST_CHK(status == FDB_RESULT_SUCCESS);
3264            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
3265            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
3266        } else {
3267            // removed document
3268            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
3269        }
3270        // free result document
3271        fdb_doc_free(rdoc);
3272        rdoc = NULL;
3273
3274        // retrieve metadata
3275        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
3276        status = fdb_get_metaonly(db, rdoc);
3277        if (i != 5) {
3278            TEST_CHK(status == FDB_RESULT_SUCCESS);
3279        } else {
3280            // If the above compaction takes longer than two secs (e.g., slow disk),
3281            // then, fdb_get_metaonly will return KEY_NOT_FOUND error.
3282            TEST_CHK(status == FDB_RESULT_SUCCESS ||
3283                     status == FDB_RESULT_KEY_NOT_FOUND);
3284        }
3285        fdb_doc_free(rdoc);
3286        rdoc = NULL;
3287    }
3288
3289    printf("wait for 3 seconds..\n");
3290    sleep(3);
3291
3292    // do one more compaction
3293    fdb_compact(dbfile, (char *) "./dummy3");
3294
3295    // retrieve documents after compaction
3296    for (i=0;i<n;++i){
3297        // search by key
3298        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
3299        status = fdb_get(db, rdoc);
3300
3301        if (i != 5) {
3302            // updated documents
3303            TEST_CHK(status == FDB_RESULT_SUCCESS);
3304            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
3305            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
3306        } else {
3307            // removed document
3308            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
3309        }
3310        // free result document
3311        fdb_doc_free(rdoc);
3312        rdoc = NULL;
3313
3314        // retrieve metadata
3315        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
3316        status = fdb_get_metaonly(db, rdoc);
3317        if (i != 5) {
3318            TEST_CHK(status == FDB_RESULT_SUCCESS);
3319        } else {
3320            // logically deletec document must be purged during the compaction
3321            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
3322        }
3323        fdb_doc_free(rdoc);
3324        rdoc = NULL;
3325    }
3326
3327    // close db file
3328    fdb_kvs_close(db);
3329    fdb_close(dbfile);
3330
3331    // free all documents
3332    for (i=0;i<n;++i){
3333        fdb_doc_free(doc[i]);
3334    }
3335
3336    // free all resources
3337    fdb_shutdown();
3338
3339    memleak_end();
3340
3341    TEST_RESULT("purge logically deleted doc test");
3342}
3343
3344void api_wrapper_test()
3345{
3346    TEST_INIT();
3347
3348    memleak_start();
3349
3350    int i, r;
3351    int n = 10;
3352    size_t valuelen;
3353    void *value;
3354    fdb_file_handle *dbfile;
3355    fdb_kvs_handle *db;
3356    fdb_status status;
3357
3358    char keybuf[256], bodybuf[256], temp[256];
3359
3360    // remove previous dummy files
3361    r = system(SHELL_DEL" dummy* > errorlog.txt");
3362    (void)r;
3363
3364    fdb_config fconfig = fdb_get_default_config();
3365    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
3366    fconfig.buffercache_size = 0;
3367    fconfig.wal_threshold = 1024;
3368    fconfig.flags = FDB_OPEN_FLAG_CREATE;
3369    fconfig.purging_interval = 0;
3370    fconfig.compaction_threshold = 0;
3371
3372    // open db
3373    fdb_open(&dbfile, "./dummy1", &fconfig);
3374    fdb_kvs_open_default(dbfile, &db, &kvs_config);
3375    status = fdb_set_log_callback(db, logCallbackFunc,
3376                                  (void *) "api_wrapper_test");
3377    TEST_CHK(status == FDB_RESULT_SUCCESS);
3378
3379    // error check
3380    status = fdb_set_kv(db, NULL, 0, NULL, 0);
3381    TEST_CHK(status == FDB_RESULT_INVALID_ARGS);
3382
3383    // insert key-value pairs
3384    for (i=0;i<n;++i){
3385        sprintf(keybuf, "key%d", i);
3386        sprintf(bodybuf, "body%d", i);
3387        status = fdb_set_kv(db, keybuf, strlen(keybuf), bodybuf, strlen(bodybuf));
3388        TEST_CHK(status == FDB_RESULT_SUCCESS);
3389    }
3390
3391    // remove key5
3392    sprintf(keybuf, "key%d", 5);
3393    status = fdb_del_kv(db, keybuf, strlen(keybuf));
3394    TEST_CHK(status == FDB_RESULT_SUCCESS);
3395
3396    // error check
3397    status = fdb_del_kv(db, NULL, 0);
3398    TEST_CHK(status == FDB_RESULT_INVALID_ARGS);
3399
3400    // retrieve key-value pairs
3401    for (i=0;i<n;++i){
3402        sprintf(keybuf, "key%d", i);
3403        status = fdb_get_kv(db, keybuf, strlen(keybuf), &value, &valuelen);
3404
3405        if (i != 5) {
3406            // updated documents
3407            TEST_CHK(status == FDB_RESULT_SUCCESS);
3408            sprintf(temp, "body%d", i);
3409            TEST_CMP(value, temp, valuelen);
3410            fdb_free_block(value);
3411        } else {
3412            // removed document
3413            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
3414        }
3415    }
3416
3417    // error check
3418    status = fdb_get_kv(db, NULL, 0, &value, &valuelen);
3419    TEST_CHK(status == FDB_RESULT_INVALID_ARGS);
3420
3421    status = fdb_get_kv(db, keybuf, strlen(keybuf), NULL, NULL);
3422    TEST_CHK(status == FDB_RESULT_INVALID_ARGS);
3423
3424    // close db file
3425    fdb_kvs_close(db);
3426    fdb_close(dbfile);
3427
3428    // free all resources
3429    fdb_shutdown();
3430
3431    memleak_end();
3432
3433    TEST_RESULT("API wrapper test");
3434}
3435
3436
3437void flush_before_commit_test()
3438{
3439    TEST_INIT();
3440
3441    memleak_start();
3442
3443    int i, r;
3444    int n = 30;
3445    fdb_file_handle *dbfile, *dbfile_txn;
3446    fdb_kvs_handle *db, *db_txn;
3447    fdb_doc **doc = alca(fdb_doc*, n);
3448    fdb_status status;
3449
3450    char keybuf[256], metabuf[256], bodybuf[256];
3451
3452    // remove previous dummy files
3453    r = system(SHELL_DEL" dummy* > errorlog.txt");
3454    (void)r;
3455
3456    fdb_config fconfig = fdb_get_default_config();
3457    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
3458    fconfig.buffercache_size = 0;
3459    fconfig.wal_threshold = 5;
3460    fconfig.flags = FDB_OPEN_FLAG_CREATE;
3461    fconfig.purging_interval = 0;
3462    fconfig.compaction_threshold = 0;
3463    fconfig.wal_flush_before_commit = true;
3464
3465    // open db
3466    fdb_open(&dbfile, "dummy1", &fconfig);
3467    fdb_open(&dbfile_txn, "dummy1", &fconfig);
3468    fdb_kvs_open_default(dbfile, &db, &kvs_config);
3469    fdb_kvs_open_default(dbfile_txn, &db_txn, &kvs_config);
3470    status = fdb_set_log_callback(db_txn, logCallbackFunc,
3471                                  (void *) "flush_before_commit_test");
3472    TEST_CHK(status == FDB_RESULT_SUCCESS);
3473
3474    // create docs
3475    for (i=0;i<n;++i){
3476        sprintf(keybuf, "key%d", i);
3477        sprintf(metabuf, "meta%d", i);
3478        sprintf(bodybuf, "body%d", i);
3479        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
3480                                (void*)metabuf, strlen(metabuf),
3481                                (void*)bodybuf, strlen(bodybuf));
3482    }
3483
3484    // non-transactional commit first, transactional commit next
3485    fdb_begin_transaction(dbfile_txn, FDB_ISOLATION_READ_COMMITTED);
3486    for (i=0;i<2;++i){
3487        fdb_set(db, doc[i]);
3488    }
3489    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
3490    for (i=0;i<2;++i){
3491        fdb_set(db_txn, doc[i]);
3492    }
3493