1/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/*
3 *     Copyright 2010 Couchbase, Inc
4 *
5 *   Licensed under the Apache License, Version 2.0 (the "License");
6 *   you may not use this file except in compliance with the License.
7 *   You may obtain a copy of the License at
8 *
9 *       http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *   Unless required by applicable law or agreed to in writing, software
12 *   distributed under the License is distributed on an "AS IS" BASIS,
13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *   See the License for the specific language governing permissions and
15 *   limitations under the License.
16 */
17
18#include <stdio.h>
19#include <stdlib.h>
20#include <string.h>
21#include <stdint.h>
22#include <time.h>
23#if !defined(WIN32) && !defined(_WIN32)
24#include <unistd.h>
25#endif
26
27#include "libforestdb/forestdb.h"
28#include "test.h"
29#include "internal_types.h"
30#include "functional_util.h"
31
32void basic_test()
33{
34    TEST_INIT();
35
36    memleak_start();
37
38    int i, r;
39    int n = 10;
40    fdb_file_handle *dbfile, *dbfile_rdonly;
41    fdb_kvs_handle *db;
42    fdb_kvs_handle *db_rdonly;
43    fdb_doc **doc = alca(fdb_doc*, n);
44    fdb_doc *rdoc = NULL;
45    fdb_status status;
46
47    char keybuf[256], metabuf[256], bodybuf[256];
48
49    // remove previous dummy test files
50    r = system(SHELL_DEL" dummy* > errorlog.txt");
51    (void)r;
52
53    // Get the ForestDB version
54    const char *version = fdb_get_lib_version();
55    TEST_CHK(version != NULL && strlen(version) > 0);
56
57    fdb_config fconfig = fdb_get_default_config();
58    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
59    fconfig.wal_threshold = 1024;
60    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
61    fconfig.compaction_threshold = 0;
62    fconfig.purging_interval = 1;
63
64    // Read-Write mode test without a create flag.
65    fconfig.flags = 0;
66    status = fdb_open(&dbfile, "./dummy1", &fconfig);
67    TEST_CHK(status == FDB_RESULT_NO_SUCH_FILE);
68    TEST_CHK(!strcmp(fdb_error_msg(status), "no such file"));
69
70    // Read-Only mode test: Must not create new file.
71    fconfig.flags = FDB_OPEN_FLAG_RDONLY;
72    status = fdb_open(&dbfile, "./dummy1", &fconfig);
73    TEST_CHK(status == FDB_RESULT_NO_SUCH_FILE);
74
75    // Read-Only and Create mode: Must not create a new file.
76    fconfig.flags = FDB_OPEN_FLAG_RDONLY | FDB_OPEN_FLAG_CREATE;
77    status = fdb_open(&dbfile, "./dummy1", &fconfig);
78    TEST_CHK(status == FDB_RESULT_INVALID_CONFIG);
79    TEST_CHK(!strcmp(fdb_error_msg(status), "invalid configuration"));
80
81    // open and close db with a create flag.
82    fconfig.flags = FDB_OPEN_FLAG_CREATE;
83    status = fdb_open(&dbfile, "./dummy1", &fconfig);
84    TEST_CHK(status == FDB_RESULT_SUCCESS);
85    TEST_CHK(!strcmp(fdb_error_msg(status), "success"));
86    const char *file_version = fdb_get_file_version(dbfile);
87    TEST_CHK(file_version != NULL && strlen(file_version) > 0);
88    fdb_close(dbfile);
89
90    // reopen db
91    r = system(SHELL_DEL" dummy* > errorlog.txt");
92    (void)r;
93    fdb_open(&dbfile, "./dummy1",&fconfig);
94    fdb_kvs_open_default(dbfile, &db, &kvs_config);
95    status = fdb_set_log_callback(db, logCallbackFunc, (void *) "basic_test");
96    TEST_CHK(status == FDB_RESULT_SUCCESS);
97
98    // insert documents
99    for (i=0;i<n;++i){
100        sprintf(keybuf, "key%d", i);
101        sprintf(metabuf, "meta%d", i);
102        sprintf(bodybuf, "body%d", i);
103        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
104            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
105        fdb_set(db, doc[i]);
106    }
107
108    // remove document #5
109    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, doc[5]->meta,
110                   doc[5]->metalen, NULL, 0);
111    status = fdb_del(db, rdoc);
112    TEST_CHK(status == FDB_RESULT_SUCCESS);
113    fdb_doc_free(rdoc);
114    rdoc = NULL;
115
116    // commit
117    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
118
119    // check the file info
120    fdb_file_info info;
121    fdb_get_file_info(dbfile, &info);
122    TEST_CHK(info.doc_count == 9);
123    TEST_CHK(info.deleted_count == 1);
124    TEST_CHK(info.space_used > 0);
125    TEST_CHK(info.num_kv_stores == 1);
126
127    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, NULL, 0, NULL, 0);
128    status = fdb_get_metaonly(db, rdoc);
129    TEST_CHK(status == FDB_RESULT_SUCCESS);
130    TEST_CHK(rdoc->deleted == true);
131    TEST_CMP(rdoc->meta, doc[5]->meta, rdoc->metalen);
132    fdb_doc_free(rdoc);
133    rdoc = NULL;
134
135    // close the db
136    fdb_kvs_close(db);
137    fdb_close(dbfile);
138
139    // reopen
140    fdb_open(&dbfile, "./dummy1", &fconfig);
141    fdb_kvs_open_default(dbfile, &db, &kvs_config);
142    status = fdb_set_log_callback(db, logCallbackFunc, (void *) "basic_test");
143    TEST_CHK(status == FDB_RESULT_SUCCESS);
144
145    // update document #0 and #1
146    for (i=0;i<2;++i){
147        sprintf(metabuf, "meta2%d", i);
148        sprintf(bodybuf, "body2%d", i);
149        fdb_doc_update(&doc[i], (void *)metabuf, strlen(metabuf),
150            (void *)bodybuf, strlen(bodybuf));
151        fdb_set(db, doc[i]);
152    }
153
154    // commit
155    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
156
157    // retrieve documents
158    for (i=0;i<n;++i){
159        // search by key
160        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
161        status = fdb_get(db, rdoc);
162
163        if (i != 5) {
164            // updated documents
165            TEST_CHK(status == FDB_RESULT_SUCCESS);
166            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
167            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
168        } else {
169            // removed document
170            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
171            TEST_CHK(!strcmp(fdb_error_msg(status), "key not found"));
172        }
173
174        // free result document
175        fdb_doc_free(rdoc);
176        rdoc = NULL;
177    }
178
179    // do compaction
180    fdb_compact(dbfile, (char *) "./dummy2");
181
182    // retrieve documents after compaction
183    for (i=0;i<n;++i){
184        // search by key
185        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
186        status = fdb_get(db, rdoc);
187
188        if (i != 5) {
189            // updated documents
190            TEST_CHK(status == FDB_RESULT_SUCCESS);
191            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
192            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
193        } else {
194            // removed document
195            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
196        }
197
198        // free result document
199        fdb_doc_free(rdoc);
200        rdoc = NULL;
201    }
202
203    // retrieve documents by sequence number
204    for (i=0; i < n+3; ++i){
205        // search by seq
206        fdb_doc_create(&rdoc, NULL, 0, NULL, 0, NULL, 0);
207        rdoc->seqnum = i + 1;
208        status = fdb_get_byseq(db, rdoc);
209        if ( (i>=2 && i<=4) || (i>=6 && i<=9) || (i>=11 && i<=12)) {
210            // updated documents
211            TEST_CHK(status == FDB_RESULT_SUCCESS);
212        } else {
213            // removed document
214            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
215        }
216
217        // free result document
218        fdb_doc_free(rdoc);
219        rdoc = NULL;
220    }
221
222    // update document #5 with an empty doc body.
223    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, doc[5]->meta,
224                   doc[5]->metalen, NULL, 0);
225    status = fdb_set(db, rdoc);
226    TEST_CHK(status == FDB_RESULT_SUCCESS);
227    fdb_doc_free(rdoc);
228    rdoc = NULL;
229    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
230
231    // Check document #5 with respect to metadata and doc body.
232    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, NULL, 0, NULL, 0);
233    status = fdb_get(db, rdoc);
234    TEST_CHK(status == FDB_RESULT_SUCCESS);
235    TEST_CHK(memcmp(rdoc->meta, doc[5]->meta, rdoc->metalen) == 0);
236    TEST_CHK(rdoc->body == NULL);
237    TEST_CHK(rdoc->bodylen == 0);
238    fdb_doc_free(rdoc);
239    rdoc = NULL;
240
241    // Read-Only mode test: Open succeeds if file exists, but disallow writes
242    fconfig.flags = FDB_OPEN_FLAG_RDONLY;
243    status = fdb_open(&dbfile_rdonly, "./dummy2", &fconfig);
244    TEST_CHK(status == FDB_RESULT_SUCCESS);
245    status = fdb_kvs_open_default(dbfile_rdonly, &db_rdonly, &kvs_config);
246    TEST_CHK(status == FDB_RESULT_SUCCESS);
247
248    fdb_doc_create(&rdoc, doc[0]->key, doc[0]->keylen, NULL, 0, NULL, 0);
249    status = fdb_get(db_rdonly, rdoc);
250    TEST_CHK(status == FDB_RESULT_SUCCESS);
251
252    status = fdb_set_log_callback(db_rdonly, logCallbackFunc,
253                                  (void *) "basic_test");
254    TEST_CHK(status == FDB_RESULT_SUCCESS);
255
256    status = fdb_set(db_rdonly, doc[i]);
257    TEST_CHK(status == FDB_RESULT_RONLY_VIOLATION);
258    TEST_CHK(!strcmp(fdb_error_msg(status), "database is read-only"));
259
260    status = fdb_commit(dbfile_rdonly, FDB_COMMIT_NORMAL);
261    TEST_CHK(status == FDB_RESULT_RONLY_VIOLATION);
262
263    fdb_doc_free(rdoc);
264    rdoc = NULL;
265    fdb_kvs_close(db_rdonly);
266    fdb_close(dbfile_rdonly);
267
268    // free all documents
269    for (i=0;i<n;++i){
270        fdb_doc_free(doc[i]);
271    }
272
273    // do one more compaction
274    fdb_compact(dbfile, (char *) "./dummy3");
275
276    // close db file
277    fdb_kvs_close(db);
278    fdb_close(dbfile);
279
280    // free all resources
281    fdb_shutdown();
282
283    memleak_end();
284
285    TEST_RESULT("basic test");
286}
287
288void init_test()
289{
290    TEST_INIT();
291    memleak_start();
292
293    int r;
294    fdb_status status;
295    fdb_file_handle *dbfile;
296    fdb_kvs_handle *db;
297    fdb_config fconfig = fdb_get_default_config();
298    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
299
300    r = system(SHELL_DEL" dummy* > errorlog.txt");
301    (void)r;
302
303    status = fdb_init(&fconfig);
304    TEST_CHK(status == FDB_RESULT_SUCCESS);
305
306    status = fdb_open(&dbfile, "./dummy1", &fconfig);
307    TEST_CHK(status == FDB_RESULT_SUCCESS);
308    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
309    TEST_CHK(status == FDB_RESULT_SUCCESS);
310
311    status = fdb_close(dbfile);
312    TEST_CHK(status == FDB_RESULT_SUCCESS);
313    status = fdb_shutdown();
314    TEST_CHK(status == FDB_RESULT_SUCCESS);
315
316    memleak_end();
317    TEST_RESULT("init test");
318}
319
320void set_get_max_keylen()
321{
322    TEST_INIT();
323    memleak_start();
324
325    int r;
326    static const int len = FDB_MAX_KEYLEN;
327    char keybuf[len];
328    void *rvalue;
329    size_t rvalue_len;
330    static const char *achar = "a";
331
332    fdb_status status;
333    fdb_file_handle *dbfile;
334    fdb_kvs_handle *db;
335    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
336    fdb_config fconfig = fdb_get_default_config();
337    fconfig.chunksize = 16;
338
339
340    r = system(SHELL_DEL" dummy* > errorlog.txt");
341    (void)r;
342
343    for (int i = 0; i < len; ++i) {
344        keybuf[i] = *achar;
345    }
346    keybuf[len-1] = '\0';
347
348    // open db
349    status = fdb_open(&dbfile, "./dummy1", &fconfig);
350    TEST_CHK(status == FDB_RESULT_SUCCESS);
351    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
352    TEST_CHK(status == FDB_RESULT_SUCCESS);
353
354    // set kv
355    status = fdb_set_kv(db, keybuf, strlen(keybuf), NULL, 0);
356    TEST_CHK(status == FDB_RESULT_SUCCESS);
357
358    // get NULL pointer
359    status = fdb_get(db, NULL);
360    TEST_CHK(status == FDB_RESULT_INVALID_ARGS);
361
362    // get kv
363    status = fdb_get_kv(db, keybuf, strlen(keybuf), &rvalue, &rvalue_len);
364    TEST_CHK(status == FDB_RESULT_SUCCESS);
365
366    fdb_close(dbfile);
367    fdb_shutdown();
368
369    memleak_end();
370    TEST_RESULT("set get max keylen");
371}
372
373void config_test()
374{
375    TEST_INIT();
376
377    memleak_start();
378
379    fdb_file_handle *dbfile;
380    fdb_kvs_handle *db;
381    fdb_status status;
382    fdb_config fconfig;
383    fdb_kvs_config kvs_config;
384    int nfiles = 4;
385    int i;
386    size_t bcache_space_used;
387    char fname[256];
388
389    // remove previous dummy test files
390    int r = system(SHELL_DEL" dummy* > errorlog.txt");
391    (void)r;
392
393    bcache_space_used = fdb_get_buffer_cache_used();
394    TEST_CHK(bcache_space_used == 0);
395
396    fconfig = fdb_get_default_config();
397    fconfig.buffercache_size= (uint64_t) -1;
398    status = fdb_open(&dbfile, "./dummy1", &fconfig);
399    TEST_CHK(status == FDB_RESULT_TOO_BIG_BUFFER_CACHE);
400
401    fconfig = fdb_get_default_config();
402    fconfig.max_writer_lock_prob = 120;
403    status = fdb_open(&dbfile, "./dummy1", &fconfig);
404    TEST_CHK(status == FDB_RESULT_INVALID_CONFIG);
405
406    fconfig = fdb_get_default_config();
407    kvs_config = fdb_get_default_kvs_config();
408    for (i = nfiles; i; --i) {
409        sprintf(fname, "dummy%d", i);
410        status = fdb_open(&dbfile, fname, &fconfig);
411        TEST_CHK(status == FDB_RESULT_SUCCESS);
412        status = fdb_kvs_open(dbfile, &db, "justonekv", &kvs_config);
413        TEST_CHK(status == FDB_RESULT_SUCCESS);
414
415        bcache_space_used = fdb_get_buffer_cache_used();
416
417        // Since V3 magic number, 7 blocks are used:
418        // 4 superblocks + KV name header + Stale-tree root node + DB header
419        TEST_CHK(bcache_space_used == fconfig.blocksize * 7);
420
421        status = fdb_close(dbfile);
422        TEST_CHK(status == FDB_RESULT_SUCCESS);
423    }
424
425    status = fdb_open(&dbfile, fname, &fconfig);
426    TEST_CHK(status == FDB_RESULT_SUCCESS);
427    status = fdb_kvs_open(dbfile, &db, "justonekv", &kvs_config);
428    TEST_CHK(status == FDB_RESULT_SUCCESS);
429    status = fdb_set_kv(db, (void*)"key", 3, (void*)"body", 5);
430    TEST_CHK(status == FDB_RESULT_SUCCESS);
431
432    bcache_space_used = fdb_get_buffer_cache_used();
433
434    // Since V3 magic number, 8 blocks are used:
435    // 7 blocks created eariler + document block for KV pair
436    TEST_CHK(bcache_space_used == fconfig.blocksize * 8);
437
438    fdb_close(dbfile);
439
440    fdb_shutdown();
441
442    memleak_end();
443    TEST_RESULT("forestdb config test");
444}
445
446void delete_reopen_test()
447{
448    TEST_INIT();
449    memleak_start();
450
451    int r;
452    fdb_file_handle *fh;
453    fdb_kvs_handle *db;
454    fdb_status status;
455    fdb_config fconfig;
456
457    r = system(SHELL_DEL " dummy* > errorlog.txt");
458    (void)r;
459
460    fconfig = fdb_get_default_config();
461    fconfig.buffercache_size = 0;
462    fconfig.num_compactor_threads = 1;
463    status = fdb_open(&fh, "./dummy3", &fconfig);
464    TEST_CHK(status == FDB_RESULT_SUCCESS);
465
466    status = fdb_kvs_open_default(fh, &db, NULL);
467    TEST_CHK(status == FDB_RESULT_SUCCESS);
468
469    status = fdb_begin_transaction(fh, FDB_ISOLATION_READ_COMMITTED);
470    TEST_CHK(status == FDB_RESULT_SUCCESS);
471
472    status = fdb_set_kv(db, (void *) "foo", 3, (void *)"value", 5);
473    TEST_CHK(status == FDB_RESULT_SUCCESS);
474
475    status = fdb_end_transaction(fh, FDB_COMMIT_NORMAL);
476    TEST_CHK(status == FDB_RESULT_SUCCESS);
477
478    void *value;
479    size_t valueSize;
480    status = fdb_get_kv(db, (void*)"foo", 3, &value, &valueSize);
481    TEST_CHK(status == FDB_RESULT_SUCCESS);
482
483    TEST_CHK(valueSize == 5);
484    TEST_CMP(value, "value", 5);
485    fdb_free_block(value);
486
487    status = fdb_begin_transaction(fh, FDB_ISOLATION_READ_COMMITTED);
488    TEST_CHK(status == FDB_RESULT_SUCCESS);
489
490    status = fdb_del_kv(db, "foo", 3);
491    TEST_CHK(status == FDB_RESULT_SUCCESS);
492
493    status = fdb_end_transaction(fh, FDB_COMMIT_NORMAL);
494    TEST_CHK(status == FDB_RESULT_SUCCESS);
495
496    status = fdb_get_kv(db, "foo", 3, &value, &valueSize);
497    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
498
499    status = fdb_close(fh);
500    TEST_CHK(status == FDB_RESULT_SUCCESS);
501
502    // Reopen:
503    status = fdb_open(&fh, "./dummy3", &fconfig);
504    TEST_CHK(status == FDB_RESULT_SUCCESS);
505
506    status = fdb_kvs_open_default(fh, &db, NULL);
507    TEST_CHK(status == FDB_RESULT_SUCCESS);
508
509    status = fdb_get_kv(db, "foo", 3, &value, &valueSize);
510    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
511
512    status = fdb_close(fh);
513    TEST_CHK(status == FDB_RESULT_SUCCESS);
514
515    fdb_shutdown();
516
517    memleak_end();
518    TEST_RESULT("end trans delete & reopen passed");
519}
520
521void deleted_doc_get_api_test()
522{
523    TEST_INIT();
524    memleak_start();
525
526    int r;
527    fdb_file_handle *dbfile;
528    fdb_kvs_handle *db;
529    fdb_doc _doc;
530    fdb_doc *doc = &_doc;
531    fdb_doc *rdoc;
532    fdb_status status;
533    fdb_config fconfig;
534    fdb_kvs_config kvs_config;
535    char keybuf[256], bodybuf[256];
536
537    r = system(SHELL_DEL " dummy* > errorlog.txt");
538    (void)r;
539
540    memset(doc, 0, sizeof(fdb_doc));
541    doc->key = &keybuf[0];
542    doc->body = &bodybuf[0];
543    doc->seqnum = SEQNUM_NOT_USED;
544
545    // open dbfile
546    fconfig = fdb_get_default_config();
547    fconfig.purging_interval = 1;
548    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
549    kvs_config = fdb_get_default_kvs_config();
550    status = fdb_open(&dbfile, "./dummy1", &fconfig);
551    TEST_CHK(status == FDB_RESULT_SUCCESS);
552    status = fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
553    TEST_CHK(status == FDB_RESULT_SUCCESS);
554
555    sprintf(keybuf, "key");
556    sprintf(bodybuf, "body");
557    doc->keylen = strlen(keybuf);
558    doc->bodylen = strlen(bodybuf);
559    status = fdb_set(db, doc);
560    TEST_CHK(status == FDB_RESULT_SUCCESS);
561
562    // Commit the doc so it goes into main index
563    status = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
564    TEST_CHK(status == FDB_RESULT_SUCCESS);
565
566    // Delete the doc
567    status = fdb_del(db, doc);
568    TEST_CHK(status == FDB_RESULT_SUCCESS);
569
570    // Commit the doc with wal flush so the delete is appended into the file
571    status = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
572    TEST_CHK(status == FDB_RESULT_SUCCESS);
573
574
575    fdb_doc_create(&rdoc, keybuf, doc->keylen, NULL, 0, NULL, 0);
576
577    // Deleted document should be accessible via fdb_get_metaonly()
578    status = fdb_get_metaonly(db, rdoc);
579    TEST_CHK(status == FDB_RESULT_SUCCESS);
580    TEST_CHK(rdoc->deleted);
581    rdoc->deleted = false;
582
583    // Deleted document should be accessible via fdb_get_metaonly_byseq()
584    status = fdb_get_metaonly_byseq(db, rdoc);
585    TEST_CHK(status == FDB_RESULT_SUCCESS);
586    TEST_CHK(rdoc->deleted);
587    rdoc->deleted = false;
588
589    // Deleted document should be accessible via fdb_get_byoffset()
590    // But the return code must be FDB_RESULT_KEY_NOT_FOUND!
591    status = fdb_get_byoffset(db, rdoc);
592    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
593    TEST_CHK(rdoc->deleted);
594    rdoc->deleted = false;
595
596    // Deleted document should NOT be accessible via fdb_get()
597    status = fdb_get(db, rdoc);
598    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
599    TEST_CHK(!rdoc->deleted);
600    rdoc->deleted = false;
601
602    status = fdb_get_byseq(db, rdoc);
603    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
604    TEST_CHK(!rdoc->deleted);
605
606    fdb_doc_free(rdoc);
607    // close without commit
608    status = fdb_kvs_close(db);
609    TEST_CHK(status == FDB_RESULT_SUCCESS);
610    status = fdb_close(dbfile);
611    TEST_CHK(status == FDB_RESULT_SUCCESS);
612    fdb_shutdown();
613    memleak_end();
614    TEST_RESULT("deleted doc get api test");
615}
616
617void deleted_doc_stat_test()
618{
619    TEST_INIT();
620    memleak_start();
621
622    int r;
623    fdb_file_handle *dbfile;
624    fdb_kvs_handle *db;
625    fdb_doc _doc;
626    fdb_doc *doc = &_doc;
627    fdb_doc *rdoc;
628    fdb_status status;
629    fdb_config fconfig;
630    fdb_file_info info;
631    fdb_kvs_config kvs_config;
632    char keybuf[256], bodybuf[256];
633
634    r = system(SHELL_DEL " dummy* > errorlog.txt");
635    (void)r;
636
637    memset(doc, 0, sizeof(fdb_doc));
638    doc->key = &keybuf[0];
639    doc->body = &bodybuf[0];
640    doc->seqnum = SEQNUM_NOT_USED;
641
642    // open dbfile
643    fconfig = fdb_get_default_config();
644    fconfig.purging_interval = 0;
645    kvs_config = fdb_get_default_kvs_config();
646    status = fdb_open(&dbfile, "./dummy1", &fconfig);
647    TEST_CHK(status == FDB_RESULT_SUCCESS);
648    status = fdb_kvs_open(dbfile, &db, "main", &kvs_config);
649    TEST_CHK(status == FDB_RESULT_SUCCESS);
650
651    sprintf(keybuf, "K"); // This is necessary to set keysize to 2 bytes so
652    sprintf(bodybuf, "body"); // it matches KV_header doc's keysize of 10
653    doc->keylen = strlen(keybuf) + 1; // in multi-kv mode and hits MB-16491
654    doc->bodylen = strlen(bodybuf) + 1;
655    status = fdb_set(db, doc);
656    TEST_CHK(status == FDB_RESULT_SUCCESS);
657
658    // Delete the doc
659    status = fdb_del(db, doc);
660    TEST_CHK(status == FDB_RESULT_SUCCESS);
661
662    // Fetch the doc back
663    fdb_doc_create(&rdoc, doc->key, doc->keylen, NULL, 0, NULL, 0);
664    status = fdb_get(db, rdoc);
665    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
666    fdb_doc_free(rdoc);
667
668    // check the file info
669    fdb_get_file_info(dbfile, &info);
670    TEST_CHK(info.doc_count == 0);
671
672    status = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
673    TEST_CHK(status == FDB_RESULT_SUCCESS);
674    // check the file info again after commit..
675    fdb_get_file_info(dbfile, &info);
676    TEST_CHK(info.doc_count == 0);
677
678    status = fdb_kvs_close(db);
679    TEST_CHK(status == FDB_RESULT_SUCCESS);
680    status = fdb_close(dbfile);
681    TEST_CHK(status == FDB_RESULT_SUCCESS);
682    fdb_shutdown();
683    memleak_end();
684    TEST_RESULT("deleted doc stat test");
685}
686
687// MB-16312
688void complete_delete_test()
689{
690    TEST_INIT();
691
692    int i, r, n = 1000;
693    fdb_file_handle *dbfile;
694    fdb_kvs_handle *db;
695    fdb_iterator *fit;
696    fdb_config config;
697    fdb_kvs_config kvs_config;
698    fdb_status s; (void)s;
699    char path[256];
700    char keybuf[256], valuebuf[256];
701
702    memleak_start();
703
704    sprintf(path, "./dummy1");
705
706    r = system(SHELL_DEL " dummy* > errorlog.txt");
707    (void)r;
708
709    config = fdb_get_default_config();
710    config.buffercache_size = 0;
711    kvs_config = fdb_get_default_kvs_config();
712
713    fdb_open(&dbfile, path, &config);
714    s = fdb_kvs_open(dbfile, &db, "db1", &kvs_config);
715    TEST_CHK(s == FDB_RESULT_SUCCESS);
716
717    for (i=0;i<n;++i){
718        sprintf(keybuf, "key%05d", i);
719        sprintf(valuebuf, "value%05d", i);
720        s = fdb_set_kv(db, keybuf, strlen(keybuf)+1, valuebuf, strlen(valuebuf)+1);
721        TEST_CHK(s == FDB_RESULT_SUCCESS);
722    }
723    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
724
725    for (i=0;i<n;++i){
726        sprintf(keybuf, "key%05d", i);
727        sprintf(valuebuf, "value%05d", i);
728        s = fdb_del_kv(db, keybuf, strlen(keybuf)+1);
729        TEST_CHK(s == FDB_RESULT_SUCCESS);
730    }
731    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
732
733    s = fdb_iterator_init(db, &fit, NULL, 0, NULL, 0, FDB_ITR_NO_DELETES);
734    TEST_CHK(s == FDB_RESULT_SUCCESS);
735    s = fdb_iterator_close(fit);
736    TEST_CHK(s == FDB_RESULT_SUCCESS);
737
738    fdb_close(dbfile);
739    fdb_shutdown();
740
741    memleak_end();
742    TEST_RESULT("complete delete");
743}
744
745void large_batch_write_no_commit_test()
746{
747    TEST_INIT();
748    memleak_start();
749
750    int i, r;
751    int n = 500000;
752    fdb_file_handle *dbfile;
753    fdb_kvs_handle *db;
754    fdb_doc **doc = (fdb_doc **) malloc(sizeof(fdb_doc *) * n);
755    fdb_status status;
756    fdb_config fconfig;
757    fdb_kvs_config kvs_config;
758    char keybuf[256], metabuf[256], bodybuf[256];
759
760    r = system(SHELL_DEL " dummy* > errorlog.txt");
761    (void)r;
762
763    // open dbfile
764    fconfig = fdb_get_default_config();
765    kvs_config = fdb_get_default_kvs_config();
766    status = fdb_open(&dbfile, "./dummy1", &fconfig);
767    TEST_CHK(status == FDB_RESULT_SUCCESS);
768    status = fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
769    TEST_CHK(status == FDB_RESULT_SUCCESS);
770
771    // Write 500K docs to eject and flush some dirty pages into disk.
772    for (i=0;i<n;++i){
773        sprintf(keybuf, "key%128d", i);
774        sprintf(metabuf, "meta%128d", i);
775        sprintf(bodybuf, "body%128d", i);
776        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
777            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
778        fdb_set(db, doc[i]);
779        fdb_doc_free(doc[i]);
780    }
781
782    // close without commit
783    status = fdb_kvs_close(db);
784    TEST_CHK(status == FDB_RESULT_SUCCESS);
785    status = fdb_close(dbfile);
786    TEST_CHK(status == FDB_RESULT_SUCCESS);
787
788    status = fdb_open(&dbfile, "./dummy1", &fconfig);
789    TEST_CHK(status == FDB_RESULT_NO_DB_HEADERS ||
790             status == FDB_RESULT_SUCCESS); // No dirty pages are flushed into disk.
791    if (status == FDB_RESULT_SUCCESS) {
792        status = fdb_close(dbfile);
793        TEST_CHK(status == FDB_RESULT_SUCCESS);
794    }
795
796    free(doc);
797    fdb_shutdown();
798    memleak_end();
799    TEST_RESULT("large batch write test with no commits");
800}
801
802void set_get_meta_test()
803{
804    TEST_INIT();
805    memleak_start();
806
807    int r;
808    char keybuf[256];
809    fdb_file_handle *dbfile;
810    fdb_kvs_handle *db;
811    fdb_doc *rdoc;
812    fdb_status status;
813    fdb_config fconfig = fdb_get_default_config();
814    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
815    fconfig.wal_threshold = 1024;
816    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
817    fconfig.purging_interval = 1;
818    fconfig.flags = FDB_OPEN_FLAG_CREATE;
819
820    // remove previous dummy files
821    r = system(SHELL_DEL" dummy* > errorlog.txt");
822    (void)r;
823
824    // open db
825    fdb_open(&dbfile, "./dummy1", &fconfig);
826    fdb_kvs_open(dbfile, &db, "db1", &kvs_config);
827
828    sprintf(keybuf, "key%d", 0);
829    fdb_doc_create(&rdoc, keybuf, strlen(keybuf), NULL, 0, NULL, 0);
830    fdb_set(db, rdoc);
831    status = fdb_get(db, rdoc);
832    assert(status == FDB_RESULT_SUCCESS);
833    status = fdb_get_byoffset(db, rdoc);
834    TEST_CHK(status == FDB_RESULT_SUCCESS);
835    status = fdb_get_metaonly(db, rdoc);
836    assert(status == FDB_RESULT_SUCCESS);
837    status = fdb_get_metaonly_byseq(db, rdoc);
838    TEST_CHK(status == FDB_RESULT_SUCCESS);
839
840    fdb_del(db, rdoc);
841    status = fdb_get(db, rdoc);
842    assert(status == FDB_RESULT_KEY_NOT_FOUND);
843    assert(rdoc->deleted == true);
844
845    status = fdb_get_metaonly(db, rdoc);
846    assert(status == FDB_RESULT_SUCCESS);
847    assert(rdoc->deleted == true);
848
849    status = fdb_get_metaonly_byseq(db, rdoc);
850    TEST_CHK(status == FDB_RESULT_SUCCESS);
851    assert(rdoc->deleted == true);
852
853    status = fdb_get_byoffset(db, rdoc);
854    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
855    assert(rdoc->deleted == true);
856
857
858    fdb_doc_free(rdoc);
859    fdb_kvs_close(db);
860    fdb_close(dbfile);
861    fdb_shutdown();
862
863    memleak_end();
864    TEST_RESULT("set get meta test");
865}
866
867void long_filename_test()
868{
869    TEST_INIT();
870    memleak_start();
871
872    int i, j, r;
873    int n=15, m=1000;
874    char keyword[] = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
875    char filename[4096], cmd[4096], temp[4096];
876    fdb_file_handle *dbfile;
877    fdb_kvs_handle *db;
878    fdb_config config;
879    fdb_kvs_config kvs_config;
880    fdb_status s;
881    size_t rvalue_len;
882    char key[256], value[256];
883    void *rvalue;
884
885    config = fdb_get_default_config();
886    kvs_config = fdb_get_default_kvs_config();
887    sprintf(temp, SHELL_DMT"%s", keyword);
888
889    // filename longer than 1024 bytes
890    sprintf(filename, "%s", keyword);
891    while (strlen(filename) < 1024) {
892        strcat(filename, keyword);
893    }
894    s = fdb_open(&dbfile, filename, &config);
895    TEST_CHK(s == FDB_RESULT_TOO_LONG_FILENAME);
896
897    // make nested directories for long path
898    // but shorter than 1024 bytes (windows: 256 bytes)
899    sprintf(cmd, SHELL_RMDIR" %s", keyword);
900    r = system(cmd);
901    (void)r;
902    for (i=0;i<n;++i) {
903        sprintf(cmd, SHELL_MKDIR" %s", keyword);
904        for (j=0;j<i;++j){
905            strcat(cmd, temp);
906        }
907        if (strlen(cmd) > SHELL_MAX_PATHLEN) break;
908        r = system(cmd);
909        (void)r;
910    }
911
912    // create DB file
913    sprintf(filename, "%s", keyword);
914    for (j=0;j<i-1;++j){
915        strcat(filename, temp);
916    }
917    strcat(filename, SHELL_DMT"dbfile");
918    s = fdb_open(&dbfile, filename, &config);
919    TEST_CHK(s == FDB_RESULT_SUCCESS);
920    s = fdb_kvs_open_default(dbfile, &db, &kvs_config);
921    TEST_CHK(s == FDB_RESULT_SUCCESS);
922
923    // === write ===
924    for (i=0;i<m;++i){
925        sprintf(key, "key%08d", i);
926        sprintf(value, "value%08d", i);
927        s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
928        TEST_CHK(s == FDB_RESULT_SUCCESS);
929    }
930    s = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
931    TEST_CHK(s == FDB_RESULT_SUCCESS);
932
933    // === read ===
934    for (i=0;i<m;++i){
935        sprintf(key, "key%08d", i);
936        s = fdb_get_kv(db, key, strlen(key)+1, &rvalue, &rvalue_len);
937        TEST_CHK(s == FDB_RESULT_SUCCESS);
938        fdb_free_block(rvalue);
939    }
940
941    s = fdb_kvs_close(db);
942    TEST_CHK(s == FDB_RESULT_SUCCESS);
943    s = fdb_close(dbfile);
944    TEST_CHK(s == FDB_RESULT_SUCCESS);
945    s = fdb_shutdown();
946    TEST_CHK(s == FDB_RESULT_SUCCESS);
947
948    sprintf(cmd, SHELL_RMDIR" %s", keyword);
949    r = system(cmd);
950    (void)r;
951
952    memleak_end();
953    TEST_RESULT("long filename test");
954}
955
956void error_to_str_test()
957{
958    TEST_INIT();
959    memleak_start();
960    int i;
961    const char *err_msg;
962
963    for (i = FDB_RESULT_SUCCESS; i >= FDB_RESULT_LAST; --i) {
964        err_msg = fdb_error_msg((fdb_status)i);
965        // Verify that all error codes have corresponding error messages
966        TEST_CHK(strcmp(err_msg, "unknown error"));
967    }
968
969    err_msg = fdb_error_msg((fdb_status)i);
970    // Verify that the last error code has been checked
971    TEST_CHK(!strcmp(err_msg, "unknown error"));
972
973    memleak_end();
974    TEST_RESULT("error to string message test");
975}
976
977void seq_tree_exception_test()
978{
979    TEST_INIT();
980
981    memleak_start();
982
983    int i, r;
984    int n = 10;
985    fdb_file_handle *dbfile;
986    fdb_kvs_handle *db;
987    fdb_doc **doc = alca(fdb_doc*, n);
988    fdb_doc *rdoc = NULL;
989    fdb_status status;
990    fdb_iterator *it;
991
992    char keybuf[256], metabuf[256], bodybuf[256];
993
994    // remove previous dummy files
995    r = system(SHELL_DEL" dummy* > errorlog.txt");
996    (void)r;
997
998    fdb_config fconfig = fdb_get_default_config();
999    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1000    fconfig.seqtree_opt = FDB_SEQTREE_NOT_USE;
1001
1002    // open db
1003    fdb_open(&dbfile, "./dummy1", &fconfig);
1004    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1005    status = fdb_set_log_callback(db, logCallbackFunc,
1006                                  (void *) "seq_tree_exception_test");
1007    TEST_CHK(status == FDB_RESULT_SUCCESS);
1008
1009    // insert documents
1010    for (i=0;i<n;++i){
1011        sprintf(keybuf, "key%d", i);
1012        sprintf(metabuf, "meta%d", i);
1013        sprintf(bodybuf, "body%d", i);
1014        fdb_doc_create(&doc[i],
1015                       (void *)keybuf,  strlen(keybuf),
1016                       (void *)metabuf, strlen(metabuf),
1017                       (void *)bodybuf, strlen(bodybuf));
1018        fdb_set(db, doc[i]);
1019    }
1020
1021    // commit
1022    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1023
1024    status = fdb_compact(dbfile, NULL);
1025    TEST_CHK(status == FDB_RESULT_SUCCESS);
1026
1027    // close the db
1028    fdb_kvs_close(db);
1029    fdb_close(dbfile);
1030
1031    // reopen with seq tree option
1032    fconfig.seqtree_opt = FDB_SEQTREE_USE;
1033    status = fdb_open(&dbfile, "./dummy1", &fconfig);
1034    // must succeed
1035    TEST_CHK(status == FDB_RESULT_SUCCESS);
1036    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
1037    TEST_CHK(status == FDB_RESULT_SUCCESS);
1038    status = fdb_set_log_callback(db, logCallbackFunc,
1039                                  (void *) "seq_tree_exception_test");
1040    TEST_CHK(status == FDB_RESULT_SUCCESS);
1041
1042    // search by seq
1043    fdb_doc_create(&rdoc, NULL, 0, NULL, 0, NULL, 0);
1044    rdoc->seqnum = 1;
1045    status = fdb_get_byseq(db, rdoc);
1046    // must fail
1047    TEST_CHK(status != FDB_RESULT_SUCCESS);
1048
1049    // search meta by seq
1050    status = fdb_get_metaonly_byseq(db, rdoc);
1051    // must fail
1052    TEST_CHK(status != FDB_RESULT_SUCCESS);
1053
1054    // init iterator by seq
1055    status = fdb_iterator_sequence_init(db , &it, 0, 0, FDB_ITR_NONE);
1056    // must fail
1057    TEST_CHK(status != FDB_RESULT_SUCCESS);
1058
1059    // close db file
1060    fdb_kvs_close(db);
1061    fdb_close(dbfile);
1062
1063    // free all documents
1064    free(rdoc);
1065    for (i=0;i<n;++i){
1066        fdb_doc_free(doc[i]);
1067    }
1068
1069    // remove previous dummy files
1070    r = system(SHELL_DEL" dummy* > errorlog.txt");
1071    (void)r;
1072
1073    // open db
1074    fconfig.seqtree_opt = FDB_SEQTREE_USE;
1075    fdb_open(&dbfile, "./dummy1", &fconfig);
1076    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1077    status = fdb_set_log_callback(db, logCallbackFunc,
1078                                  (void *) "seq_tree_exception_test");
1079    TEST_CHK(status == FDB_RESULT_SUCCESS);
1080
1081    // insert documents
1082    for (i=0;i<n;++i){
1083        sprintf(keybuf, "key%d", i);
1084        sprintf(metabuf, "meta%d", i);
1085        sprintf(bodybuf, "body%d", i);
1086        fdb_doc_create(&doc[i],
1087                       (void *)keybuf,  strlen(keybuf),
1088                       (void *)metabuf, strlen(metabuf),
1089                       (void *)bodybuf, strlen(bodybuf));
1090        fdb_set(db, doc[i]);
1091    }
1092
1093    // commit
1094    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1095
1096    // close the db
1097    fdb_kvs_close(db);
1098    fdb_close(dbfile);
1099
1100    // reopen with an option disabling seq tree
1101    fconfig.seqtree_opt = FDB_SEQTREE_NOT_USE;
1102    status = fdb_open(&dbfile, "./dummy1", &fconfig);
1103    // must succeed
1104    TEST_CHK(status == FDB_RESULT_SUCCESS);
1105    status = fdb_close(dbfile);
1106    TEST_CHK(status == FDB_RESULT_SUCCESS);
1107
1108    // free all documents
1109    for (i=0;i<n;++i){
1110        fdb_doc_free(doc[i]);
1111    }
1112
1113    // free all resources
1114    fdb_shutdown();
1115
1116    memleak_end();
1117
1118    TEST_RESULT("sequence tree exception test");
1119}
1120
1121void wal_commit_test()
1122{
1123    TEST_INIT();
1124
1125    memleak_start();
1126
1127    int i, r;
1128    int n = 10;
1129    fdb_file_handle *dbfile;
1130    fdb_kvs_handle *db;
1131    fdb_doc **doc = alca(fdb_doc*, n);
1132    fdb_doc *rdoc = NULL;
1133    fdb_status status;
1134
1135    char keybuf[256], metabuf[256], bodybuf[256];
1136
1137    // remove previous dummy files
1138    r = system(SHELL_DEL" dummy* > errorlog.txt");
1139    (void)r;
1140
1141    fdb_config fconfig = fdb_get_default_config();
1142    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1143    fconfig.buffercache_size = 0;
1144    fconfig.wal_threshold = 1024;
1145    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1146    fconfig.compaction_threshold = 0;
1147
1148    // open db
1149    fdb_open(&dbfile, "./dummy1", &fconfig);
1150    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1151    status = fdb_set_log_callback(db, logCallbackFunc, (void *) "wal_commit_test");
1152    TEST_CHK(status == FDB_RESULT_SUCCESS);
1153
1154    // insert half documents
1155    for (i=0;i<n/2;++i){
1156        sprintf(keybuf, "key%d", i);
1157        sprintf(metabuf, "meta%d", i);
1158        sprintf(bodybuf, "body%d", i);
1159        fdb_doc_create(&doc[i], (void *)keybuf, strlen(keybuf),
1160            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1161        fdb_set(db, doc[i]);
1162    }
1163
1164    // commit
1165    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1166
1167    // insert the other half documents
1168    for (i=n/2;i<n;++i){
1169        sprintf(keybuf, "key%d", i);
1170        sprintf(metabuf, "meta%d", i);
1171        sprintf(bodybuf, "body%d", i);
1172        fdb_doc_create(&doc[i], (void *)keybuf, strlen(keybuf),
1173            (void *)metabuf, strlen(metabuf), (void *)bodybuf, strlen(bodybuf));
1174        fdb_set(db, doc[i]);
1175    }
1176
1177    // close the db
1178    fdb_kvs_close(db);
1179    fdb_close(dbfile);
1180
1181    // reopen
1182    fdb_open(&dbfile, "./dummy1", &fconfig);
1183    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1184    status = fdb_set_log_callback(db, logCallbackFunc, (void *) "wal_commit_test");
1185    TEST_CHK(status == FDB_RESULT_SUCCESS);
1186
1187    // retrieve documents
1188    for (i=0;i<n;++i){
1189        // search by key
1190        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
1191        status = fdb_get(db, rdoc);
1192
1193        if (i < n/2) {
1194            // committed documents
1195            TEST_CHK(status == FDB_RESULT_SUCCESS);
1196            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
1197            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
1198        } else {
1199            // not committed document
1200            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
1201        }
1202
1203        // free result document
1204        fdb_doc_free(rdoc);
1205        rdoc = NULL;
1206    }
1207
1208    // free all documents
1209    for (i=0;i<n;++i){
1210        fdb_doc_free(doc[i]);
1211    }
1212
1213    // close db file
1214    fdb_kvs_close(db);
1215    fdb_close(dbfile);
1216
1217    // free all resources
1218    fdb_shutdown();
1219
1220    memleak_end();
1221
1222    TEST_RESULT("WAL commit test");
1223}
1224
1225void db_close_and_remove()
1226{
1227
1228    TEST_INIT();
1229    memleak_start();
1230
1231    int i, r;
1232    int n = 10;
1233    fdb_file_handle *dbfile;
1234    fdb_kvs_handle *db;
1235    fdb_doc **doc = alca(fdb_doc *, n);
1236    fdb_status status;
1237    fdb_config fconfig;
1238    fdb_kvs_config kvs_config;
1239    char keybuf[256], metabuf[256], bodybuf[256];
1240
1241    r = system(SHELL_DEL " dummy* > errorlog.txt");
1242    (void)r;
1243
1244    // open dbfile
1245    fconfig = fdb_get_default_config();
1246    kvs_config = fdb_get_default_kvs_config();
1247    fconfig.cleanup_cache_onclose = false;
1248    fdb_open(&dbfile, "./dummy1", &fconfig);
1249    fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
1250
1251    // write to db
1252    for (i=0;i<n;++i){
1253        sprintf(keybuf, "key%d", i);
1254        sprintf(metabuf, "meta%d", i);
1255        sprintf(bodybuf, "body%d", i);
1256        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1257            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1258        fdb_set(db, doc[i]);
1259        fdb_doc_free(doc[i]);
1260    }
1261    status = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1262    TEST_CHK(status == FDB_RESULT_SUCCESS);
1263
1264    // close
1265    fdb_kvs_close(db);
1266    fdb_close(dbfile);
1267
1268    // remove dbfile
1269    r = system(SHELL_DEL " dummy* > errorlog.txt");
1270    (void)r;
1271
1272    // re-open read-only
1273    fconfig.flags = FDB_OPEN_FLAG_RDONLY;
1274    status = fdb_open(&dbfile, "./dummy1", &fconfig);
1275    TEST_CHK(status == FDB_RESULT_NO_SUCH_FILE);
1276
1277    fdb_shutdown();
1278    memleak_end();
1279    TEST_RESULT("db close and remove");
1280}
1281
1282void db_drop_test()
1283{
1284    TEST_INIT();
1285
1286    memleak_start();
1287
1288    int i, r;
1289    int n = 3;
1290    fdb_file_handle *dbfile;
1291    fdb_kvs_handle *db;
1292    fdb_doc **doc = alca(fdb_doc *, n);
1293    fdb_doc *rdoc = NULL;
1294    fdb_status status;
1295
1296    char keybuf[256], metabuf[256], bodybuf[256];
1297
1298    // remove previous dummy files
1299    r = system(SHELL_DEL " dummy* > errorlog.txt");
1300    (void)r;
1301
1302    fdb_config fconfig = fdb_get_default_config();
1303    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1304    fconfig.buffercache_size = 16777216;
1305    fconfig.wal_threshold = 1024;
1306    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1307    fconfig.compaction_threshold = 0;
1308
1309    // open db
1310    fdb_open(&dbfile, "./dummy1", &fconfig);
1311    fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
1312    status = fdb_set_log_callback(db, logCallbackFunc,
1313                                  (void *) "db_drop_test");
1314    TEST_CHK(status == FDB_RESULT_SUCCESS);
1315
1316    // insert first two documents
1317    for (i=0;i<2;++i){
1318        sprintf(keybuf, "key%d", i);
1319        sprintf(metabuf, "meta%d", i);
1320        sprintf(bodybuf, "body%d", i);
1321        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1322            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1323        fdb_set(db, doc[i]);
1324    }
1325
1326    // commit
1327    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1328    fdb_kvs_close(db);
1329    fdb_close(dbfile);
1330
1331    // Remove the database file manually.
1332    r = system(SHELL_DEL " dummy1 > errorlog.txt");
1333    (void)r;
1334
1335    // Open the empty db with the same name.
1336    fdb_open(&dbfile, "./dummy1", &fconfig);
1337    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1338    status = fdb_set_log_callback(db, logCallbackFunc,
1339                                  (void *) "db_drop_test");
1340    TEST_CHK(status == FDB_RESULT_SUCCESS);
1341
1342    // now insert a new doc.
1343    sprintf(keybuf, "key%d", 0);
1344    sprintf(metabuf, "meta%d", 0);
1345    sprintf(bodybuf, "body%d", 0);
1346    fdb_doc_free(doc[0]);
1347    fdb_doc_create(&doc[0], (void*)keybuf, strlen(keybuf),
1348        (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1349    fdb_set(db, doc[0]);
1350
1351    // commit
1352    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1353
1354    // search by key
1355    fdb_doc_create(&rdoc, doc[0]->key, doc[0]->keylen, NULL, 0, NULL, 0);
1356    status = fdb_get(db, rdoc);
1357    TEST_CHK(status == FDB_RESULT_SUCCESS);
1358    // Make sure that a doc seqnum starts with one.
1359    TEST_CHK(rdoc->seqnum == 1);
1360
1361    fdb_kvs_close(db);
1362    fdb_close(dbfile);
1363
1364    // free all documents
1365    fdb_doc_free(rdoc);
1366    rdoc = NULL;
1367    for (i=0;i<2;++i){
1368        fdb_doc_free(doc[i]);
1369    }
1370
1371    // free all resources
1372    fdb_shutdown();
1373
1374    memleak_end();
1375
1376    TEST_RESULT("Database drop test");
1377}
1378
1379void db_destroy_test()
1380{
1381    TEST_INIT();
1382
1383    memleak_start();
1384
1385    int i, r;
1386    int n = 30;
1387    fdb_file_handle *dbfile, *dbfile2;
1388    fdb_kvs_handle *db, *db2;
1389    fdb_doc **doc = alca(fdb_doc *, n);
1390    fdb_doc *rdoc = NULL;
1391    fdb_status status;
1392    fdb_config fconfig;
1393    fdb_kvs_config kvs_config;
1394
1395    char keybuf[256], metabuf[256], bodybuf[256];
1396
1397    // remove previous dummy files
1398    r = system(SHELL_DEL " dummy* > errorlog.txt");
1399    (void)r;
1400
1401    fconfig = fdb_get_default_config();
1402    kvs_config = fdb_get_default_kvs_config();
1403    fconfig.buffercache_size = 16777216;
1404    fconfig.wal_threshold = 1024;
1405    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1406    fconfig.compaction_threshold = 0;
1407
1408    // open db
1409    fdb_open(&dbfile, "./dummy1", &fconfig);
1410    fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
1411    status = fdb_set_log_callback(db, logCallbackFunc,
1412                                  (void *) "db_destroy_test");
1413    TEST_CHK(status == FDB_RESULT_SUCCESS);
1414
1415    // insert 30 documents
1416    for (i=0;i<n;++i){
1417        sprintf(keybuf, "key%d", i);
1418        sprintf(metabuf, "meta%d", i);
1419        sprintf(bodybuf, "body%d", i);
1420        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1421            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1422        fdb_set(db, doc[i]);
1423    }
1424
1425    // commit
1426    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1427
1428    // Open the empty db with the same name.
1429    fdb_open(&dbfile2, "./dummy2", &fconfig);
1430    fdb_kvs_open(dbfile2, &db2, NULL, &kvs_config);
1431    status = fdb_set_log_callback(db2, logCallbackFunc,
1432                                  (void *) "db_destroy_test");
1433    TEST_CHK(status == FDB_RESULT_SUCCESS);
1434    // insert 30 documents
1435    for (i=0;i<n;++i){
1436        fdb_set(db2, doc[i]);
1437    }
1438
1439    // commit
1440    fdb_commit(dbfile2, FDB_COMMIT_NORMAL);
1441
1442    // Only close db not db2 and try to destroy
1443    fdb_close(dbfile);
1444
1445    status = fdb_destroy("./dummy2", &fconfig);
1446    TEST_CHK(status == FDB_RESULT_FILE_IS_BUSY);
1447
1448    //Now close the open db file
1449    fdb_close(dbfile2);
1450
1451    status = fdb_destroy("./dummy1", &fconfig);
1452    TEST_CHK(status == FDB_RESULT_SUCCESS);
1453
1454    // Open the same db with the same names.
1455    fdb_open(&dbfile, "./dummy1", &fconfig);
1456    fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
1457    status = fdb_set_log_callback(db, logCallbackFunc,
1458                                  (void *) "db_destroy_test");
1459    TEST_CHK(status == FDB_RESULT_SUCCESS);
1460
1461    // search by key
1462    fdb_doc_create(&rdoc, doc[0]->key, doc[0]->keylen, NULL, 0, NULL, 0);
1463    status = fdb_get(db, rdoc);
1464    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
1465    fdb_close(dbfile);
1466
1467    // free all documents
1468    fdb_doc_free(rdoc);
1469    rdoc = NULL;
1470    for (i=0;i<n;++i){
1471        fdb_doc_free(doc[i]);
1472    }
1473
1474    // free all resources
1475    fdb_shutdown();
1476
1477    memleak_end();
1478
1479    TEST_RESULT("Database destroy test");
1480}
1481
1482// Test for MB-16348
1483void db_destroy_test_full_path()
1484{
1485    TEST_INIT();
1486
1487    memleak_start();
1488    randomize();
1489
1490    int r;
1491    fdb_file_handle *dbfile;
1492    fdb_config config;
1493    fdb_status s;
1494    char path[256];
1495    char cmd[256];
1496
1497    sprintf(path, "/tmp/fdb_destroy_test_%d", random(10000));
1498
1499    sprintf(cmd, "rm -rf %s*", path);
1500    r = system(cmd); (void)r;
1501
1502    config = fdb_get_default_config();
1503    config.compaction_mode = FDB_COMPACTION_AUTO;
1504
1505    fdb_open(&dbfile, path, &config);
1506    fdb_close(dbfile);
1507
1508    s = fdb_destroy(path, &config);
1509    TEST_CHK(s == FDB_RESULT_SUCCESS);
1510
1511    fdb_shutdown();
1512
1513    memleak_end();
1514
1515    TEST_RESULT("Database destroy (full path) test");
1516}
1517
1518void operational_stats_test(bool multi_kv)
1519{
1520    TEST_INIT();
1521
1522    memleak_start();
1523
1524    int i, r;
1525    int n = 10;
1526    int num_kv = 4;
1527    fdb_file_handle *dbfile;
1528    fdb_kvs_handle **db = alca(fdb_kvs_handle*, num_kv);
1529    fdb_doc **doc = alca(fdb_doc*, n);
1530    fdb_doc real_doc;
1531    fdb_doc *rdoc = &real_doc;
1532    fdb_status status;
1533    fdb_iterator *iterator;
1534    fdb_kvs_ops_info info, rinfo;
1535
1536    char keybuf[256], bodybuf[256];
1537    memset(&info, 0, sizeof(fdb_kvs_ops_info));
1538    memset(&real_doc, 0, sizeof(fdb_doc));
1539    real_doc.key = &keybuf;
1540    real_doc.body = &bodybuf;
1541
1542    // remove previous dummy files
1543    r = system(SHELL_DEL" dummy* > errorlog.txt");
1544    (void)r;
1545
1546    fdb_config fconfig = fdb_get_default_config();
1547    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1548
1549    fconfig.buffercache_size = 0;
1550    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
1551    fconfig.wal_threshold = 1024;
1552    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1553    fconfig.compaction_threshold = 0;
1554    fconfig.multi_kv_instances = multi_kv;
1555    r = 0;
1556
1557    fdb_open(&dbfile, "./dummy1", &fconfig);
1558    if (multi_kv) {
1559        num_kv = 4;
1560        for (r = num_kv - 1; r >= 0; --r) {
1561            char tmp[16];
1562            sprintf(tmp, "kv%d", r);
1563            status = fdb_kvs_open(dbfile, &db[r], tmp, &kvs_config);
1564            TEST_CHK(status == FDB_RESULT_SUCCESS);
1565            status = fdb_set_log_callback(db[r], logCallbackFunc,
1566                                          (void *) "operational_stats_test");
1567            TEST_CHK(status == FDB_RESULT_SUCCESS);
1568        }
1569    } else {
1570        num_kv = 1;
1571        status = fdb_kvs_open_default(dbfile, &db[r], &kvs_config);
1572        TEST_CHK(status == FDB_RESULT_SUCCESS);
1573        status = fdb_set_log_callback(db[r], logCallbackFunc,
1574                (void *) "operational_stats_test");
1575        TEST_CHK(status == FDB_RESULT_SUCCESS);
1576    }
1577
1578    for (i = 0; i < n; ++i){
1579        sprintf(keybuf, "key%d", i);
1580        sprintf(bodybuf, "body%d", i);
1581        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf) + 1, NULL, 0,
1582            (void*)bodybuf, strlen(bodybuf)+1);
1583        for (r = num_kv - 1; r >= 0; --r) {
1584            status = fdb_set(db[r], doc[i]);
1585            TEST_CHK(status == FDB_RESULT_SUCCESS);
1586            status = fdb_get_kvs_ops_info(db[r], &rinfo);
1587            TEST_CHK(status == FDB_RESULT_SUCCESS);
1588            info.num_sets = i + 1;
1589            TEST_CMP(&rinfo, &info, sizeof(fdb_kvs_ops_info));
1590        }
1591    }
1592
1593    for (r = num_kv - 1; r >= 0; --r) {
1594        // range scan (before flushing WAL)
1595        fdb_iterator_init(db[r], &iterator, NULL, 0, NULL, 0, 0x0);
1596        i = 0;
1597        do {
1598            status = fdb_iterator_get(iterator, &rdoc);
1599            TEST_CHK(status == FDB_RESULT_SUCCESS);
1600            TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
1601            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
1602            status = fdb_get_kvs_ops_info(db[r], &rinfo);
1603            TEST_CHK(status == FDB_RESULT_SUCCESS);
1604            ++info.num_iterator_gets;
1605            ++info.num_iterator_moves;
1606            TEST_CMP(&rinfo, &info, sizeof(fdb_kvs_ops_info));
1607            ++i;
1608        } while(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
1609        ++info.num_iterator_moves; // account for the last move that failed
1610        fdb_iterator_close(iterator);
1611
1612        fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1613        ++info.num_commits;
1614
1615        status = fdb_get_kvs_ops_info(db[r], &rinfo);
1616        TEST_CHK(status == FDB_RESULT_SUCCESS);
1617        TEST_CMP(&rinfo, &info, sizeof(fdb_kvs_ops_info));
1618
1619        if (r) {
1620            info.num_iterator_gets = 0;
1621            info.num_iterator_moves = 0;
1622        }
1623    }
1624
1625    ++info.num_compacts;
1626    // do compaction
1627    fdb_compact(dbfile, (char *) "./dummy2");
1628
1629    status = fdb_get_kvs_ops_info(db[0], &rinfo);
1630    TEST_CHK(status == FDB_RESULT_SUCCESS);
1631    TEST_CMP(&rinfo, &info, sizeof(fdb_kvs_ops_info));
1632
1633    for (i = 0; i < n; ++i){
1634        sprintf(keybuf, "key%d", i);
1635        for (r = num_kv - 1; r >= 0; --r) {
1636            if (i % 2 == 0) {
1637                if (i % 4 == 0) {
1638                    status = fdb_get_metaonly(db[r], rdoc);
1639                } else {
1640                    rdoc->seqnum = i + 1;
1641                    status = fdb_get_byseq(db[r], rdoc);
1642                }
1643            } else {
1644                status = fdb_get(db[r], rdoc);
1645            }
1646            TEST_CHK(status == FDB_RESULT_SUCCESS);
1647            status = fdb_get_kvs_ops_info(db[r], &rinfo);
1648            TEST_CHK(status == FDB_RESULT_SUCCESS);
1649            info.num_gets = i + 1;
1650            TEST_CMP(&rinfo, &info, sizeof(fdb_kvs_ops_info));
1651        }
1652    }
1653    // also get latency stats..
1654    for (int i = 0; i < FDB_LATENCY_NUM_STATS; ++i) {
1655        fdb_latency_stat stat;
1656        memset(&stat, 0, sizeof(fdb_latency_stat));
1657        status = fdb_get_latency_stats(dbfile, &stat, i);
1658        TEST_CHK(status == FDB_RESULT_SUCCESS);
1659        fprintf(stderr, "%d:\t%u\t%u\t%u\t%" _F64 "\n", i,
1660                stat.lat_max, stat.lat_avg, stat.lat_max, stat.lat_count);
1661    }
1662
1663    fdb_close(dbfile);
1664
1665    // free all documents
1666    for (i=0;i<n;++i){
1667        fdb_doc_free(doc[i]);
1668    }
1669
1670    // free all resources
1671    fdb_shutdown();
1672
1673    memleak_end();
1674
1675    sprintf(bodybuf,"Operational stats test %s", multi_kv ?
1676            "multiple kv instances" : "single kv instance");
1677    TEST_RESULT(bodybuf);
1678}
1679
1680struct work_thread_args{
1681    int tid;
1682    size_t nthreads;
1683    size_t ndocs;
1684    size_t writer;
1685    fdb_doc **doc;
1686    size_t time_sec;
1687    size_t nbatch;
1688    size_t compact_term;
1689    int *n_opened;
1690    int *filename_count;
1691    spin_t *filename_count_lock;
1692    size_t nops;
1693    fdb_config *config;
1694    fdb_kvs_config *kvs_config;
1695};
1696
1697//#define FILENAME "./hdd/dummy"
1698#define FILENAME "dummy"
1699
1700#define KSIZE (100)
1701#define VSIZE (100)
1702#define IDX_DIGIT (7)
1703#define IDX_DIGIT_STR "7"
1704
1705void *_worker_thread(void *voidargs)
1706{
1707    TEST_INIT();
1708
1709    struct work_thread_args *args = (struct work_thread_args *)voidargs;
1710    int i, c, commit_count, filename_count;
1711    struct timeval ts_begin, ts_cur, ts_gap;
1712    fdb_file_handle *dbfile;
1713    fdb_kvs_handle *db;
1714    fdb_status status;
1715    fdb_doc *rdoc = NULL;
1716    char temp[1024];
1717
1718    char cnt_str[IDX_DIGIT+1];
1719    int cnt_int;
1720
1721    filename_count = *args->filename_count;
1722    sprintf(temp, FILENAME"%d", filename_count);
1723    fdb_open(&dbfile, temp, args->config);
1724    fdb_kvs_open_default(dbfile, &db, args->kvs_config);
1725    status = fdb_set_log_callback(db, logCallbackFunc,
1726                                  (void *) "worker_thread");
1727    TEST_CHK(status == FDB_RESULT_SUCCESS);
1728
1729    // wait until all other threads open the DB file.
1730    // (to avoid performing compaction before opening the file)
1731    spin_lock(args->filename_count_lock);
1732    *args->n_opened += 1;
1733    spin_unlock(args->filename_count_lock);
1734    do {
1735        spin_lock(args->filename_count_lock);
1736        if ((size_t)(*args->n_opened) == args->nthreads) {
1737            // all threads open the DB file
1738            spin_unlock(args->filename_count_lock);
1739            break;
1740        }
1741        spin_unlock(args->filename_count_lock);
1742        // sleep 1 sec
1743        sleep(1);
1744    } while (1);
1745
1746    gettimeofday(&ts_begin, NULL);
1747
1748    c = cnt_int = commit_count = 0;
1749    cnt_str[IDX_DIGIT] = 0;
1750
1751    while (1){
1752        i = rand() % args->ndocs;
1753        fdb_doc_create(&rdoc, args->doc[i]->key, args->doc[i]->keylen, NULL, 0, NULL, 0);
1754        status = fdb_get(db, rdoc);
1755
1756        TEST_CHK(status == FDB_RESULT_SUCCESS);
1757        TEST_CMP(rdoc->body, args->doc[i]->body, (IDX_DIGIT+1));
1758
1759        if (args->writer) {
1760            // if writer,
1761            // copy and parse the counter in body
1762            memcpy(cnt_str, (uint8_t *)rdoc->body + (IDX_DIGIT+1), IDX_DIGIT);
1763            cnt_int = atoi(cnt_str);
1764
1765            // increase and rephrase
1766            sprintf(cnt_str, "%0 " IDX_DIGIT_STR "d", ++cnt_int);
1767            memcpy((uint8_t *)rdoc->body + (IDX_DIGIT+1), cnt_str, IDX_DIGIT);
1768
1769            // update and commit
1770            status = fdb_set(db, rdoc);
1771            TEST_CHK(status == FDB_RESULT_SUCCESS);
1772
1773            if (args->nbatch > 0) {
1774                if (c % args->nbatch == 0) {
1775                    // commit for every NBATCH
1776                    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1777                    commit_count++;
1778                    fdb_file_info info;
1779                    fdb_get_file_info(dbfile, &info);
1780                    if (args->compact_term == (size_t)commit_count &&
1781                        args->compact_term > 0 &&
1782                        info.new_filename == NULL &&
1783                        args->tid == 0) {
1784                        // do compaction for every COMPACT_TERM batch
1785                        spin_lock(args->filename_count_lock);
1786                        *args->filename_count += 1;
1787                        filename_count = *args->filename_count;
1788                        spin_unlock(args->filename_count_lock);
1789
1790                        sprintf(temp, FILENAME"%d", filename_count);
1791
1792                        status = fdb_compact(dbfile, temp);
1793                        if (status != FDB_RESULT_SUCCESS) {
1794                            spin_lock(args->filename_count_lock);
1795                            *args->filename_count -= 1;
1796                            spin_unlock(args->filename_count_lock);
1797                        }
1798
1799                        commit_count = 0;
1800                    }
1801                }
1802            }
1803        }
1804        fdb_doc_free(rdoc);
1805        rdoc = NULL;
1806        c++;
1807
1808        gettimeofday(&ts_cur, NULL);
1809        ts_gap = _utime_gap(ts_begin, ts_cur);
1810        if ((size_t)ts_gap.tv_sec >= args->time_sec) break;
1811    }
1812
1813    DBG("Thread #%d (%s) %d ops / %d seconds\n",
1814        args->tid, (args->writer)?("writer"):("reader"), c, (int)args->time_sec);
1815    args->nops = c;
1816
1817    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1818
1819    fdb_kvs_close(db);
1820    fdb_close(dbfile);
1821    thread_exit(0);
1822    return NULL;
1823}
1824
1825void multi_thread_test(
1826    size_t ndocs, size_t wal_threshold, size_t time_sec,
1827    size_t nbatch, size_t compact_term, size_t nwriters, size_t nreaders)
1828{
1829    TEST_INIT();
1830
1831    size_t nwrites, nreads;
1832    int i, r;
1833    int n = nwriters + nreaders;;
1834    thread_t *tid = alca(thread_t, n);
1835    void **thread_ret = alca(void *, n);
1836    struct work_thread_args *args = alca(struct work_thread_args, n);
1837    struct timeval ts_begin, ts_cur, ts_gap;
1838    fdb_file_handle *dbfile;
1839    fdb_kvs_handle *db;
1840    fdb_doc **doc = alca(fdb_doc*, ndocs);
1841    fdb_status status;
1842    fdb_kvs_info kvs_info;
1843
1844    int filename_count = 1;
1845    int n_opened = 0;
1846    spin_t filename_count_lock;
1847    spin_init(&filename_count_lock);
1848
1849    char keybuf[1024], metabuf[1024], bodybuf[1024], temp[1024];
1850
1851    // remove previous dummy files
1852    r = system(SHELL_DEL" " FILENAME "* > errorlog.txt");
1853    (void)r;
1854
1855    fdb_config fconfig = fdb_get_default_config();
1856    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1857    fconfig.buffercache_size = 16777216;
1858    fconfig.wal_threshold = 1024;
1859    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1860    fconfig.compaction_threshold = 0;
1861
1862    memleak_start();
1863
1864    // initial population ===
1865    DBG("Initialize..\n");
1866
1867    // open db
1868    sprintf(temp, FILENAME"%d", filename_count);
1869    fdb_open(&dbfile, temp, &fconfig);
1870    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1871    status = fdb_set_log_callback(db, logCallbackFunc,
1872                                  (void *) "multi_thread_test");
1873    TEST_CHK(status == FDB_RESULT_SUCCESS);
1874
1875    gettimeofday(&ts_begin, NULL);
1876
1877    // insert documents
1878    for (i = 0; (size_t)i < ndocs; ++i){
1879        _set_random_string_smallabt(temp, KSIZE - (IDX_DIGIT+1));
1880        sprintf(keybuf, "k%0" IDX_DIGIT_STR "d%s", i, temp);
1881
1882        sprintf(metabuf, "m%0" IDX_DIGIT_STR "d", i);
1883
1884        _set_random_string_smallabt(temp, VSIZE-(IDX_DIGIT*2+1));
1885        sprintf(bodybuf, "b%0" IDX_DIGIT_STR "d%0" IDX_DIGIT_STR "d%s", i, 0, temp);
1886
1887        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1888            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1889        fdb_set(db, doc[i]);
1890    }
1891
1892    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1893
1894    gettimeofday(&ts_cur, NULL);
1895    ts_gap = _utime_gap(ts_begin, ts_cur);
1896    //DBG("%d.%09d seconds elapsed\n", (int)ts_gap.tv_sec, (int)ts_gap.tv_nsec);
1897
1898    fdb_kvs_close(db);
1899    fdb_close(dbfile);
1900    // end of population ===
1901
1902    // drop OS's page cache
1903    //r = system("free && sync && echo 3 > /proc/sys/vm/drop_caches && free");
1904
1905    // create workers
1906    for (i=0;i<n;++i){
1907        args[i].tid = i;
1908        args[i].nthreads = n;
1909        args[i].writer = (((size_t)i<nwriters)?(1):(0));
1910        args[i].ndocs = ndocs;
1911        args[i].doc = doc;
1912        args[i].time_sec = time_sec;
1913        args[i].nbatch = nbatch;
1914        args[i].compact_term = compact_term;
1915        args[i].n_opened = &n_opened;
1916        args[i].filename_count = &filename_count;
1917        args[i].filename_count_lock = &filename_count_lock;
1918        args[i].config = &fconfig;
1919        args[i].kvs_config = &kvs_config;
1920        thread_create(&tid[i], _worker_thread, &args[i]);
1921    }
1922
1923    printf("wait for %d seconds..\n", (int)time_sec);
1924
1925    // wait for thread termination
1926    for (i=0;i<n;++i){
1927        thread_join(tid[i], &thread_ret[i]);
1928    }
1929
1930    // free all documents
1931    for (i=0;(size_t)i<ndocs;++i){
1932        fdb_doc_free(doc[i]);
1933    }
1934
1935    nwrites = nreads = 0;
1936    for (i=0;i<n;++i){
1937        if (args[i].writer) {
1938            nwrites += args[i].nops;
1939        } else {
1940            nreads += args[i].nops;
1941        }
1942    }
1943    printf("read: %.1f ops/sec\n", (double)nreads/time_sec);
1944    printf("write: %.1f ops/sec\n", (double)nwrites/time_sec);
1945
1946    // check sequence number
1947    sprintf(temp, FILENAME"%d", filename_count);
1948    fdb_open(&dbfile, temp, &fconfig);
1949    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1950    fdb_get_kvs_info(db, &kvs_info);
1951    TEST_CHK(kvs_info.last_seqnum == ndocs+nwrites);
1952    fdb_kvs_close(db);
1953    fdb_close(dbfile);
1954
1955    // shutdown
1956    fdb_shutdown();
1957
1958    memleak_end();
1959
1960    TEST_RESULT("multi thread test");
1961}
1962
1963void *multi_thread_client_shutdown(void *args)
1964{
1965
1966    TEST_INIT();
1967
1968    int i, r;
1969    int nclients;
1970    fdb_file_handle *tdbfile;
1971    fdb_status status;
1972    fdb_config fconfig;
1973    fdb_kvs_config kvs_config;
1974    thread_t *tid;
1975    void **thread_ret;
1976
1977    if (args == NULL)
1978    { // parent
1979        memleak_start();
1980
1981        r = system(SHELL_DEL" dummy* > errorlog.txt");
1982        (void)r;
1983        nclients = 2;
1984        tid = alca(thread_t, nclients);
1985        thread_ret = alca(void *, nclients);
1986        for (i=0;i<nclients;++i){
1987            thread_create(&tid[i], multi_thread_client_shutdown, (void *)&i);
1988        }
1989        for (i=0;i<nclients;++i){
1990            thread_join(tid[i], &thread_ret[i]);
1991        }
1992
1993        memleak_end();
1994        TEST_RESULT("multi thread client shutdown");
1995        return NULL;
1996    }
1997
1998    // threads enter here //
1999
2000    fconfig = fdb_get_default_config();
2001    kvs_config = fdb_get_default_kvs_config();
2002    fconfig.wal_threshold = 1024;
2003    fconfig.compaction_threshold = 0;
2004
2005    // open/close db
2006    status = fdb_open(&tdbfile, "./dummy1", &fconfig);
2007    TEST_CHK(status == FDB_RESULT_SUCCESS);
2008    TEST_CHK(fdb_close(tdbfile) == FDB_RESULT_SUCCESS);
2009
2010    // shutdown
2011    fdb_shutdown();
2012    thread_exit(0);
2013    return NULL;
2014}
2015
2016void *multi_thread_kvs_client(void *args)
2017{
2018
2019    TEST_INIT();
2020
2021    int i, j, r;
2022    int n = 50;
2023    int nclients = 20;
2024    int *tid_args = alca(int, nclients);
2025    char dbstr[256];
2026    char keybuf[256], metabuf[256], bodybuf[256];
2027    fdb_file_handle *dbfile;
2028    fdb_kvs_handle *tdb;
2029    fdb_kvs_handle **db = alca(fdb_kvs_handle*, nclients);
2030    fdb_doc **doc = alca(fdb_doc*, n);
2031    fdb_doc *rdoc = NULL;
2032    fdb_status status;
2033    fdb_config fconfig;
2034    fdb_kvs_config kvs_config;
2035    fdb_seqnum_t seqnum;
2036    thread_t *tid;
2037    void **thread_ret;
2038
2039    if (args == NULL)
2040    { // parent
2041        memleak_start();
2042
2043        r = system(SHELL_DEL" dummy* > errorlog.txt");
2044        (void)r;
2045
2046        // init dbfile
2047        fconfig = fdb_get_default_config();
2048        fconfig.buffercache_size = 0;
2049        fconfig.wal_threshold = 1024;
2050        fconfig.compaction_threshold = 0;
2051
2052        status = fdb_open(&dbfile, "./dummy1", &fconfig);
2053        TEST_CHK(status == FDB_RESULT_SUCCESS);
2054
2055        tid = alca(thread_t, nclients);
2056        thread_ret = alca(void *, nclients);
2057        for (i=0;i<nclients;++i){
2058            sprintf(dbstr, "db%d", i);
2059            kvs_config = fdb_get_default_kvs_config();
2060            status = fdb_kvs_open(dbfile, &db[i], dbstr, &kvs_config);
2061            TEST_CHK(status == FDB_RESULT_SUCCESS);
2062            status = fdb_kvs_close(db[i]);
2063            TEST_CHK(status == FDB_RESULT_SUCCESS);
2064        }
2065        for (i=0;i<nclients;++i){
2066            tid_args[i] = i;
2067            thread_create(&tid[i], multi_thread_kvs_client,
2068                          (void *)&tid_args[i]);
2069        }
2070        for (i=0;i<nclients;++i){
2071            thread_join(tid[i], &thread_ret[i]);
2072        }
2073
2074        status = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2075        TEST_CHK(status == FDB_RESULT_SUCCESS);
2076
2077        // check threads updated kvs
2078        for (i=0; i<nclients; i++){
2079            sprintf(dbstr, "db%d", i);
2080            kvs_config = fdb_get_default_kvs_config();
2081            status = fdb_kvs_open(dbfile, &db[i], dbstr, &kvs_config);
2082            TEST_CHK(status == FDB_RESULT_SUCCESS);
2083
2084            // verify seqnum
2085            status = fdb_get_kvs_seqnum(db[i], &seqnum);
2086            TEST_CHK(status == FDB_RESULT_SUCCESS);
2087            TEST_CHK(seqnum == (fdb_seqnum_t)n);
2088
2089            for (j=0; j<n; j++){
2090                sprintf(keybuf, "key%d", j);
2091                sprintf(metabuf, "meta%d", j);
2092                sprintf(bodybuf, "body%d", j);
2093                fdb_doc_create(&rdoc, keybuf, strlen(keybuf),
2094                                      NULL, 0, NULL, 0);
2095                status = fdb_get(db[i], rdoc);
2096                TEST_CHK(status == FDB_RESULT_SUCCESS);
2097                TEST_CHK(!memcmp(rdoc->key, keybuf, strlen(keybuf)));
2098                TEST_CHK(!memcmp(rdoc->meta, metabuf, rdoc->metalen));
2099                TEST_CHK(!memcmp(rdoc->body, bodybuf, rdoc->bodylen));
2100                fdb_doc_free(rdoc);
2101            }
2102            status = fdb_kvs_close(db[i]);
2103            TEST_CHK(status == FDB_RESULT_SUCCESS);
2104        }
2105
2106        status = fdb_close(dbfile);
2107        TEST_CHK(status == FDB_RESULT_SUCCESS);
2108        fdb_shutdown();
2109        memleak_end();
2110        TEST_RESULT("multi thread kvs client");
2111        return NULL;
2112    }
2113
2114    // threads enter here //
2115
2116    // open fhandle
2117    fconfig = fdb_get_default_config();
2118    status = fdb_open(&dbfile, "./dummy1", &fconfig);
2119    TEST_CHK(status == FDB_RESULT_SUCCESS);
2120
2121    // get kvs ID from args
2122    memcpy(&i, args, sizeof(int));
2123    sprintf(dbstr, "db%d", i);
2124    kvs_config = fdb_get_default_kvs_config();
2125    status = fdb_kvs_open(dbfile, &tdb, dbstr, &kvs_config);
2126    TEST_CHK(status == FDB_RESULT_SUCCESS);
2127
2128    // insert documents
2129    for (i=0;i<n;++i){
2130        sprintf(keybuf, "key%d", i);
2131        sprintf(metabuf, "meta%d", i);
2132        sprintf(bodybuf, "body%d", i);
2133        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2134                                (void*)metabuf, strlen(metabuf),
2135                                (void*)bodybuf, strlen(bodybuf));
2136        status = fdb_set(tdb, doc[i]);
2137        TEST_CHK(status == FDB_RESULT_SUCCESS);
2138        fdb_doc_free(doc[i]);
2139    }
2140    status = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2141    TEST_CHK(status == FDB_RESULT_SUCCESS);
2142
2143    fdb_close(dbfile);
2144    return NULL;
2145}
2146
2147void incomplete_block_test()
2148{
2149    TEST_INIT();
2150
2151    memleak_start();
2152
2153    int i, r;
2154    int n = 2;
2155    fdb_file_handle *dbfile;
2156    fdb_kvs_handle *db;
2157    fdb_doc **doc = alca(fdb_doc*, n);
2158    fdb_doc *rdoc = NULL;
2159    fdb_status status;
2160
2161    char keybuf[256], metabuf[256], bodybuf[256];
2162
2163    // remove previous dummy files
2164    r = system(SHELL_DEL" dummy* > errorlog.txt");
2165    (void)r;
2166
2167    fdb_config fconfig = fdb_get_default_config();
2168    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2169    fconfig.buffercache_size = 0;
2170    fconfig.wal_threshold = 1024;
2171    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2172    fconfig.compaction_threshold = 0;
2173
2174    // open db
2175    fdb_open(&dbfile, "./dummy1", &fconfig);
2176    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2177    status = fdb_set_log_callback(db, logCallbackFunc,
2178                                  (void *) "incomplete_block_test");
2179    TEST_CHK(status == FDB_RESULT_SUCCESS);
2180
2181    // insert documents
2182    for (i=0;i<n;++i){
2183        sprintf(keybuf, "key%d", i);
2184        sprintf(metabuf, "meta%d", i);
2185        sprintf(bodybuf, "body%d", i);
2186        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2187            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2188        fdb_set(db, doc[i]);
2189    }
2190
2191    // retrieve documents
2192    for (i=0;i<n;++i){
2193        // search by key
2194        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
2195        status = fdb_get(db, rdoc);
2196
2197        // updated documents
2198        TEST_CHK(status == FDB_RESULT_SUCCESS);
2199        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2200        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2201
2202        // free result document
2203        fdb_doc_free(rdoc);
2204        rdoc = NULL;
2205    }
2206
2207    // close db file
2208    fdb_kvs_close(db);
2209    fdb_close(dbfile);
2210
2211    // free all documents
2212    for (i=0;i<n;++i){
2213        fdb_doc_free(doc[i]);
2214    }
2215
2216    // free all resources
2217    fdb_shutdown();
2218
2219    memleak_end();
2220
2221    TEST_RESULT("incomplete block test");
2222}
2223
2224
2225static int _cmp_double(void *key1, size_t keylen1, void *key2, size_t keylen2)
2226{
2227    double aa, bb;
2228
2229    if (!keylen1) {
2230        // key1 not set
2231        return -1;
2232    }
2233    if (!keylen2) {
2234        // key2 not set
2235        return 1;
2236    }
2237
2238    aa = *(double *)key1;
2239    bb = *(double *)key2;
2240
2241    if (aa<bb) {
2242        return -1;
2243    } else if (aa>bb) {
2244        return 1;
2245    } else {
2246        return 0;
2247    }
2248}
2249
2250void custom_compare_primitive_test()
2251{
2252    TEST_INIT();
2253
2254    memleak_start();
2255
2256    int i, r;
2257    int n = 10;
2258    fdb_file_handle *dbfile;
2259    fdb_kvs_handle *db;
2260    fdb_doc **doc = alca(fdb_doc*, n);
2261    fdb_doc *rdoc = NULL;
2262    fdb_status status;
2263    fdb_iterator *iterator;
2264
2265    char keybuf[256], bodybuf[256];
2266    double key_double, key_double_prev;
2267
2268    // remove previous dummy files
2269    r = system(SHELL_DEL" dummy* > errorlog.txt");
2270    (void)r;
2271
2272    fdb_config fconfig = fdb_get_default_config();
2273    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2274    fconfig.buffercache_size = 0;
2275    fconfig.wal_threshold = 1024;
2276    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2277    fconfig.compaction_threshold = 0;
2278    fconfig.multi_kv_instances = true;
2279
2280    kvs_config.custom_cmp = _cmp_double;
2281
2282    // open db with custom compare function for double key type
2283    fdb_open(&dbfile, "./dummy1", &fconfig);
2284    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2285    status = fdb_set_log_callback(db, logCallbackFunc,
2286                                  (void *) "custom_compare_primitive_test");
2287    TEST_CHK(status == FDB_RESULT_SUCCESS);
2288
2289    for (i=0;i<n;++i){
2290        key_double = 10000/(i*11.0);
2291        memcpy(keybuf, &key_double, sizeof(key_double));
2292        sprintf(bodybuf, "value: %d, %f", i, key_double);
2293        fdb_doc_create(&doc[i], (void*)keybuf, sizeof(key_double), NULL, 0,
2294            (void*)bodybuf, strlen(bodybuf)+1);
2295        fdb_set(db, doc[i]);
2296    }
2297
2298    // range scan (before flushing WAL)
2299    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2300    key_double_prev = -1;
2301    do {
2302        status = fdb_iterator_get(iterator, &rdoc);
2303        TEST_CHK(status == FDB_RESULT_SUCCESS);
2304        memcpy(&key_double, rdoc->key, rdoc->keylen);
2305        TEST_CHK(key_double > key_double_prev);
2306        key_double_prev = key_double;
2307        fdb_doc_free(rdoc);
2308        rdoc = NULL;
2309    } while(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2310    fdb_iterator_close(iterator);
2311
2312    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2313
2314    // range scan (after flushing WAL)
2315    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2316    key_double_prev = -1;
2317    do {
2318        status = fdb_iterator_get(iterator, &rdoc);
2319        TEST_CHK(status == FDB_RESULT_SUCCESS);
2320        memcpy(&key_double, rdoc->key, rdoc->keylen);
2321        TEST_CHK(key_double > key_double_prev);
2322        key_double_prev = key_double;
2323        fdb_doc_free(rdoc);
2324        rdoc = NULL;
2325    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2326    fdb_iterator_close(iterator);
2327
2328    // do compaction
2329    fdb_compact(dbfile, (char *) "./dummy2");
2330
2331    // range scan (after compaction)
2332    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2333    key_double_prev = -1;
2334    do {
2335        status = fdb_iterator_get(iterator, &rdoc);
2336        TEST_CHK(status == FDB_RESULT_SUCCESS);
2337        memcpy(&key_double, rdoc->key, rdoc->keylen);
2338        TEST_CHK(key_double > key_double_prev);
2339        key_double_prev = key_double;
2340        fdb_doc_free(rdoc);
2341        rdoc = NULL;
2342    } while(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2343    fdb_iterator_close(iterator);
2344
2345    // close db file
2346    fdb_kvs_close(db);
2347    fdb_close(dbfile);
2348
2349    // free all documents
2350    for (i=0;i<n;++i){
2351        fdb_doc_free(doc[i]);
2352    }
2353
2354    // free all resources
2355    fdb_shutdown();
2356
2357    memleak_end();
2358
2359    TEST_RESULT("custom compare function for primitive key test");
2360}
2361
2362static int _cmp_variable(void *key1, size_t keylen1, void *key2, size_t keylen2)
2363{
2364    if (keylen1 < 6 || keylen2 < 6) {
2365        return (keylen1 - keylen2);
2366    }
2367    // compare only 3rd~8th bytes (ignore the others)
2368    return memcmp((uint8_t*)key1+2, (uint8_t*)key2+2, 6);
2369}
2370
2371void custom_compare_variable_test()
2372{
2373    TEST_INIT();
2374
2375    memleak_start();
2376
2377    int i, j, r;
2378    int n = 1000;
2379    int count;
2380    fdb_file_handle *dbfile;
2381    fdb_kvs_handle *db, *db2;
2382    fdb_doc **doc = alca(fdb_doc*, n);
2383    fdb_doc *rdoc = NULL;
2384    fdb_status status;
2385    fdb_iterator *iterator;
2386
2387    size_t keylen = 16;
2388    size_t prev_keylen;
2389    char keybuf[256], bodybuf[256];
2390    char prev_key[256];
2391
2392    // remove previous dummy files
2393    r = system(SHELL_DEL" dummy* > errorlog.txt");
2394    (void)r;
2395
2396    fdb_config fconfig = fdb_get_default_config();
2397    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2398    fconfig.seqtree_opt = FDB_SEQTREE_USE;
2399    fconfig.buffercache_size = 0;
2400    fconfig.wal_threshold = 1024;
2401    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2402    fconfig.compaction_threshold = 0;
2403    fconfig.multi_kv_instances = true;
2404
2405    kvs_config.custom_cmp = _cmp_variable;
2406
2407    // open db with custom compare function for variable length key type
2408    //fdb_open_cmp_variable(&dbfile, "./dummy1", &fconfig);
2409    fdb_open(&dbfile, "./dummy1", &fconfig);
2410    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2411    status = fdb_set_log_callback(db, logCallbackFunc,
2412                                  (void *) "custom_compare_variable_test");
2413    TEST_CHK(status == FDB_RESULT_SUCCESS);
2414
2415    for (i=0;i<n;++i){
2416        for (j=0;j<2;++j){
2417            keybuf[j] = 'a' + rand()%('z'-'a');
2418        }
2419        sprintf(keybuf+2, "%06d", i);
2420        for (j=8;(size_t)j<keylen-1;++j){
2421            keybuf[j] = 'a' + rand()%('z'-'a');
2422        }
2423        keybuf[keylen-1] = 0;
2424        sprintf(bodybuf, "value: %d", i);
2425        fdb_doc_create(&doc[i], (void*)keybuf, keylen, NULL, 0,
2426            (void*)bodybuf, strlen(bodybuf)+1);
2427        fdb_set(db, doc[i]);
2428    }
2429
2430    // point query
2431    for (i=0;i<n;++i){
2432        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
2433        status = fdb_get(db, rdoc);
2434
2435        TEST_CHK(status == FDB_RESULT_SUCCESS);
2436        TEST_CHK(rdoc->bodylen == doc[i]->bodylen);
2437        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2438
2439        fdb_doc_free(rdoc);
2440        rdoc = NULL;
2441    }
2442
2443    // range scan (before flushing WAL)
2444    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2445    sprintf(prev_key, "%016d", 0);
2446    count = 0;
2447    prev_keylen = 16;
2448    do {
2449        status = fdb_iterator_get(iterator, &rdoc);
2450        TEST_CHK(status == FDB_RESULT_SUCCESS);
2451        TEST_CHK(_cmp_variable(prev_key, prev_keylen, rdoc->key, rdoc->keylen) <= 0);
2452        prev_keylen = rdoc->keylen;
2453        memcpy(prev_key, rdoc->key, rdoc->keylen);
2454        fdb_doc_free(rdoc);
2455        rdoc = NULL;
2456        count++;
2457    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2458    TEST_CHK(count == n);
2459    fdb_iterator_close(iterator);
2460
2461    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2462
2463    // range scan (after flushing WAL)
2464    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2465    sprintf(prev_key, "%016d", 0);
2466    count = 0;
2467    prev_keylen = 16;
2468    do {
2469        status = fdb_iterator_get(iterator, &rdoc);
2470        TEST_CHK(status == FDB_RESULT_SUCCESS);
2471        TEST_CHK(_cmp_variable(prev_key, prev_keylen, rdoc->key, rdoc->keylen)
2472                 <= 0);
2473        prev_keylen = rdoc->keylen;
2474        memcpy(prev_key, rdoc->key, rdoc->keylen);
2475        fdb_doc_free(rdoc);
2476        rdoc = NULL;
2477        count++;
2478    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2479    TEST_CHK(count == n);
2480    fdb_iterator_close(iterator);
2481
2482    // do compaction
2483    fdb_compact(dbfile, (char *) "./dummy2");
2484
2485    // range scan (after compaction)
2486    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2487    sprintf(prev_key, "%016d", 0);
2488    count = 0;
2489    prev_keylen = 16;
2490    do {
2491        status = fdb_iterator_get(iterator, &rdoc);
2492        TEST_CHK(status == FDB_RESULT_SUCCESS);
2493        TEST_CHK(_cmp_variable(prev_key, prev_keylen, rdoc->key, rdoc->keylen) <= 0);
2494        prev_keylen = rdoc->keylen;
2495        memcpy(prev_key, rdoc->key, rdoc->keylen);
2496        fdb_doc_free(rdoc);
2497        rdoc = NULL;
2498        count++;
2499    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2500    TEST_CHK(count == n);
2501    fdb_iterator_close(iterator);
2502
2503    // range scan by sequence
2504    fdb_iterator_sequence_init(db, &iterator, 0, 0, 0x0);
2505    count = 0;
2506    do { // forward
2507        status = fdb_iterator_get(iterator, &rdoc);
2508        TEST_CHK(status == FDB_RESULT_SUCCESS);
2509        fdb_doc_free(rdoc);
2510        rdoc = NULL;
2511        count++;
2512    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2513    TEST_CHK(count == n);
2514
2515    // Reverse direction
2516    for (; fdb_iterator_prev(iterator) != FDB_RESULT_ITERATOR_FAIL; --count) {
2517        status = fdb_iterator_get(iterator, &rdoc);
2518        TEST_CHK(status == FDB_RESULT_SUCCESS);
2519        fdb_doc_free(rdoc);
2520        rdoc = NULL;
2521    };
2522    TEST_CHK(count == 0);
2523    fdb_iterator_close(iterator);
2524
2525    // open another handle
2526    kvs_config.custom_cmp = NULL;
2527    fdb_kvs_open_default(dbfile, &db2, &kvs_config);
2528
2529    // point query
2530    for (i=0;i<n;++i){
2531        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
2532        status = fdb_get(db2, rdoc);
2533
2534        TEST_CHK(status == FDB_RESULT_SUCCESS);
2535        TEST_CHK(rdoc->bodylen == doc[i]->bodylen);
2536        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2537
2538        fdb_doc_free(rdoc);
2539        rdoc = NULL;
2540    }
2541
2542    // close db file
2543    fdb_kvs_close(db);
2544    fdb_kvs_close(db2);
2545    fdb_close(dbfile);
2546
2547    // free all documents
2548    for (i=0;i<n;++i){
2549        fdb_doc_free(doc[i]);
2550    }
2551
2552    // free all resources
2553    fdb_shutdown();
2554
2555    memleak_end();
2556
2557    TEST_RESULT("custom compare function for variable length key test");
2558}
2559
2560/*
2561 * custom compare test with commit and compact
2562 *    eqkeys:  boolean to toggle whether bytes in
2563 *             comparision range are equal
2564 */
2565void custom_compare_commit_compact(bool eqkeys)
2566{
2567    TEST_INIT();
2568
2569    memleak_start();
2570
2571    int i, j, r;
2572    int count;
2573    int n = 10;
2574    static const int len = 1024;
2575    char keybuf[len];
2576    static const char *achar = "a";
2577    fdb_doc *rdoc = NULL;
2578    fdb_status status;
2579    fdb_file_handle *dbfile;
2580    fdb_kvs_handle *db;
2581    fdb_iterator *iterator;
2582    fdb_config fconfig = fdb_get_default_config();
2583    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2584
2585    // remove previous dummy files
2586    r = system(SHELL_DEL" dummy* > errorlog.txt");
2587    (void)r;
2588
2589    fconfig.buffercache_size = 0;
2590    fconfig.wal_threshold = 1024;
2591    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2592    fconfig.compaction_threshold = 0;
2593    fconfig.multi_kv_instances = true;
2594
2595    kvs_config.custom_cmp = _cmp_variable;
2596
2597
2598
2599    r = system(SHELL_DEL" dummy* > errorlog.txt");
2600    (void)r;
2601
2602
2603    // open db
2604    status = fdb_open(&dbfile, "./dummy1", &fconfig);
2605    TEST_CHK(status == FDB_RESULT_SUCCESS);
2606    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
2607    TEST_CHK(status == FDB_RESULT_SUCCESS);
2608
2609
2610    for (i=0;i<n;++i){
2611        if(eqkeys){
2612            sprintf(keybuf, "%d", i);
2613            for (j=1;j<len;++j) {
2614                keybuf[j] = *achar;
2615            }
2616        } else {
2617            sprintf(keybuf, "000%d", i);
2618            for (j=4;j<len;++j) {
2619                keybuf[j] = *achar;
2620            }
2621        }
2622        keybuf[len-1] = '\0';
2623        // set kv
2624        status = fdb_set_kv(db, keybuf, strlen(keybuf), NULL, 0);
2625        TEST_CHK(status == FDB_RESULT_SUCCESS);
2626    }
2627
2628    // compact pre & post commit
2629    fdb_compact(dbfile, NULL);
2630    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2631    fdb_compact(dbfile, NULL);
2632
2633    // scan after flush
2634    count = 0;
2635    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2636    do {
2637        status = fdb_iterator_get(iterator, &rdoc);
2638        TEST_CHK(status == FDB_RESULT_SUCCESS);
2639        fdb_doc_free(rdoc);
2640        rdoc = NULL;
2641        count++;
2642    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2643
2644    if (eqkeys) {
2645        // since the custom cmp function compares only 3rd~8th bytes,
2646        // all keys are identified as the same key.
2647        TEST_CHK(count == 1);
2648    } else {
2649        TEST_CHK(count == n);
2650    }
2651
2652    fdb_iterator_close(iterator);
2653
2654    fdb_close(dbfile);
2655    fdb_shutdown();
2656
2657    memleak_end();
2658    TEST_RESULT("custom compare commit compact");
2659
2660}
2661
2662void custom_seqnum_test(bool multi_kv)
2663{
2664    TEST_INIT();
2665
2666    memleak_start();
2667
2668    int i, r;
2669    int n = 10;
2670    int num_kv = 4;
2671    fdb_file_handle *dbfile;
2672    fdb_kvs_handle **db = alca(fdb_kvs_handle*, num_kv);
2673    fdb_doc **doc = alca(fdb_doc*, n);
2674    fdb_doc real_doc;
2675    fdb_doc *rdoc = &real_doc;
2676    fdb_status status;
2677    fdb_iterator *iterator;
2678
2679    char keybuf[256], bodybuf[256];
2680    memset(&real_doc, 0, sizeof(fdb_doc));
2681    real_doc.key = &keybuf;
2682    real_doc.body = &bodybuf;
2683
2684    // remove previous dummy files
2685    r = system(SHELL_DEL" dummy* > errorlog.txt");
2686    (void)r;
2687
2688    fdb_config fconfig = fdb_get_default_config();
2689    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2690
2691    fconfig.buffercache_size = 0;
2692    fconfig.wal_threshold = 1024;
2693    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2694    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
2695    fconfig.compaction_threshold = 0;
2696    fconfig.multi_kv_instances = multi_kv;
2697    r = 0;
2698
2699    fdb_open(&dbfile, "./dummy1", &fconfig);
2700    if (multi_kv) {
2701        num_kv = 4;
2702        for (r = num_kv - 1; r >= 0; --r) {
2703            char tmp[16];
2704            sprintf(tmp, "kv%d", r);
2705            status = fdb_kvs_open(dbfile, &db[r], tmp, &kvs_config);
2706            TEST_CHK(status == FDB_RESULT_SUCCESS);
2707            status = fdb_set_log_callback(db[r], logCallbackFunc,
2708                                          (void *) "custom_seqnum_test");
2709            TEST_CHK(status == FDB_RESULT_SUCCESS);
2710        }
2711    } else {
2712        num_kv = 1;
2713        status = fdb_kvs_open_default(dbfile, &db[r], &kvs_config);
2714        TEST_CHK(status == FDB_RESULT_SUCCESS);
2715        status = fdb_set_log_callback(db[r], logCallbackFunc,
2716                (void *) "custom_seqnum_test");
2717        TEST_CHK(status == FDB_RESULT_SUCCESS);
2718    }
2719
2720    for (i = 0; i < n/2; ++i){
2721        sprintf(keybuf, "key%d", i);
2722        sprintf(bodybuf, "body%d", i);
2723        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf) + 1, NULL, 0,
2724            (void*)bodybuf, strlen(bodybuf)+1);
2725        for (r = num_kv - 1; r >= 0; --r) {
2726            status = fdb_set(db[r], doc[i]);
2727            TEST_CHK(status == FDB_RESULT_SUCCESS);
2728        }
2729    }
2730
2731    for (i = n/2; i < n; ++i){
2732        sprintf(keybuf, "key%d", i);
2733        sprintf(bodybuf, "body%d", i);
2734        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf) + 1, NULL, 0,
2735            (void*)bodybuf, strlen(bodybuf)+1);
2736        for (r = num_kv - 1; r >= 0; --r) {
2737            fdb_doc_set_seqnum(doc[i], (i+1)*2); // double seqnum instead of ++
2738            status = fdb_set(db[r], doc[i]);
2739            TEST_CHK(status == FDB_RESULT_SUCCESS);
2740        }
2741    }
2742
2743    for (r = num_kv - 1; r >= 0; --r) {
2744        // range scan (before flushing WAL)
2745        fdb_iterator_init(db[r], &iterator, NULL, 0, NULL, 0, 0x0);
2746        i = 0;
2747        do {
2748            status = fdb_iterator_get(iterator, &rdoc);
2749            TEST_CHK(status == FDB_RESULT_SUCCESS);
2750            TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2751            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2752            ++i;
2753            if (i <= n/2) {
2754                TEST_CHK(rdoc->seqnum == (fdb_seqnum_t)i);
2755            } else {
2756                TEST_CHK(rdoc->seqnum == (fdb_seqnum_t)i*2);
2757            }
2758        } while(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2759        fdb_iterator_close(iterator);
2760
2761        fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2762    }
2763
2764    // do compaction
2765    fdb_compact(dbfile, (char *) "./dummy2");
2766
2767    for (i = n/2; i < n; ++i){
2768        sprintf(keybuf, "key%d", i);
2769        for (r = num_kv - 1; r >= 0; --r) {
2770            rdoc->seqnum = (i + 1)*2;
2771            status = fdb_get_byseq(db[r], rdoc);
2772            TEST_CHK(status == FDB_RESULT_SUCCESS);
2773        }
2774    }
2775
2776    fdb_close(dbfile);
2777
2778    // free all documents
2779    for (i=0;i<n;++i){
2780        fdb_doc_free(doc[i]);
2781    }
2782
2783    // free all resources
2784    fdb_shutdown();
2785
2786    memleak_end();
2787
2788    sprintf(bodybuf,"Custom sequence number test %s", multi_kv ?
2789            "multiple kv instances" : "single kv instance");
2790    TEST_RESULT(bodybuf);
2791}
2792
2793void doc_compression_test()
2794{
2795    TEST_INIT();
2796
2797    memleak_start();
2798
2799    int i, r;
2800    int n = 10;
2801    int dummy_len = 32;
2802    fdb_file_handle *dbfile;
2803    fdb_kvs_handle *db;
2804    fdb_doc **doc = alca(fdb_doc*, n);
2805    fdb_doc *rdoc = NULL;
2806    fdb_status status;
2807
2808    char keybuf[256], metabuf[256], bodybuf[256], temp[256];
2809
2810    // remove previous dummy files
2811    r = system(SHELL_DEL" dummy* > errorlog.txt");
2812    (void)r;
2813
2814    fdb_config fconfig = fdb_get_default_config();
2815    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2816    fconfig.buffercache_size = 0;
2817    fconfig.wal_threshold = 1024;
2818    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2819    fconfig.compress_document_body = true;
2820    fconfig.compaction_threshold = 0;
2821    fconfig.purging_interval = 1;
2822
2823    // open db
2824    fdb_open(&dbfile, "./dummy1", &fconfig);
2825    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2826    status = fdb_set_log_callback(db, logCallbackFunc,
2827                                  (void *) "doc_compression_test");
2828    TEST_CHK(status == FDB_RESULT_SUCCESS);
2829
2830    // set dummy str
2831    memset(temp, 'a', dummy_len);
2832    temp[dummy_len]=0;
2833
2834    // insert documents
2835    for (i=0;i<n;++i){
2836        sprintf(keybuf, "key%d", i);
2837        sprintf(metabuf, "meta%d", i);
2838        sprintf(bodybuf, "body%d_%s", i, temp);
2839        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2840            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2841        fdb_set(db, doc[i]);
2842    }
2843
2844    // remove document #5
2845    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, doc[5]->meta, doc[5]->metalen, NULL, 0);
2846    status = fdb_del(db, rdoc);
2847    TEST_CHK(status == FDB_RESULT_SUCCESS);
2848    fdb_doc_free(rdoc);
2849    rdoc = NULL;
2850
2851    // commit
2852    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2853
2854    // close the db
2855    fdb_kvs_close(db);
2856    fdb_close(dbfile);
2857
2858    // reopen
2859    fdb_open(&dbfile, "./dummy1", &fconfig);
2860    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2861    status = fdb_set_log_callback(db, logCallbackFunc,
2862                                  (void *) "doc_compression_test");
2863    TEST_CHK(status == FDB_RESULT_SUCCESS);
2864
2865    // update dummy str
2866    dummy_len = 64;
2867    memset(temp, 'b', dummy_len);
2868    temp[dummy_len]=0;
2869
2870    // update document #0 and #1
2871    for (i=0;i<2;++i){
2872        sprintf(metabuf, "newmeta%d", i);
2873        sprintf(bodybuf, "newbody%d_%s", i, temp);
2874        fdb_doc_update(&doc[i], (void *)metabuf, strlen(metabuf),
2875            (void *)bodybuf, strlen(bodybuf));
2876        fdb_set(db, doc[i]);
2877    }
2878
2879    // commit
2880    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2881
2882    // retrieve documents
2883    for (i=0;i<n;++i){
2884        // search by key
2885        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
2886        status = fdb_get(db, rdoc);
2887
2888        if (i != 5) {
2889            // updated documents
2890            TEST_CHK(status == FDB_RESULT_SUCCESS);
2891            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2892            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2893        } else {
2894            // removed document
2895            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
2896        }
2897
2898        // free result document
2899        fdb_doc_free(rdoc);
2900        rdoc = NULL;
2901    }
2902
2903    // do compaction
2904    fdb_compact(dbfile, (char *) "./dummy2");
2905
2906    // retrieve documents after compaction
2907    for (i=0;i<n;++i){
2908        // search by key
2909        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
2910        status = fdb_get(db, rdoc);
2911
2912        if (i != 5) {
2913            // updated documents
2914            TEST_CHK(status == FDB_RESULT_SUCCESS);
2915            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2916            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2917        } else {
2918            // removed document
2919            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
2920        }
2921
2922        // free result document
2923        fdb_doc_free(rdoc);
2924        rdoc = NULL;
2925    }
2926
2927    // free all documents
2928    for (i=0;i<n;++i){
2929        fdb_doc_free(doc[i]);
2930    }
2931
2932    // close db file
2933    fdb_kvs_close(db);
2934    fdb_close(dbfile);
2935
2936    // free all resources
2937    fdb_shutdown();
2938
2939    memleak_end();
2940
2941    TEST_RESULT("document compression test");
2942}
2943
2944void read_doc_by_offset_test()
2945{
2946    TEST_INIT();
2947    memleak_start();
2948
2949    int i, r;
2950    int n = 100;
2951    fdb_file_handle *dbfile;
2952    fdb_kvs_handle *db;
2953    fdb_doc **doc = alca(fdb_doc*, n);
2954    fdb_doc *rdoc, *rdoc1;
2955    fdb_status status;
2956
2957    char keybuf[256], metabuf[256], bodybuf[256];
2958
2959    // remove previous dummy files
2960    r = system(SHELL_DEL" dummy* > errorlog.txt");
2961    (void)r;
2962
2963    fdb_config fconfig = fdb_get_default_config();
2964    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2965    fconfig.buffercache_size = 0;
2966    fconfig.wal_threshold = 1024;
2967    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2968    fconfig.purging_interval = 3600;
2969    fconfig.compaction_threshold = 0;
2970
2971    // open db
2972    fdb_open(&dbfile, "./dummy1", &fconfig);
2973    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2974    status = fdb_set_log_callback(db, logCallbackFunc,
2975                                  (void *) "read_doc_by_offset_test");
2976    TEST_CHK(status == FDB_RESULT_SUCCESS);
2977
2978    // insert documents
2979    for (i=0;i<n;++i){
2980        sprintf(keybuf, "key%d", i);
2981        sprintf(metabuf, "meta%d", i);
2982        sprintf(bodybuf, "body%d", i);
2983        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2984            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2985        fdb_set(db, doc[i]);
2986    }
2987
2988    // commit
2989    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2990
2991    // update documents from #0 to #49
2992    for (i=0;i<n/2;++i){
2993        sprintf(metabuf, "meta2%d", i);
2994        sprintf(bodybuf, "body2%d", i);
2995        fdb_doc_update(&doc[i], (void *)metabuf, strlen(metabuf),
2996            (void *)bodybuf, strlen(bodybuf));
2997        fdb_set(db, doc[i]);
2998    }
2999
3000    // remove document #50
3001    fdb_doc_create(&rdoc, doc[50]->key, doc[50]->keylen, doc[50]->meta,
3002                   doc[50]->metalen, NULL, 0);
3003    status = fdb_del(db, rdoc);
3004    TEST_CHK(status == FDB_RESULT_SUCCESS);
3005    fdb_doc_free(rdoc);
3006    rdoc = NULL;
3007
3008    // commit
3009    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
3010
3011    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, NULL, 0, NULL, 0);
3012    status = fdb_get_metaonly(db, rdoc);
3013    TEST_CHK(status == FDB_RESULT_SUCCESS);
3014    TEST_CHK(rdoc->deleted == false);
3015    TEST_CMP(rdoc->meta, doc[5]->meta, rdoc->metalen);
3016    // Fetch #5 doc using its offset.
3017    status = fdb_get_byoffset(db, rdoc);
3018    TEST_CHK(status == FDB_RESULT_SUCCESS);
3019    TEST_CHK(rdoc->deleted == false);
3020    TEST_CMP(rdoc->meta, doc[5]->meta, rdoc->metalen);
3021    TEST_CMP(rdoc->body, doc[5]->body, rdoc->bodylen);
3022
3023    // MB-13095
3024    fdb_doc_create(&rdoc1, NULL, 0, NULL, 0, NULL, 0);
3025    rdoc1->offset = rdoc->offset;
3026    status = fdb_get_byoffset(db, rdoc1);
3027    TEST_CHK(status == FDB_RESULT_SUCCESS);
3028    TEST_CMP(rdoc1->key, doc[5]->key, rdoc1->keylen);
3029    TEST_CMP(rdoc1->meta, doc[5]->meta, rdoc1->metalen);
3030    TEST_CMP(rdoc1->body, doc[5]->body, rdoc1->bodylen);
3031
3032    fdb_doc_free(rdoc);
3033    rdoc = NULL;
3034    fdb_doc_free(rdoc1);
3035
3036    // do compaction
3037    fdb_compact(dbfile, (char *) "./dummy2");
3038
3039    fdb_doc_create(&rdoc, doc[50]->key, doc[50]->keylen, NULL, 0, NULL, 0);
3040    status = fdb_get_metaonly(db, rdoc);
3041    TEST_CHK(status == FDB_RESULT_SUCCESS);
3042    TEST_CHK(rdoc->deleted == true);
3043    TEST_CMP(rdoc->meta, doc[50]->meta, rdoc->metalen);
3044    // Fetch #50 doc using its offset.
3045    status = fdb_get_byoffset(db, rdoc);
3046    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
3047    TEST_CHK(rdoc->deleted == true);
3048    fdb_doc_free(rdoc);
3049    rdoc = NULL;
3050
3051    // free all documents
3052    for (i=0;i<n;++i){
3053        fdb_doc_free(doc[i]);
3054    }
3055
3056    // close db file
3057    fdb_kvs_close(db);
3058    fdb_close(dbfile);
3059
3060    // free all resources
3061    fdb_shutdown();
3062
3063    memleak_end();
3064
3065    TEST_RESULT("read_doc_by_offset test");
3066}
3067
3068void purge_logically_deleted_doc_test()
3069{
3070    TEST_INIT();
3071
3072    memleak_start();
3073
3074    int i, r;
3075    int n = 10;
3076    fdb_file_handle *dbfile;
3077    fdb_kvs_handle *db;
3078    fdb_doc **doc = alca(fdb_doc*, n);
3079    fdb_doc *rdoc = NULL;
3080    fdb_status status;
3081
3082    char keybuf[256], metabuf[256], bodybuf[256];
3083
3084    // remove previous dummy files
3085    r = system(SHELL_DEL" dummy* fdb_test_config.json > errorlog.txt");
3086    (void)r;
3087
3088    fdb_config fconfig = fdb_get_default_config();
3089    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
3090    fconfig.buffercache_size = 0;
3091    fconfig.wal_threshold = 1024;
3092    fconfig.flags = FDB_OPEN_FLAG_CREATE;
3093    fconfig.purging_interval = 2;
3094    fconfig.compaction_threshold = 0;
3095
3096    // open db
3097    fdb_open(&dbfile, "./dummy1", &fconfig);
3098    fdb_kvs_open_default(dbfile, &db, &kvs_config);
3099    status = fdb_set_log_callback(db, logCallbackFunc,
3100                                  (void *) "purge_logically_deleted_doc_test");
3101    TEST_CHK(status == FDB_RESULT_SUCCESS);
3102
3103    // insert documents
3104    for (i=0;i<n;++i){
3105        sprintf(keybuf, "key%d", i);
3106        sprintf(metabuf, "meta%d", i);
3107        sprintf(bodybuf, "body%d", i);
3108        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
3109            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
3110        fdb_set(db, doc[i]);
3111    }
3112
3113    // remove document #5
3114    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, doc[5]->meta, doc[5]->metalen, NULL, 0);
3115    status = fdb_del(db, rdoc);
3116    TEST_CHK(status == FDB_RESULT_SUCCESS);
3117    fdb_doc_free(rdoc);
3118    rdoc = NULL;
3119
3120    // commit
3121    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
3122
3123    // do compaction
3124    fdb_compact(dbfile, (char *) "./dummy2");
3125
3126    // retrieve documents after compaction
3127    for (i=0;i<n;++i){
3128        // search by key
3129        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
3130        status = fdb_get(db, rdoc);
3131
3132        if (i != 5) {
3133            // updated documents
3134            TEST_CHK(status == FDB_RESULT_SUCCESS);
3135            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
3136            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
3137        } else {
3138            // removed document
3139            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
3140        }
3141        // free result document
3142        fdb_doc_free(rdoc);
3143        rdoc = NULL;
3144
3145        // retrieve metadata
3146        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
3147        status = fdb_get_metaonly(db, rdoc);
3148        if (i != 5) {
3149            TEST_CHK(status == FDB_RESULT_SUCCESS);
3150        } else {
3151            // If the above compaction takes longer than two secs (e.g., slow disk),
3152            // then, fdb_get_metaonly will return KEY_NOT_FOUND error.
3153            TEST_CHK(status == FDB_RESULT_SUCCESS ||
3154                     status == FDB_RESULT_KEY_NOT_FOUND);
3155        }
3156        fdb_doc_free(rdoc);
3157        rdoc = NULL;
3158    }
3159
3160    printf("wait for 3 seconds..\n");
3161    sleep(3);
3162
3163    // do one more compaction
3164    fdb_compact(dbfile, (char *) "./dummy3");
3165
3166    // retrieve documents after compaction
3167    for (i=0;i<n;++i){
3168        // search by key
3169        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
3170        status = fdb_get(db, rdoc);
3171
3172        if (i != 5) {
3173            // updated documents
3174            TEST_CHK(status == FDB_RESULT_SUCCESS);
3175            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
3176            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
3177        } else {
3178            // removed document
3179            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
3180        }
3181        // free result document
3182        fdb_doc_free(rdoc);
3183        rdoc = NULL;
3184
3185        // retrieve metadata
3186        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
3187        status = fdb_get_metaonly(db, rdoc);
3188        if (i != 5) {
3189            TEST_CHK(status == FDB_RESULT_SUCCESS);
3190        } else {
3191            // logically deletec document must be purged during the compaction
3192            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
3193        }
3194        fdb_doc_free(rdoc);
3195        rdoc = NULL;
3196    }
3197
3198    // close db file
3199    fdb_kvs_close(db);
3200    fdb_close(dbfile);
3201
3202    // free all documents
3203    for (i=0;i<n;++i){
3204        fdb_doc_free(doc[i]);
3205    }
3206
3207    // free all resources
3208    fdb_shutdown();
3209
3210    memleak_end();
3211
3212    TEST_RESULT("purge logically deleted doc test");
3213}
3214
3215void api_wrapper_test()
3216{
3217    TEST_INIT();
3218
3219    memleak_start();
3220
3221    int i, r;
3222    int n = 10;
3223    size_t valuelen;
3224    void *value;
3225    fdb_file_handle *dbfile;
3226    fdb_kvs_handle *db;
3227    fdb_status status;
3228
3229    char keybuf[256], bodybuf[256], temp[256];
3230
3231    // remove previous dummy files
3232    r = system(SHELL_DEL" dummy* > errorlog.txt");
3233    (void)r;
3234
3235    fdb_config fconfig = fdb_get_default_config();
3236    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
3237    fconfig.buffercache_size = 0;
3238    fconfig.wal_threshold = 1024;
3239    fconfig.flags = FDB_OPEN_FLAG_CREATE;
3240    fconfig.purging_interval = 0;
3241    fconfig.compaction_threshold = 0;
3242
3243    // open db
3244    fdb_open(&dbfile, "./dummy1", &fconfig);
3245    fdb_kvs_open_default(dbfile, &db, &kvs_config);
3246    status = fdb_set_log_callback(db, logCallbackFunc,
3247                                  (void *) "api_wrapper_test");
3248    TEST_CHK(status == FDB_RESULT_SUCCESS);
3249
3250    // error check
3251    status = fdb_set_kv(db, NULL, 0, NULL, 0);
3252    TEST_CHK(status == FDB_RESULT_INVALID_ARGS);
3253
3254    // insert key-value pairs
3255    for (i=0;i<n;++i){
3256        sprintf(keybuf, "key%d", i);
3257        sprintf(bodybuf, "body%d", i);
3258        status = fdb_set_kv(db, keybuf, strlen(keybuf), bodybuf, strlen(bodybuf));
3259        TEST_CHK(status == FDB_RESULT_SUCCESS);
3260    }
3261
3262    // remove key5
3263    sprintf(keybuf, "key%d", 5);
3264    status = fdb_del_kv(db, keybuf, strlen(keybuf));
3265    TEST_CHK(status == FDB_RESULT_SUCCESS);
3266
3267    // error check
3268    status = fdb_del_kv(db, NULL, 0);
3269    TEST_CHK(status == FDB_RESULT_INVALID_ARGS);
3270
3271    // retrieve key-value pairs
3272    for (i=0;i<n;++i){
3273        sprintf(keybuf, "key%d", i);
3274        status = fdb_get_kv(db, keybuf, strlen(keybuf), &value, &valuelen);
3275
3276        if (i != 5) {
3277            // updated documents
3278            TEST_CHK(status == FDB_RESULT_SUCCESS);
3279            sprintf(temp, "body%d", i);
3280            TEST_CMP(value, temp, valuelen);
3281            fdb_free_block(value);
3282        } else {
3283            // removed document
3284            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
3285        }
3286    }
3287
3288    // error check
3289    status = fdb_get_kv(db, NULL, 0, &value, &valuelen);
3290    TEST_CHK(status == FDB_RESULT_INVALID_ARGS);
3291
3292    status = fdb_get_kv(db, keybuf, strlen(keybuf), NULL, NULL);
3293    TEST_CHK(status == FDB_RESULT_INVALID_ARGS);
3294
3295    // close db file
3296    fdb_kvs_close(db);
3297    fdb_close(dbfile);
3298
3299    // free all resources
3300    fdb_shutdown();
3301
3302    memleak_end();
3303
3304    TEST_RESULT("API wrapper test");
3305}
3306
3307
3308void flush_before_commit_test()
3309{
3310    TEST_INIT();
3311
3312    memleak_start();
3313
3314    int i, r;
3315    int n = 30;
3316    fdb_file_handle *dbfile, *dbfile_txn;
3317    fdb_kvs_handle *db, *db_txn;
3318    fdb_doc **doc = alca(fdb_doc*, n);
3319    fdb_status status;
3320
3321    char keybuf[256], metabuf[256], bodybuf[256];
3322
3323    // remove previous dummy files
3324    r = system(SHELL_DEL" dummy* > errorlog.txt");
3325    (void)r;
3326
3327    fdb_config fconfig = fdb_get_default_config();
3328    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
3329    fconfig.buffercache_size = 0;
3330    fconfig.wal_threshold = 5;
3331    fconfig.flags = FDB_OPEN_FLAG_CREATE;
3332    fconfig.purging_interval = 0;
3333    fconfig.compaction_threshold = 0;
3334    fconfig.wal_flush_before_commit = true;
3335
3336    // open db
3337    fdb_open(&dbfile, "dummy1", &fconfig);
3338    fdb_open(&dbfile_txn, "dummy1", &fconfig);
3339    fdb_kvs_open_default(dbfile, &db, &kvs_config);
3340    fdb_kvs_open_default(dbfile_txn, &db_txn, &kvs_config);
3341    status = fdb_set_log_callback(db_txn, logCallbackFunc,
3342                                  (void *) "flush_before_commit_test");
3343    TEST_CHK(status == FDB_RESULT_SUCCESS);
3344
3345    // create docs
3346    for (i=0;i<n;++i){
3347        sprintf(keybuf, "key%d", i);
3348        sprintf(metabuf, "meta%d", i);
3349        sprintf(bodybuf, "body%d", i);
3350        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
3351                                (void*)metabuf, strlen(metabuf),
3352                                (void*)bodybuf, strlen(bodybuf));
3353    }
3354
3355    // non-transactional commit first, transactional commit next
3356    fdb_begin_transaction(dbfile_txn, FDB_ISOLATION_READ_COMMITTED);
3357    for (i=0;i<2;++i){
3358        fdb_set(db, doc[i]);
3359    }
3360    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
3361    for (i=0;i<2;++i){
3362        fdb_set(db_txn, doc[i]);
3363    }
3364    fdb_end_transaction(dbfile_txn, FDB_COMMIT_NORMAL);
3365    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
3366
3367    // transactional commit first, non-transactional commit next
3368    fdb_begin_transaction(dbfile_txn, FDB_ISOLATION_READ_COMMITTED);
3369    for (i=0;i<2;++i){
3370        fdb_set(db_txn, doc[i]);
3371    }
3372    fdb_end_transaction(dbfile_txn, FDB_COMMIT_NORMAL);
3373    for (i=0;i<2;++i){
3374        fdb_set(db, doc[i]);
3375    }
3376    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
3377    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
3378
3379    // concurrent update (non-txn commit first, txn commit next)
3380    fdb_begin_transaction(dbfile_txn, FDB_ISOLATION_READ_COMMITTED);
3381    for (i=0;i<2;++i){
3382        fdb_set(db_txn, doc[i]);
3383    }
3384    for (i=0;i<2;++i){
3385        fdb_set(db, doc[i]);
3386    }
3387    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
3388    fdb_end_transaction(dbfile_txn, FDB_COMMIT_NORMAL);
3389    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
3390
3391    // concurrent update (txn commit first, non-txn commit next)
3392    fdb_begin_transaction(dbfile_txn, FDB_ISOLATION_READ_COMMITTED);
3393    for (i=0;i<2;++i){
3394        fdb_set(db, doc[i]);
3395    }
3396    for (i=0;i<2;++i){
3397        fdb_set(db_txn, doc[i]);
3398    }
3399    fdb_end_transaction(dbfile_txn, FDB_COMMIT_NORMAL);
3400    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
3401    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
3402
3403    // begin transaction
3404    fdb_begin_transaction(dbfile_txn, FDB_ISOLATION_READ_COMMITTED);
3405
3406    // insert docs using transaction
3407    for (i=0;i<10;++i){
3408        fdb_set(db_txn, doc[i]);
3409    }
3410
3411    // insert docs without transaction
3412    for (i=10;i<20;++i){
3413        fdb_set(db, doc[i]);
3414    }
3415
3416    // do compaction
3417    fdb_compact(dbfile, "dummy2");
3418
3419    for (i=20;i<25;++i){
3420        fdb_set(db_txn, doc[i]);
3421    }
3422    // end transaction
3423    fdb_end_transaction(dbfile_txn, FDB_COMMIT_NORMAL);
3424
3425    for (i=25;i<30;++i){
3426        fdb_set(db, doc[i]);
3427    }
3428
3429    // close db file
3430    fdb_close(dbfile);
3431    fdb_close(dbfile_txn);
3432
3433    // free all documents
3434    for (i=0;i<n;++i){
3435        fdb_doc_free(doc[i]);
3436    }
3437
3438    // free all resources
3439    fdb_shutdown();
3440
3441    memleak_end();
3442
3443    TEST_RESULT("flush before commit test");
3444}
3445
3446void flush_before_commit_multi_writers_test()
3447{
3448    TEST_INIT();
3449
3450    memleak_start();
3451
3452    int i, r;
3453    int n = 10;
3454    fdb_file_handle *dbfile1, *dbfile2;
3455    fdb_kvs_handle *db1, *db2;
3456    fdb_doc **doc = alca(fdb_doc*, n);
3457    fdb_doc *rdoc = NULL;
3458    fdb_status status;
3459    fdb_config fconfig;
3460    fdb_kvs_config kvs_config;
3461
3462    char keybuf[256], metabuf[256], bodybuf[256];
3463
3464    // remove previous dummy files
3465    r = system(SHELL_DEL" dummy* > errorlog.txt");
3466    (void)r;
3467
3468    fconfig = fdb_get_default_config();
3469    fconfig.buffercache_size = 0;
3470    fconfig.wal_threshold = 8;
3471    fconfig.flags = FDB_OPEN_FLAG_CREATE;
3472    fconfig.purging_interval = 0;
3473    fconfig.compaction_threshold = 0;
3474    fconfig.wal_flush_before_commit = true;
3475
3476    kvs_config = fdb_get_default_kvs_config();
3477
3478    // open db
3479    fdb_open(&dbfile1, "dummy1", &fconfig);
3480    fdb_kvs_open(dbfile1, &db1, NULL, &kvs_config);
3481
3482    // create & insert docs
3483    for (i=0;i<n;++i){
3484        sprintf(keybuf, "key%d", i);
3485        sprintf(metabuf, "meta%d", i);
3486        sprintf(bodybuf, "body%d", i);
3487        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
3488                                (void*)metabuf, strlen(metabuf),
3489                                (void*)bodybuf, strlen(bodybuf));
3490        fdb_set(db1, doc[i]);
3491    }
3492    fdb_commit(dbfile1, FDB_COMMIT_MANUAL_WAL_FLUSH);
3493
3494    // open second writer
3495    fdb_open(&dbfile2, "dummy1", &fconfig);
3496    fdb_kvs_open(dbfile2, &db2, NULL, &kvs_config);
3497
3498    for (i=0;i<n/2;++i){
3499        sprintf(metabuf, "meta2%d", i);
3500        sprintf(bodybuf, "body2%d(db2)", i);
3501        fdb_doc_update(&doc[i], (void *)metabuf, strlen(metabuf),
3502            (void *)bodybuf, strlen(bodybuf));
3503        fdb_set(db2, doc[i]);
3504    }
3505    for (i=n/2;i<n;++i){
3506        sprintf(metabuf, "meta2%d", i);
3507        sprintf(bodybuf, "body2%d(db1)", i);
3508        fdb_doc_update(&doc[i], (void *)metabuf, strlen(metabuf),
3509            (void *)bodybuf, strlen(bodybuf));
3510        fdb_set(db1, doc[i]);
3511    }
3512
3513    // retrieve before commit
3514    for (i=0;i<n;++i){
3515        sprintf(keybuf, "key%d", i);
3516        sprintf(metabuf, "meta2%d", i);
3517        if (i < n/2) {
3518            sprintf(bodybuf, "body2%d(db2)", i);
3519        } else {
3520            sprintf(bodybuf, "body2%d(db1)", i);
3521        }
3522        // retrieve through db1
3523        fdb_doc_create(&rdoc, (void*)keybuf, strlen(keybuf),
3524                                NULL, 0, NULL, 0);
3525        status = fdb_get(db1, rdoc);
3526        TEST_CHK(status == FDB_RESULT_SUCCESS);
3527        TEST_CMP(metabuf, rdoc->meta, rdoc->metalen);
3528        TEST_CMP(bodybuf, rdoc->body, rdoc->bodylen);
3529        fdb_doc_free(rdoc);
3530        rdoc = NULL;
3531
3532        // retrieve through db2
3533        fdb_doc_create(&rdoc, (void*)keybuf, strlen(keybuf),
3534                                NULL, 0, NULL, 0);
3535        status = fdb_get(db2, rdoc);
3536        TEST_CHK(status == FDB_RESULT_SUCCESS);
3537        TEST_CMP(metabuf, r