1/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/*
3 *     Copyright 2010 Couchbase, Inc
4 *
5 *   Licensed under the Apache License, Version 2.0 (the "License");
6 *   you may not use this file except in compliance with the License.
7 *   You may obtain a copy of the License at
8 *
9 *       http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *   Unless required by applicable law or agreed to in writing, software
12 *   distributed under the License is distributed on an "AS IS" BASIS,
13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *   See the License for the specific language governing permissions and
15 *   limitations under the License.
16 */
17
18#include <stdio.h>
19#include <stdlib.h>
20#include <string.h>
21#include <stdint.h>
22#include <time.h>
23#if !defined(WIN32) && !defined(_WIN32)
24#include <unistd.h>
25#endif
26
27#include "libforestdb/forestdb.h"
28#include "test.h"
29#include "internal_types.h"
30#include "functional_util.h"
31
32void basic_test()
33{
34    TEST_INIT();
35
36    memleak_start();
37
38    int i, r;
39    int n = 10;
40    fdb_file_handle *dbfile, *dbfile_rdonly;
41    fdb_kvs_handle *db;
42    fdb_kvs_handle *db_rdonly;
43    fdb_doc **doc = alca(fdb_doc*, n);
44    fdb_doc *rdoc = NULL;
45    fdb_status status;
46
47    char keybuf[256], metabuf[256], bodybuf[256];
48
49    // remove previous dummy test files
50    r = system(SHELL_DEL" dummy* > errorlog.txt");
51    (void)r;
52
53    fdb_config fconfig = fdb_get_default_config();
54    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
55    fconfig.wal_threshold = 1024;
56    fconfig.compaction_threshold = 0;
57
58    // Read-Write mode test without a create flag.
59    fconfig.flags = 0;
60    status = fdb_open(&dbfile, "./dummy1", &fconfig);
61    TEST_CHK(status == FDB_RESULT_NO_SUCH_FILE);
62    TEST_CHK(!strcmp(fdb_error_msg(status), "no such file"));
63
64    // Read-Only mode test: Must not create new file.
65    fconfig.flags = FDB_OPEN_FLAG_RDONLY;
66    status = fdb_open(&dbfile, "./dummy1", &fconfig);
67    TEST_CHK(status == FDB_RESULT_NO_SUCH_FILE);
68
69    // Read-Only and Create mode: Must not create a new file.
70    fconfig.flags = FDB_OPEN_FLAG_RDONLY | FDB_OPEN_FLAG_CREATE;
71    status = fdb_open(&dbfile, "./dummy1", &fconfig);
72    TEST_CHK(status == FDB_RESULT_INVALID_CONFIG);
73    TEST_CHK(!strcmp(fdb_error_msg(status), "invalid configuration"));
74
75    // open and close db with a create flag.
76    fconfig.flags = FDB_OPEN_FLAG_CREATE;
77    status = fdb_open(&dbfile, "./dummy1", &fconfig);
78    TEST_CHK(status == FDB_RESULT_SUCCESS);
79    TEST_CHK(!strcmp(fdb_error_msg(status), "success"));
80    fdb_close(dbfile);
81
82    // reopen db
83    fdb_open(&dbfile, "./dummy1",&fconfig);
84    fdb_kvs_open_default(dbfile, &db, &kvs_config);
85    status = fdb_set_log_callback(db, logCallbackFunc, (void *) "basic_test");
86    TEST_CHK(status == FDB_RESULT_SUCCESS);
87
88    // insert documents
89    for (i=0;i<n;++i){
90        sprintf(keybuf, "key%d", i);
91        sprintf(metabuf, "meta%d", i);
92        sprintf(bodybuf, "body%d", i);
93        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
94            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
95        fdb_set(db, doc[i]);
96    }
97
98    // remove document #5
99    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, doc[5]->meta, doc[5]->metalen, NULL, 0);
100    status = fdb_del(db, rdoc);
101    TEST_CHK(status == FDB_RESULT_SUCCESS);
102    fdb_doc_free(rdoc);
103    rdoc = NULL;
104
105    // commit
106    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
107
108    // check the file info
109    fdb_file_info info;
110    fdb_get_file_info(dbfile, &info);
111    TEST_CHK(info.doc_count == 9);
112    TEST_CHK(info.space_used > 0);
113
114    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, NULL, 0, NULL, 0);
115    status = fdb_get_metaonly(db, rdoc);
116    TEST_CHK(status == FDB_RESULT_SUCCESS);
117    TEST_CHK(rdoc->deleted == true);
118    TEST_CMP(rdoc->meta, doc[5]->meta, rdoc->metalen);
119    fdb_doc_free(rdoc);
120    rdoc = NULL;
121
122    // close the db
123    fdb_kvs_close(db);
124    fdb_close(dbfile);
125
126    // reopen
127    fdb_open(&dbfile, "./dummy1", &fconfig);
128    fdb_kvs_open_default(dbfile, &db, &kvs_config);
129    status = fdb_set_log_callback(db, logCallbackFunc, (void *) "basic_test");
130    TEST_CHK(status == FDB_RESULT_SUCCESS);
131
132    // update document #0 and #1
133    for (i=0;i<2;++i){
134        sprintf(metabuf, "meta2%d", i);
135        sprintf(bodybuf, "body2%d", i);
136        fdb_doc_update(&doc[i], (void *)metabuf, strlen(metabuf),
137            (void *)bodybuf, strlen(bodybuf));
138        fdb_set(db, doc[i]);
139    }
140
141    // commit
142    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
143
144    // retrieve documents
145    for (i=0;i<n;++i){
146        // search by key
147        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
148        status = fdb_get(db, rdoc);
149
150        if (i != 5) {
151            // updated documents
152            TEST_CHK(status == FDB_RESULT_SUCCESS);
153            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
154            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
155        } else {
156            // removed document
157            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
158            TEST_CHK(!strcmp(fdb_error_msg(status), "key not found"));
159        }
160
161        // free result document
162        fdb_doc_free(rdoc);
163        rdoc = NULL;
164    }
165
166    // do compaction
167    fdb_compact(dbfile, (char *) "./dummy2");
168
169    // retrieve documents after compaction
170    for (i=0;i<n;++i){
171        // search by key
172        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
173        status = fdb_get(db, rdoc);
174
175        if (i != 5) {
176            // updated documents
177            TEST_CHK(status == FDB_RESULT_SUCCESS);
178            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
179            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
180        } else {
181            // removed document
182            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
183        }
184
185        // free result document
186        fdb_doc_free(rdoc);
187        rdoc = NULL;
188    }
189
190    // retrieve documents by sequence number
191    for (i=0; i < n+3; ++i){
192        // search by seq
193        fdb_doc_create(&rdoc, NULL, 0, NULL, 0, NULL, 0);
194        rdoc->seqnum = i + 1;
195        status = fdb_get_byseq(db, rdoc);
196        if ( (i>=2 && i<=4) || (i>=6 && i<=9) || (i>=11 && i<=12)) {
197            // updated documents
198            TEST_CHK(status == FDB_RESULT_SUCCESS);
199        } else {
200            // removed document
201            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
202        }
203
204        // free result document
205        fdb_doc_free(rdoc);
206        rdoc = NULL;
207    }
208
209    // update document #5 with an empty doc body.
210    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, doc[5]->meta, doc[5]->metalen, NULL, 0);
211    status = fdb_set(db, rdoc);
212    TEST_CHK(status == FDB_RESULT_SUCCESS);
213    fdb_doc_free(rdoc);
214    rdoc = NULL;
215    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
216
217    // Check document #5 with respect to metadata and doc body.
218    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, NULL, 0, NULL, 0);
219    status = fdb_get(db, rdoc);
220    TEST_CHK(status == FDB_RESULT_SUCCESS);
221    TEST_CHK(memcmp(rdoc->meta, doc[5]->meta, rdoc->metalen) == 0);
222    TEST_CHK(rdoc->body == NULL);
223    TEST_CHK(rdoc->bodylen == 0);
224    fdb_doc_free(rdoc);
225    rdoc = NULL;
226
227    // Read-Only mode test: Open succeeds if file exists, but disallow writes
228    fconfig.flags = FDB_OPEN_FLAG_RDONLY;
229    status = fdb_open(&dbfile_rdonly, "./dummy2", &fconfig);
230    TEST_CHK(status == FDB_RESULT_SUCCESS);
231    status = fdb_kvs_open_default(dbfile_rdonly, &db_rdonly, &kvs_config);
232    TEST_CHK(status == FDB_RESULT_SUCCESS);
233
234    fdb_doc_create(&rdoc, doc[0]->key, doc[0]->keylen, NULL, 0, NULL, 0);
235    status = fdb_get(db_rdonly, rdoc);
236    TEST_CHK(status == FDB_RESULT_SUCCESS);
237
238    status = fdb_set_log_callback(db_rdonly, logCallbackFunc,
239                                  (void *) "basic_test");
240    TEST_CHK(status == FDB_RESULT_SUCCESS);
241
242    status = fdb_set(db_rdonly, doc[i]);
243    TEST_CHK(status == FDB_RESULT_RONLY_VIOLATION);
244    TEST_CHK(!strcmp(fdb_error_msg(status), "database is read-only"));
245
246    status = fdb_commit(dbfile_rdonly, FDB_COMMIT_NORMAL);
247    TEST_CHK(status == FDB_RESULT_RONLY_VIOLATION);
248
249    fdb_doc_free(rdoc);
250    rdoc = NULL;
251    fdb_kvs_close(db_rdonly);
252    fdb_close(dbfile_rdonly);
253
254    // free all documents
255    for (i=0;i<n;++i){
256        fdb_doc_free(doc[i]);
257    }
258
259    // do one more compaction
260    fdb_compact(dbfile, (char *) "./dummy3");
261
262    // close db file
263    fdb_kvs_close(db);
264    fdb_close(dbfile);
265
266    // free all resources
267    fdb_shutdown();
268
269    memleak_end();
270
271    TEST_RESULT("basic test");
272}
273
274void set_get_max_keylen()
275{
276    TEST_INIT();
277    memleak_start();
278
279    int r;
280    static const int len = FDB_MAX_KEYLEN;
281    char keybuf[len];
282    void *rvalue;
283    size_t rvalue_len;
284    static const char *achar = "a";
285
286    fdb_status status;
287    fdb_file_handle *dbfile;
288    fdb_kvs_handle *db;
289    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
290    fdb_config fconfig = fdb_get_default_config();
291    fconfig.chunksize = 16;
292
293
294    r = system(SHELL_DEL" dummy* > errorlog.txt");
295    (void)r;
296
297    for (int i = 0; i < len; ++i) {
298        keybuf[i] = *achar;
299    }
300    keybuf[len-1] = '\0';
301
302    // open db
303    status = fdb_open(&dbfile, "./dummy1", &fconfig);
304    TEST_CHK(status == FDB_RESULT_SUCCESS);
305    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
306    TEST_CHK(status == FDB_RESULT_SUCCESS);
307
308    // set kv
309    status = fdb_set_kv(db, keybuf, strlen(keybuf), NULL, 0);
310    TEST_CHK(status == FDB_RESULT_SUCCESS);
311
312    // get kv
313    status = fdb_get_kv(db, keybuf, strlen(keybuf), &rvalue, &rvalue_len);
314    TEST_CHK(status == FDB_RESULT_SUCCESS);
315
316    fdb_close(dbfile);
317    fdb_shutdown();
318
319    memleak_end();
320    TEST_RESULT("set get max keylen");
321}
322
323void config_test()
324{
325    TEST_INIT();
326
327    memleak_start();
328
329    fdb_file_handle *dbfile;
330    fdb_kvs_handle *db;
331    fdb_status status;
332    fdb_config fconfig;
333    fdb_kvs_config kvs_config;
334    int nfiles = 4;
335    int i;
336    size_t bcache_space_used;
337    char fname[256];
338
339    // remove previous dummy test files
340    int r = system(SHELL_DEL" dummy* > errorlog.txt");
341    (void)r;
342
343    bcache_space_used = fdb_get_buffer_cache_used();
344    TEST_CHK(bcache_space_used == 0);
345
346    fconfig = fdb_get_default_config();
347    fconfig.buffercache_size= (uint64_t) -1;
348    status = fdb_open(&dbfile, "./dummy1", &fconfig);
349    TEST_CHK(status == FDB_RESULT_TOO_BIG_BUFFER_CACHE);
350
351    fconfig = fdb_get_default_config();
352    fconfig.max_writer_lock_prob = 120;
353    status = fdb_open(&dbfile, "./dummy1", &fconfig);
354    TEST_CHK(status == FDB_RESULT_INVALID_CONFIG);
355
356    fconfig = fdb_get_default_config();
357    kvs_config = fdb_get_default_kvs_config();
358    for (i = nfiles; i; --i) {
359        sprintf(fname, "dummy%d", i);
360        status = fdb_open(&dbfile, fname, &fconfig);
361        TEST_CHK(status == FDB_RESULT_SUCCESS);
362        status = fdb_kvs_open(dbfile, &db, "justonekv", &kvs_config);
363        TEST_CHK(status == FDB_RESULT_SUCCESS);
364
365        bcache_space_used = fdb_get_buffer_cache_used();
366
367        // Ensure just one block is used from the buffercache to store KV name
368        // DB header and it does not change since files are duly closed
369        TEST_CHK(bcache_space_used == fconfig.blocksize);
370
371        status = fdb_close(dbfile);
372        TEST_CHK(status == FDB_RESULT_SUCCESS);
373    }
374
375    status = fdb_open(&dbfile, fname, &fconfig);
376    TEST_CHK(status == FDB_RESULT_SUCCESS);
377    status = fdb_kvs_open(dbfile, &db, "justonekv", &kvs_config);
378    TEST_CHK(status == FDB_RESULT_SUCCESS);
379    status = fdb_set_kv(db, (void*)"key", 3, (void*)"body", 5);
380    TEST_CHK(status == FDB_RESULT_SUCCESS);
381
382    bcache_space_used = fdb_get_buffer_cache_used();
383
384    // Two blocks must be used - 1 by DB header created earlier and
385    // One for the document block created by the fdb_set_kv
386    TEST_CHK(bcache_space_used == fconfig.blocksize * 2);
387
388    fdb_close(dbfile);
389
390    fdb_shutdown();
391
392    memleak_end();
393    TEST_RESULT("forestdb config test");
394}
395
396void deleted_doc_get_api_test()
397{
398    TEST_INIT();
399    memleak_start();
400
401    int r;
402    fdb_file_handle *dbfile;
403    fdb_kvs_handle *db;
404    fdb_doc _doc;
405    fdb_doc *doc = &_doc;
406    fdb_doc *rdoc;
407    fdb_status status;
408    fdb_config fconfig;
409    fdb_kvs_config kvs_config;
410    char keybuf[256], bodybuf[256];
411
412    r = system(SHELL_DEL " dummy* > errorlog.txt");
413    (void)r;
414
415    memset(doc, 0, sizeof(fdb_doc));
416    doc->key = &keybuf[0];
417    doc->body = &bodybuf[0];
418
419    // open dbfile
420    fconfig = fdb_get_default_config();
421    kvs_config = fdb_get_default_kvs_config();
422    status = fdb_open(&dbfile, "./dummy1", &fconfig);
423    TEST_CHK(status == FDB_RESULT_SUCCESS);
424    status = fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
425    TEST_CHK(status == FDB_RESULT_SUCCESS);
426
427    sprintf(keybuf, "key");
428    sprintf(bodybuf, "body");
429    doc->keylen = strlen(keybuf);
430    doc->bodylen = strlen(bodybuf);
431    status = fdb_set(db, doc);
432    TEST_CHK(status == FDB_RESULT_SUCCESS);
433
434    // Commit the doc so it goes into main index
435    status = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
436    TEST_CHK(status == FDB_RESULT_SUCCESS);
437
438    // Delete the doc
439    status = fdb_del(db, doc);
440    TEST_CHK(status == FDB_RESULT_SUCCESS);
441
442    // Commit the doc with wal flush so the delete is appended into the file
443    status = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
444    TEST_CHK(status == FDB_RESULT_SUCCESS);
445
446
447    fdb_doc_create(&rdoc, keybuf, doc->keylen, NULL, 0, NULL, 0);
448
449    // Deleted document should be accessible via fdb_get_metaonly()
450    status = fdb_get_metaonly(db, rdoc);
451    TEST_CHK(status == FDB_RESULT_SUCCESS);
452    TEST_CHK(rdoc->deleted);
453    rdoc->deleted = false;
454
455    // Deleted document should be accessible via fdb_get_metaonly_byseq()
456    status = fdb_get_metaonly_byseq(db, rdoc);
457    TEST_CHK(status == FDB_RESULT_SUCCESS);
458    TEST_CHK(rdoc->deleted);
459    rdoc->deleted = false;
460
461    // Deleted document should be accessible via fdb_get_byoffset()
462    // But the return code must be FDB_RESULT_KEY_NOT_FOUND!
463    status = fdb_get_byoffset(db, rdoc);
464    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
465    TEST_CHK(rdoc->deleted);
466    rdoc->deleted = false;
467
468    // Deleted document should NOT be accessible via fdb_get()
469    status = fdb_get(db, rdoc);
470    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
471    TEST_CHK(!rdoc->deleted);
472    rdoc->deleted = false;
473
474    status = fdb_get_byseq(db, rdoc);
475    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
476    TEST_CHK(!rdoc->deleted);
477
478    fdb_doc_free(rdoc);
479    // close without commit
480    status = fdb_kvs_close(db);
481    TEST_CHK(status == FDB_RESULT_SUCCESS);
482    status = fdb_close(dbfile);
483    TEST_CHK(status == FDB_RESULT_SUCCESS);
484    fdb_shutdown();
485    memleak_end();
486    TEST_RESULT("deleted doc get api test");
487}
488
489void large_batch_write_no_commit_test()
490{
491    TEST_INIT();
492    memleak_start();
493
494    int i, r;
495    int n = 500000;
496    fdb_file_handle *dbfile;
497    fdb_kvs_handle *db;
498    fdb_doc **doc = (fdb_doc **) malloc(sizeof(fdb_doc *) * n);
499    fdb_status status;
500    fdb_config fconfig;
501    fdb_kvs_config kvs_config;
502    char keybuf[256], metabuf[256], bodybuf[256];
503
504    r = system(SHELL_DEL " dummy* > errorlog.txt");
505    (void)r;
506
507    // open dbfile
508    fconfig = fdb_get_default_config();
509    kvs_config = fdb_get_default_kvs_config();
510    status = fdb_open(&dbfile, "./dummy1", &fconfig);
511    TEST_CHK(status == FDB_RESULT_SUCCESS);
512    status = fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
513    TEST_CHK(status == FDB_RESULT_SUCCESS);
514
515    // Write 500K docs to eject and flush some dirty pages into disk.
516    for (i=0;i<n;++i){
517        sprintf(keybuf, "key%128d", i);
518        sprintf(metabuf, "meta%128d", i);
519        sprintf(bodybuf, "body%128d", i);
520        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
521            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
522        fdb_set(db, doc[i]);
523        fdb_doc_free(doc[i]);
524    }
525
526    // close without commit
527    status = fdb_kvs_close(db);
528    TEST_CHK(status == FDB_RESULT_SUCCESS);
529    status = fdb_close(dbfile);
530    TEST_CHK(status == FDB_RESULT_SUCCESS);
531
532    status = fdb_open(&dbfile, "./dummy1", &fconfig);
533    TEST_CHK(status == FDB_RESULT_NO_DB_HEADERS ||
534             status == FDB_RESULT_SUCCESS); // No dirty pages are flushed into disk.
535    if (status == FDB_RESULT_SUCCESS) {
536        status = fdb_close(dbfile);
537        TEST_CHK(status == FDB_RESULT_SUCCESS);
538    }
539
540    free(doc);
541    fdb_shutdown();
542    memleak_end();
543    TEST_RESULT("large batch write test with no commits");
544}
545
546void set_get_meta_test()
547{
548    TEST_INIT();
549    memleak_start();
550
551    int r;
552    char keybuf[256];
553    fdb_file_handle *dbfile;
554    fdb_kvs_handle *db;
555    fdb_doc *rdoc;
556    fdb_status status;
557    fdb_config fconfig = fdb_get_default_config();
558    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
559    fconfig.wal_threshold = 1024;
560    fconfig.flags = FDB_OPEN_FLAG_CREATE;
561
562    // remove previous dummy files
563    r = system(SHELL_DEL" dummy* > errorlog.txt");
564    (void)r;
565
566    // open db
567    fdb_open(&dbfile, "./dummy1", &fconfig);
568    fdb_kvs_open(dbfile, &db, "db1", &kvs_config);
569
570    sprintf(keybuf, "key%d", 0);
571    fdb_doc_create(&rdoc, keybuf, strlen(keybuf), NULL, 0, NULL, 0);
572    fdb_set(db, rdoc);
573    status = fdb_get(db, rdoc);
574    assert(status == FDB_RESULT_SUCCESS);
575    status = fdb_get_byoffset(db, rdoc);
576    TEST_CHK(status == FDB_RESULT_SUCCESS);
577    status = fdb_get_metaonly(db, rdoc);
578    assert(status == FDB_RESULT_SUCCESS);
579    status = fdb_get_metaonly_byseq(db, rdoc);
580    TEST_CHK(status == FDB_RESULT_SUCCESS);
581
582    fdb_del(db, rdoc);
583    status = fdb_get(db, rdoc);
584    assert(status == FDB_RESULT_KEY_NOT_FOUND);
585    assert(rdoc->deleted == true);
586
587    status = fdb_get_metaonly(db, rdoc);
588    assert(status == FDB_RESULT_SUCCESS);
589    assert(rdoc->deleted == true);
590
591    status = fdb_get_metaonly_byseq(db, rdoc);
592    TEST_CHK(status == FDB_RESULT_SUCCESS);
593    assert(rdoc->deleted == true);
594
595    status = fdb_get_byoffset(db, rdoc);
596    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
597    assert(rdoc->deleted == true);
598
599
600    fdb_doc_free(rdoc);
601    fdb_kvs_close(db);
602    fdb_close(dbfile);
603    fdb_shutdown();
604
605    memleak_end();
606    TEST_RESULT("set get meta test");
607}
608
609void long_filename_test()
610{
611    TEST_INIT();
612    memleak_start();
613
614    int i, j, r;
615    int n=15, m=1000;
616    char keyword[] = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
617    char filename[4096], cmd[4096], temp[4096];
618    fdb_file_handle *dbfile;
619    fdb_kvs_handle *db;
620    fdb_config config;
621    fdb_kvs_config kvs_config;
622    fdb_status s;
623    size_t rvalue_len;
624    char key[256], value[256];
625    void *rvalue;
626
627    config = fdb_get_default_config();
628    kvs_config = fdb_get_default_kvs_config();
629    sprintf(temp, SHELL_DMT"%s", keyword);
630
631    // filename longer than 1024 bytes
632    sprintf(filename, "%s", keyword);
633    while (strlen(filename) < 1024) {
634        strcat(filename, keyword);
635    }
636    s = fdb_open(&dbfile, filename, &config);
637    TEST_CHK(s == FDB_RESULT_TOO_LONG_FILENAME);
638
639    // make nested directories for long path
640    // but shorter than 1024 bytes (windows: 256 bytes)
641    sprintf(cmd, SHELL_RMDIR" %s", keyword);
642    r = system(cmd);
643    (void)r;
644    for (i=0;i<n;++i) {
645        sprintf(cmd, SHELL_MKDIR" %s", keyword);
646        for (j=0;j<i;++j){
647            strcat(cmd, temp);
648        }
649        if (strlen(cmd) > SHELL_MAX_PATHLEN) break;
650        r = system(cmd);
651        (void)r;
652    }
653
654    // create DB file
655    sprintf(filename, "%s", keyword);
656    for (j=0;j<i-1;++j){
657        strcat(filename, temp);
658    }
659    strcat(filename, SHELL_DMT"dbfile");
660    s = fdb_open(&dbfile, filename, &config);
661    TEST_CHK(s == FDB_RESULT_SUCCESS);
662    s = fdb_kvs_open_default(dbfile, &db, &kvs_config);
663    TEST_CHK(s == FDB_RESULT_SUCCESS);
664
665    // === write ===
666    for (i=0;i<m;++i){
667        sprintf(key, "key%08d", i);
668        sprintf(value, "value%08d", i);
669        s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
670        TEST_CHK(s == FDB_RESULT_SUCCESS);
671    }
672    s = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
673    TEST_CHK(s == FDB_RESULT_SUCCESS);
674
675    // === read ===
676    for (i=0;i<m;++i){
677        sprintf(key, "key%08d", i);
678        s = fdb_get_kv(db, key, strlen(key)+1, &rvalue, &rvalue_len);
679        TEST_CHK(s == FDB_RESULT_SUCCESS);
680        fdb_free_block(rvalue);
681    }
682
683    s = fdb_kvs_close(db);
684    TEST_CHK(s == FDB_RESULT_SUCCESS);
685    s = fdb_close(dbfile);
686    TEST_CHK(s == FDB_RESULT_SUCCESS);
687    s = fdb_shutdown();
688    TEST_CHK(s == FDB_RESULT_SUCCESS);
689
690    sprintf(cmd, SHELL_RMDIR" %s", keyword);
691    r = system(cmd);
692    (void)r;
693
694    memleak_end();
695    TEST_RESULT("long filename test");
696}
697
698void error_to_str_test()
699{
700    TEST_INIT();
701    memleak_start();
702    int i;
703    const char *err_msg;
704
705    for (i = FDB_RESULT_SUCCESS; i >= FDB_RESULT_AIO_GETEVENTS_FAIL; --i) {
706        err_msg = fdb_error_msg((fdb_status)i);
707        // Verify that all error codes have corresponding error messages
708        TEST_CHK(strcmp(err_msg, "unknown error"));
709    }
710
711    err_msg = fdb_error_msg((fdb_status)i);
712    // Verify that the last error code has been checked
713    TEST_CHK(!strcmp(err_msg, "unknown error"));
714
715    memleak_end();
716    TEST_RESULT("error to string message test");
717}
718
719void seq_tree_exception_test()
720{
721    TEST_INIT();
722
723    memleak_start();
724
725    int i, r;
726    int n = 10;
727    fdb_file_handle *dbfile;
728    fdb_kvs_handle *db;
729    fdb_doc **doc = alca(fdb_doc*, n);
730    fdb_doc *rdoc = NULL;
731    fdb_status status;
732    fdb_iterator *it;
733
734    char keybuf[256], metabuf[256], bodybuf[256];
735
736    // remove previous dummy files
737    r = system(SHELL_DEL" dummy* > errorlog.txt");
738    (void)r;
739
740    fdb_config fconfig = fdb_get_default_config();
741    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
742    fconfig.seqtree_opt = FDB_SEQTREE_NOT_USE;
743
744    // open db
745    fdb_open(&dbfile, "./dummy1", &fconfig);
746    fdb_kvs_open_default(dbfile, &db, &kvs_config);
747    status = fdb_set_log_callback(db, logCallbackFunc,
748                                  (void *) "seq_tree_exception_test");
749    TEST_CHK(status == FDB_RESULT_SUCCESS);
750
751    // insert documents
752    for (i=0;i<n;++i){
753        sprintf(keybuf, "key%d", i);
754        sprintf(metabuf, "meta%d", i);
755        sprintf(bodybuf, "body%d", i);
756        fdb_doc_create(&doc[i],
757                       (void *)keybuf,  strlen(keybuf),
758                       (void *)metabuf, strlen(metabuf),
759                       (void *)bodybuf, strlen(bodybuf));
760        fdb_set(db, doc[i]);
761    }
762
763    // commit
764    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
765
766    // close the db
767    fdb_kvs_close(db);
768    fdb_close(dbfile);
769
770    // reopen with seq tree option
771    fconfig.seqtree_opt = FDB_SEQTREE_USE;
772    status = fdb_open(&dbfile, "./dummy1", &fconfig);
773    // must succeed
774    TEST_CHK(status == FDB_RESULT_SUCCESS);
775    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
776    TEST_CHK(status == FDB_RESULT_SUCCESS);
777    status = fdb_set_log_callback(db, logCallbackFunc,
778                                  (void *) "seq_tree_exception_test");
779    TEST_CHK(status == FDB_RESULT_SUCCESS);
780
781    // search by seq
782    fdb_doc_create(&rdoc, NULL, 0, NULL, 0, NULL, 0);
783    rdoc->seqnum = 1;
784    status = fdb_get_byseq(db, rdoc);
785    // must fail
786    TEST_CHK(status != FDB_RESULT_SUCCESS);
787
788    // search meta by seq
789    status = fdb_get_metaonly_byseq(db, rdoc);
790    // must fail
791    TEST_CHK(status != FDB_RESULT_SUCCESS);
792
793    // init iterator by seq
794    status = fdb_iterator_sequence_init(db , &it, 0, 0, FDB_ITR_NONE);
795    // must fail
796    TEST_CHK(status != FDB_RESULT_SUCCESS);
797
798    // close db file
799    fdb_kvs_close(db);
800    fdb_close(dbfile);
801
802    // free all documents
803    free(rdoc);
804    for (i=0;i<n;++i){
805        fdb_doc_free(doc[i]);
806    }
807
808    // remove previous dummy files
809    r = system(SHELL_DEL" dummy* > errorlog.txt");
810    (void)r;
811
812    // open db
813    fconfig.seqtree_opt = FDB_SEQTREE_USE;
814    fdb_open(&dbfile, "./dummy1", &fconfig);
815    fdb_kvs_open_default(dbfile, &db, &kvs_config);
816    status = fdb_set_log_callback(db, logCallbackFunc,
817                                  (void *) "seq_tree_exception_test");
818    TEST_CHK(status == FDB_RESULT_SUCCESS);
819
820    // insert documents
821    for (i=0;i<n;++i){
822        sprintf(keybuf, "key%d", i);
823        sprintf(metabuf, "meta%d", i);
824        sprintf(bodybuf, "body%d", i);
825        fdb_doc_create(&doc[i],
826                       (void *)keybuf,  strlen(keybuf),
827                       (void *)metabuf, strlen(metabuf),
828                       (void *)bodybuf, strlen(bodybuf));
829        fdb_set(db, doc[i]);
830    }
831
832    // commit
833    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
834
835    // close the db
836    fdb_kvs_close(db);
837    fdb_close(dbfile);
838
839    // reopen with an option disabling seq tree
840    fconfig.seqtree_opt = FDB_SEQTREE_NOT_USE;
841    status = fdb_open(&dbfile, "./dummy1", &fconfig);
842    // must succeed
843    TEST_CHK(status == FDB_RESULT_SUCCESS);
844    status = fdb_close(dbfile);
845    TEST_CHK(status == FDB_RESULT_SUCCESS);
846
847    // free all documents
848    for (i=0;i<n;++i){
849        fdb_doc_free(doc[i]);
850    }
851
852    // free all resources
853    fdb_shutdown();
854
855    memleak_end();
856
857    TEST_RESULT("sequence tree exception test");
858}
859
860void wal_commit_test()
861{
862    TEST_INIT();
863
864    memleak_start();
865
866    int i, r;
867    int n = 10;
868    fdb_file_handle *dbfile;
869    fdb_kvs_handle *db;
870    fdb_doc **doc = alca(fdb_doc*, n);
871    fdb_doc *rdoc = NULL;
872    fdb_status status;
873
874    char keybuf[256], metabuf[256], bodybuf[256];
875
876    // remove previous dummy files
877    r = system(SHELL_DEL" dummy* > errorlog.txt");
878    (void)r;
879
880    fdb_config fconfig = fdb_get_default_config();
881    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
882    fconfig.buffercache_size = 0;
883    fconfig.wal_threshold = 1024;
884    fconfig.flags = FDB_OPEN_FLAG_CREATE;
885    fconfig.compaction_threshold = 0;
886
887    // open db
888    fdb_open(&dbfile, "./dummy1", &fconfig);
889    fdb_kvs_open_default(dbfile, &db, &kvs_config);
890    status = fdb_set_log_callback(db, logCallbackFunc, (void *) "wal_commit_test");
891    TEST_CHK(status == FDB_RESULT_SUCCESS);
892
893    // insert half documents
894    for (i=0;i<n/2;++i){
895        sprintf(keybuf, "key%d", i);
896        sprintf(metabuf, "meta%d", i);
897        sprintf(bodybuf, "body%d", i);
898        fdb_doc_create(&doc[i], (void *)keybuf, strlen(keybuf),
899            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
900        fdb_set(db, doc[i]);
901    }
902
903    // commit
904    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
905
906    // insert the other half documents
907    for (i=n/2;i<n;++i){
908        sprintf(keybuf, "key%d", i);
909        sprintf(metabuf, "meta%d", i);
910        sprintf(bodybuf, "body%d", i);
911        fdb_doc_create(&doc[i], (void *)keybuf, strlen(keybuf),
912            (void *)metabuf, strlen(metabuf), (void *)bodybuf, strlen(bodybuf));
913        fdb_set(db, doc[i]);
914    }
915
916    // close the db
917    fdb_kvs_close(db);
918    fdb_close(dbfile);
919
920    // reopen
921    fdb_open(&dbfile, "./dummy1", &fconfig);
922    fdb_kvs_open_default(dbfile, &db, &kvs_config);
923    status = fdb_set_log_callback(db, logCallbackFunc, (void *) "wal_commit_test");
924    TEST_CHK(status == FDB_RESULT_SUCCESS);
925
926    // retrieve documents
927    for (i=0;i<n;++i){
928        // search by key
929        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
930        status = fdb_get(db, rdoc);
931
932        if (i < n/2) {
933            // committed documents
934            TEST_CHK(status == FDB_RESULT_SUCCESS);
935            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
936            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
937        } else {
938            // not committed document
939            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
940        }
941
942        // free result document
943        fdb_doc_free(rdoc);
944        rdoc = NULL;
945    }
946
947    // free all documents
948    for (i=0;i<n;++i){
949        fdb_doc_free(doc[i]);
950    }
951
952    // close db file
953    fdb_kvs_close(db);
954    fdb_close(dbfile);
955
956    // free all resources
957    fdb_shutdown();
958
959    memleak_end();
960
961    TEST_RESULT("WAL commit test");
962}
963
964void db_close_and_remove()
965{
966
967    TEST_INIT();
968    memleak_start();
969
970    int i, r;
971    int n = 10;
972    fdb_file_handle *dbfile;
973    fdb_kvs_handle *db;
974    fdb_doc **doc = alca(fdb_doc *, n);
975    fdb_status status;
976    fdb_config fconfig;
977    fdb_kvs_config kvs_config;
978    char keybuf[256], metabuf[256], bodybuf[256];
979
980    r = system(SHELL_DEL " dummy* > errorlog.txt");
981    (void)r;
982
983    // open dbfile
984    fconfig = fdb_get_default_config();
985    kvs_config = fdb_get_default_kvs_config();
986    fconfig.cleanup_cache_onclose = false;
987    fdb_open(&dbfile, "./dummy1", &fconfig);
988    fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
989
990    // write to db
991    for (i=0;i<n;++i){
992        sprintf(keybuf, "key%d", i);
993        sprintf(metabuf, "meta%d", i);
994        sprintf(bodybuf, "body%d", i);
995        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
996            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
997        fdb_set(db, doc[i]);
998        fdb_doc_free(doc[i]);
999    }
1000    status = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1001    TEST_CHK(status == FDB_RESULT_SUCCESS);
1002
1003    // close
1004    fdb_kvs_close(db);
1005    fdb_close(dbfile);
1006
1007    // remove dbfile
1008    r = system(SHELL_DEL " dummy* > errorlog.txt");
1009    (void)r;
1010
1011    // re-open read-only
1012    fconfig.flags = FDB_OPEN_FLAG_RDONLY;
1013    status = fdb_open(&dbfile, "./dummy1", &fconfig);
1014    TEST_CHK(status == FDB_RESULT_NO_SUCH_FILE);
1015
1016    fdb_shutdown();
1017    memleak_end();
1018    TEST_RESULT("db close and remove");
1019}
1020
1021void db_drop_test()
1022{
1023    TEST_INIT();
1024
1025    memleak_start();
1026
1027    int i, r;
1028    int n = 3;
1029    fdb_file_handle *dbfile;
1030    fdb_kvs_handle *db;
1031    fdb_doc **doc = alca(fdb_doc *, n);
1032    fdb_doc *rdoc = NULL;
1033    fdb_status status;
1034
1035    char keybuf[256], metabuf[256], bodybuf[256];
1036
1037    // remove previous dummy files
1038    r = system(SHELL_DEL " dummy* > errorlog.txt");
1039    (void)r;
1040
1041    fdb_config fconfig = fdb_get_default_config();
1042    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1043    fconfig.buffercache_size = 16777216;
1044    fconfig.wal_threshold = 1024;
1045    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1046    fconfig.compaction_threshold = 0;
1047
1048    // open db
1049    fdb_open(&dbfile, "./dummy1", &fconfig);
1050    fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
1051    status = fdb_set_log_callback(db, logCallbackFunc,
1052                                  (void *) "db_drop_test");
1053    TEST_CHK(status == FDB_RESULT_SUCCESS);
1054
1055    // insert first two documents
1056    for (i=0;i<2;++i){
1057        sprintf(keybuf, "key%d", i);
1058        sprintf(metabuf, "meta%d", i);
1059        sprintf(bodybuf, "body%d", i);
1060        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1061            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1062        fdb_set(db, doc[i]);
1063    }
1064
1065    // commit
1066    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1067    fdb_kvs_close(db);
1068    fdb_close(dbfile);
1069
1070    // Remove the database file manually.
1071    r = system(SHELL_DEL " dummy1 > errorlog.txt");
1072    (void)r;
1073
1074    // Open the empty db with the same name.
1075    fdb_open(&dbfile, "./dummy1", &fconfig);
1076    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1077    status = fdb_set_log_callback(db, logCallbackFunc,
1078                                  (void *) "db_drop_test");
1079    TEST_CHK(status == FDB_RESULT_SUCCESS);
1080
1081    // now insert a new doc.
1082    sprintf(keybuf, "key%d", 0);
1083    sprintf(metabuf, "meta%d", 0);
1084    sprintf(bodybuf, "body%d", 0);
1085    fdb_doc_free(doc[0]);
1086    fdb_doc_create(&doc[0], (void*)keybuf, strlen(keybuf),
1087        (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1088    fdb_set(db, doc[0]);
1089
1090    // commit
1091    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1092
1093    // search by key
1094    fdb_doc_create(&rdoc, doc[0]->key, doc[0]->keylen, NULL, 0, NULL, 0);
1095    status = fdb_get(db, rdoc);
1096    TEST_CHK(status == FDB_RESULT_SUCCESS);
1097    // Make sure that a doc seqnum starts with one.
1098    TEST_CHK(rdoc->seqnum == 1);
1099
1100    fdb_kvs_close(db);
1101    fdb_close(dbfile);
1102
1103    // free all documents
1104    fdb_doc_free(rdoc);
1105    rdoc = NULL;
1106    for (i=0;i<2;++i){
1107        fdb_doc_free(doc[i]);
1108    }
1109
1110    // free all resources
1111    fdb_shutdown();
1112
1113    memleak_end();
1114
1115    TEST_RESULT("Database drop test");
1116}
1117
1118void db_destroy_test()
1119{
1120    TEST_INIT();
1121
1122    memleak_start();
1123
1124    int i, r;
1125    int n = 30;
1126    fdb_file_handle *dbfile, *dbfile2;
1127    fdb_kvs_handle *db, *db2;
1128    fdb_doc **doc = alca(fdb_doc *, n);
1129    fdb_doc *rdoc = NULL;
1130    fdb_status status;
1131    fdb_config fconfig;
1132    fdb_kvs_config kvs_config;
1133
1134    char keybuf[256], metabuf[256], bodybuf[256];
1135
1136    // remove previous dummy files
1137    r = system(SHELL_DEL " dummy* > errorlog.txt");
1138    (void)r;
1139
1140    fconfig = fdb_get_default_config();
1141    kvs_config = fdb_get_default_kvs_config();
1142    fconfig.buffercache_size = 16777216;
1143    fconfig.wal_threshold = 1024;
1144    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1145    fconfig.compaction_threshold = 0;
1146
1147    // open db
1148    fdb_open(&dbfile, "./dummy1", &fconfig);
1149    fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
1150    status = fdb_set_log_callback(db, logCallbackFunc,
1151                                  (void *) "db_destroy_test");
1152    TEST_CHK(status == FDB_RESULT_SUCCESS);
1153
1154    // insert 30 documents
1155    for (i=0;i<n;++i){
1156        sprintf(keybuf, "key%d", i);
1157        sprintf(metabuf, "meta%d", i);
1158        sprintf(bodybuf, "body%d", i);
1159        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1160            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1161        fdb_set(db, doc[i]);
1162    }
1163
1164    // commit
1165    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1166
1167    // Open the empty db with the same name.
1168    fdb_open(&dbfile2, "./dummy2", &fconfig);
1169    fdb_kvs_open(dbfile2, &db2, NULL, &kvs_config);
1170    status = fdb_set_log_callback(db2, logCallbackFunc,
1171                                  (void *) "db_destroy_test");
1172    TEST_CHK(status == FDB_RESULT_SUCCESS);
1173    // insert 30 documents
1174    for (i=0;i<n;++i){
1175        fdb_set(db2, doc[i]);
1176    }
1177
1178    // commit
1179    fdb_commit(dbfile2, FDB_COMMIT_NORMAL);
1180
1181    // Only close db not db2 and try to destroy
1182    fdb_close(dbfile);
1183
1184    status = fdb_destroy("./dummy2", &fconfig);
1185    TEST_CHK(status == FDB_RESULT_FILE_IS_BUSY);
1186
1187    //Now close the open db file
1188    fdb_close(dbfile2);
1189
1190    status = fdb_destroy("./dummy1", &fconfig);
1191    TEST_CHK(status == FDB_RESULT_SUCCESS);
1192
1193    // Open the same db with the same names.
1194    fdb_open(&dbfile, "./dummy1", &fconfig);
1195    fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
1196    status = fdb_set_log_callback(db, logCallbackFunc,
1197                                  (void *) "db_destroy_test");
1198    TEST_CHK(status == FDB_RESULT_SUCCESS);
1199
1200    // search by key
1201    fdb_doc_create(&rdoc, doc[0]->key, doc[0]->keylen, NULL, 0, NULL, 0);
1202    status = fdb_get(db, rdoc);
1203    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
1204    fdb_close(dbfile);
1205
1206    // free all documents
1207    fdb_doc_free(rdoc);
1208    rdoc = NULL;
1209    for (i=0;i<n;++i){
1210        fdb_doc_free(doc[i]);
1211    }
1212
1213    // free all resources
1214    fdb_shutdown();
1215
1216    memleak_end();
1217
1218    TEST_RESULT("Database destroy test");
1219}
1220
1221
1222void operational_stats_test(bool multi_kv)
1223{
1224    TEST_INIT();
1225
1226    memleak_start();
1227
1228    int i, r;
1229    int n = 10;
1230    int num_kv = 4;
1231    fdb_file_handle *dbfile;
1232    fdb_kvs_handle **db = alca(fdb_kvs_handle*, num_kv);
1233    fdb_doc **doc = alca(fdb_doc*, n);
1234    fdb_doc real_doc;
1235    fdb_doc *rdoc = &real_doc;
1236    fdb_status status;
1237    fdb_iterator *iterator;
1238    fdb_kvs_ops_info info, rinfo;
1239
1240    char keybuf[256], bodybuf[256];
1241    memset(&info, 0, sizeof(fdb_kvs_ops_info));
1242    memset(&real_doc, 0, sizeof(fdb_doc));
1243    real_doc.key = &keybuf;
1244    real_doc.body = &bodybuf;
1245
1246    // remove previous dummy files
1247    r = system(SHELL_DEL" dummy* > errorlog.txt");
1248    (void)r;
1249
1250    fdb_config fconfig = fdb_get_default_config();
1251    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1252
1253    fconfig.buffercache_size = 0;
1254    fconfig.wal_threshold = 1024;
1255    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1256    fconfig.compaction_threshold = 0;
1257    fconfig.multi_kv_instances = multi_kv;
1258    r = 0;
1259
1260    fdb_open(&dbfile, "./dummy1", &fconfig);
1261    if (multi_kv) {
1262        num_kv = 4;
1263        for (r = num_kv - 1; r >= 0; --r) {
1264            char tmp[16];
1265            sprintf(tmp, "kv%d", r);
1266            status = fdb_kvs_open(dbfile, &db[r], tmp, &kvs_config);
1267            TEST_CHK(status == FDB_RESULT_SUCCESS);
1268            status = fdb_set_log_callback(db[r], logCallbackFunc,
1269                                          (void *) "operational_stats_test");
1270            TEST_CHK(status == FDB_RESULT_SUCCESS);
1271        }
1272    } else {
1273        num_kv = 1;
1274        status = fdb_kvs_open_default(dbfile, &db[r], &kvs_config);
1275        TEST_CHK(status == FDB_RESULT_SUCCESS);
1276        status = fdb_set_log_callback(db[r], logCallbackFunc,
1277                (void *) "operational_stats_test");
1278        TEST_CHK(status == FDB_RESULT_SUCCESS);
1279    }
1280
1281    for (i = 0; i < n; ++i){
1282        sprintf(keybuf, "key%d", i);
1283        sprintf(bodybuf, "body%d", i);
1284        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf) + 1, NULL, 0,
1285            (void*)bodybuf, strlen(bodybuf)+1);
1286        for (r = num_kv - 1; r >= 0; --r) {
1287            status = fdb_set(db[r], doc[i]);
1288            TEST_CHK(status == FDB_RESULT_SUCCESS);
1289            status = fdb_get_kvs_ops_info(db[r], &rinfo);
1290            TEST_CHK(status == FDB_RESULT_SUCCESS);
1291            info.num_sets = i + 1;
1292            TEST_CMP(&rinfo, &info, sizeof(fdb_kvs_ops_info));
1293        }
1294    }
1295
1296    for (r = num_kv - 1; r >= 0; --r) {
1297        // range scan (before flushing WAL)
1298        fdb_iterator_init(db[r], &iterator, NULL, 0, NULL, 0, 0x0);
1299        i = 0;
1300        do {
1301            status = fdb_iterator_get(iterator, &rdoc);
1302            TEST_CHK(status == FDB_RESULT_SUCCESS);
1303            TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
1304            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
1305            status = fdb_get_kvs_ops_info(db[r], &rinfo);
1306            TEST_CHK(status == FDB_RESULT_SUCCESS);
1307            ++info.num_iterator_gets;
1308            ++info.num_iterator_moves;
1309            TEST_CMP(&rinfo, &info, sizeof(fdb_kvs_ops_info));
1310            ++i;
1311        } while(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
1312        ++info.num_iterator_moves; // account for the last move that failed
1313        fdb_iterator_close(iterator);
1314
1315        fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1316        ++info.num_commits;
1317
1318        status = fdb_get_kvs_ops_info(db[r], &rinfo);
1319        TEST_CHK(status == FDB_RESULT_SUCCESS);
1320        TEST_CMP(&rinfo, &info, sizeof(fdb_kvs_ops_info));
1321
1322        if (r) {
1323            info.num_iterator_gets = 0;
1324            info.num_iterator_moves = 0;
1325        }
1326    }
1327
1328    ++info.num_compacts;
1329    // do compaction
1330    fdb_compact(dbfile, (char *) "./dummy2");
1331
1332    status = fdb_get_kvs_ops_info(db[0], &rinfo);
1333    TEST_CHK(status == FDB_RESULT_SUCCESS);
1334    TEST_CMP(&rinfo, &info, sizeof(fdb_kvs_ops_info));
1335
1336    for (i = 0; i < n; ++i){
1337        sprintf(keybuf, "key%d", i);
1338        for (r = num_kv - 1; r >= 0; --r) {
1339            if (i % 2 == 0) {
1340                if (i % 4 == 0) {
1341                    status = fdb_get_metaonly(db[r], rdoc);
1342                } else {
1343                    rdoc->seqnum = i + 1;
1344                    status = fdb_get_byseq(db[r], rdoc);
1345                }
1346            } else {
1347                status = fdb_get(db[r], rdoc);
1348            }
1349            TEST_CHK(status == FDB_RESULT_SUCCESS);
1350            status = fdb_get_kvs_ops_info(db[r], &rinfo);
1351            TEST_CHK(status == FDB_RESULT_SUCCESS);
1352            info.num_gets = i + 1;
1353            TEST_CMP(&rinfo, &info, sizeof(fdb_kvs_ops_info));
1354        }
1355    }
1356
1357    fdb_close(dbfile);
1358
1359    // free all documents
1360    for (i=0;i<n;++i){
1361        fdb_doc_free(doc[i]);
1362    }
1363
1364    // free all resources
1365    fdb_shutdown();
1366
1367    memleak_end();
1368
1369    sprintf(bodybuf,"Operational stats test %s", multi_kv ?
1370            "multiple kv instances" : "single kv instance");
1371    TEST_RESULT(bodybuf);
1372}
1373
1374struct work_thread_args{
1375    int tid;
1376    size_t nthreads;
1377    size_t ndocs;
1378    size_t writer;
1379    fdb_doc **doc;
1380    size_t time_sec;
1381    size_t nbatch;
1382    size_t compact_term;
1383    int *n_opened;
1384    int *filename_count;
1385    spin_t *filename_count_lock;
1386    size_t nops;
1387    fdb_config *config;
1388    fdb_kvs_config *kvs_config;
1389};
1390
1391//#define FILENAME "./hdd/dummy"
1392#define FILENAME "dummy"
1393
1394#define KSIZE (100)
1395#define VSIZE (100)
1396#define IDX_DIGIT (7)
1397#define IDX_DIGIT_STR "7"
1398
1399void *_worker_thread(void *voidargs)
1400{
1401    TEST_INIT();
1402
1403    struct work_thread_args *args = (struct work_thread_args *)voidargs;
1404    int i, c, commit_count, filename_count;
1405    struct timeval ts_begin, ts_cur, ts_gap;
1406    fdb_file_handle *dbfile;
1407    fdb_kvs_handle *db;
1408    fdb_status status;
1409    fdb_doc *rdoc = NULL;
1410    char temp[1024];
1411
1412    char cnt_str[IDX_DIGIT+1];
1413    int cnt_int;
1414
1415    filename_count = *args->filename_count;
1416    sprintf(temp, FILENAME"%d", filename_count);
1417    fdb_open(&dbfile, temp, args->config);
1418    fdb_kvs_open_default(dbfile, &db, args->kvs_config);
1419    status = fdb_set_log_callback(db, logCallbackFunc,
1420                                  (void *) "worker_thread");
1421    TEST_CHK(status == FDB_RESULT_SUCCESS);
1422
1423    // wait until all other threads open the DB file.
1424    // (to avoid performing compaction before opening the file)
1425    spin_lock(args->filename_count_lock);
1426    *args->n_opened += 1;
1427    spin_unlock(args->filename_count_lock);
1428    do {
1429        spin_lock(args->filename_count_lock);
1430        if ((size_t)(*args->n_opened) == args->nthreads) {
1431            // all threads open the DB file
1432            spin_unlock(args->filename_count_lock);
1433            break;
1434        }
1435        spin_unlock(args->filename_count_lock);
1436        // sleep 1 sec
1437        sleep(1);
1438    } while (1);
1439
1440    gettimeofday(&ts_begin, NULL);
1441
1442    c = cnt_int = commit_count = 0;
1443    cnt_str[IDX_DIGIT] = 0;
1444
1445    while (1){
1446        i = rand() % args->ndocs;
1447        fdb_doc_create(&rdoc, args->doc[i]->key, args->doc[i]->keylen, NULL, 0, NULL, 0);
1448        status = fdb_get(db, rdoc);
1449
1450        TEST_CHK(status == FDB_RESULT_SUCCESS);
1451        TEST_CMP(rdoc->body, args->doc[i]->body, (IDX_DIGIT+1));
1452
1453        if (args->writer) {
1454            // if writer,
1455            // copy and parse the counter in body
1456            memcpy(cnt_str, (uint8_t *)rdoc->body + (IDX_DIGIT+1), IDX_DIGIT);
1457            cnt_int = atoi(cnt_str);
1458
1459            // increase and rephrase
1460            sprintf(cnt_str, "%0 " IDX_DIGIT_STR "d", ++cnt_int);
1461            memcpy((uint8_t *)rdoc->body + (IDX_DIGIT+1), cnt_str, IDX_DIGIT);
1462
1463            // update and commit
1464            status = fdb_set(db, rdoc);
1465            TEST_CHK(status == FDB_RESULT_SUCCESS);
1466
1467            if (args->nbatch > 0) {
1468                if (c % args->nbatch == 0) {
1469                    // commit for every NBATCH
1470                    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1471                    commit_count++;
1472                    fdb_file_info info;
1473                    fdb_get_file_info(dbfile, &info);
1474                    if (args->compact_term == (size_t)commit_count &&
1475                        args->compact_term > 0 &&
1476                        info.new_filename == NULL &&
1477                        args->tid == 0) {
1478                        // do compaction for every COMPACT_TERM batch
1479                        spin_lock(args->filename_count_lock);
1480                        *args->filename_count += 1;
1481                        filename_count = *args->filename_count;
1482                        spin_unlock(args->filename_count_lock);
1483
1484                        sprintf(temp, FILENAME"%d", filename_count);
1485
1486                        status = fdb_compact(dbfile, temp);
1487                        if (status != FDB_RESULT_SUCCESS) {
1488                            spin_lock(args->filename_count_lock);
1489                            *args->filename_count -= 1;
1490                            spin_unlock(args->filename_count_lock);
1491                        }
1492
1493                        commit_count = 0;
1494                    }
1495                }
1496            }
1497        }
1498        fdb_doc_free(rdoc);
1499        rdoc = NULL;
1500        c++;
1501
1502        gettimeofday(&ts_cur, NULL);
1503        ts_gap = _utime_gap(ts_begin, ts_cur);
1504        if ((size_t)ts_gap.tv_sec >= args->time_sec) break;
1505    }
1506
1507    DBG("Thread #%d (%s) %d ops / %d seconds\n",
1508        args->tid, (args->writer)?("writer"):("reader"), c, (int)args->time_sec);
1509    args->nops = c;
1510
1511    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1512
1513    fdb_kvs_close(db);
1514    fdb_close(dbfile);
1515    thread_exit(0);
1516    return NULL;
1517}
1518
1519void multi_thread_test(
1520    size_t ndocs, size_t wal_threshold, size_t time_sec,
1521    size_t nbatch, size_t compact_term, size_t nwriters, size_t nreaders)
1522{
1523    TEST_INIT();
1524
1525    size_t nwrites, nreads;
1526    int i, r;
1527    int n = nwriters + nreaders;;
1528    thread_t *tid = alca(thread_t, n);
1529    void **thread_ret = alca(void *, n);
1530    struct work_thread_args *args = alca(struct work_thread_args, n);
1531    struct timeval ts_begin, ts_cur, ts_gap;
1532    fdb_file_handle *dbfile;
1533    fdb_kvs_handle *db;
1534    fdb_doc **doc = alca(fdb_doc*, ndocs);
1535    fdb_status status;
1536    fdb_kvs_info kvs_info;
1537
1538    int filename_count = 1;
1539    int n_opened = 0;
1540    spin_t filename_count_lock;
1541    spin_init(&filename_count_lock);
1542
1543    char keybuf[1024], metabuf[1024], bodybuf[1024], temp[1024];
1544
1545    // remove previous dummy files
1546    r = system(SHELL_DEL" " FILENAME "* > errorlog.txt");
1547    (void)r;
1548
1549    fdb_config fconfig = fdb_get_default_config();
1550    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1551    fconfig.buffercache_size = 16777216;
1552    fconfig.wal_threshold = 1024;
1553    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1554    fconfig.compaction_threshold = 0;
1555
1556    memleak_start();
1557
1558    // initial population ===
1559    DBG("Initialize..\n");
1560
1561    // open db
1562    sprintf(temp, FILENAME"%d", filename_count);
1563    fdb_open(&dbfile, temp, &fconfig);
1564    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1565    status = fdb_set_log_callback(db, logCallbackFunc,
1566                                  (void *) "multi_thread_test");
1567    TEST_CHK(status == FDB_RESULT_SUCCESS);
1568
1569    gettimeofday(&ts_begin, NULL);
1570
1571    // insert documents
1572    for (i = 0; (size_t)i < ndocs; ++i){
1573        _set_random_string_smallabt(temp, KSIZE - (IDX_DIGIT+1));
1574        sprintf(keybuf, "k%0" IDX_DIGIT_STR "d%s", i, temp);
1575
1576        sprintf(metabuf, "m%0" IDX_DIGIT_STR "d", i);
1577
1578        _set_random_string_smallabt(temp, VSIZE-(IDX_DIGIT*2+1));
1579        sprintf(bodybuf, "b%0" IDX_DIGIT_STR "d%0" IDX_DIGIT_STR "d%s", i, 0, temp);
1580
1581        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1582            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1583        fdb_set(db, doc[i]);
1584    }
1585
1586    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1587
1588    gettimeofday(&ts_cur, NULL);
1589    ts_gap = _utime_gap(ts_begin, ts_cur);
1590    //DBG("%d.%09d seconds elapsed\n", (int)ts_gap.tv_sec, (int)ts_gap.tv_nsec);
1591
1592    fdb_kvs_close(db);
1593    fdb_close(dbfile);
1594    // end of population ===
1595
1596    // drop OS's page cache
1597    //r = system("free && sync && echo 3 > /proc/sys/vm/drop_caches && free");
1598
1599    // create workers
1600    for (i=0;i<n;++i){
1601        args[i].tid = i;
1602        args[i].nthreads = n;
1603        args[i].writer = (((size_t)i<nwriters)?(1):(0));
1604        args[i].ndocs = ndocs;
1605        args[i].doc = doc;
1606        args[i].time_sec = time_sec;
1607        args[i].nbatch = nbatch;
1608        args[i].compact_term = compact_term;
1609        args[i].n_opened = &n_opened;
1610        args[i].filename_count = &filename_count;
1611        args[i].filename_count_lock = &filename_count_lock;
1612        args[i].config = &fconfig;
1613        args[i].kvs_config = &kvs_config;
1614        thread_create(&tid[i], _worker_thread, &args[i]);
1615    }
1616
1617    printf("wait for %d seconds..\n", (int)time_sec);
1618
1619    // wait for thread termination
1620    for (i=0;i<n;++i){
1621        thread_join(tid[i], &thread_ret[i]);
1622    }
1623
1624    // free all documents
1625    for (i=0;(size_t)i<ndocs;++i){
1626        fdb_doc_free(doc[i]);
1627    }
1628
1629    nwrites = nreads = 0;
1630    for (i=0;i<n;++i){
1631        if (args[i].writer) {
1632            nwrites += args[i].nops;
1633        } else {
1634            nreads += args[i].nops;
1635        }
1636    }
1637    printf("read: %.1f ops/sec\n", (double)nreads/time_sec);
1638    printf("write: %.1f ops/sec\n", (double)nwrites/time_sec);
1639
1640    // check sequence number
1641    sprintf(temp, FILENAME"%d", filename_count);
1642    fdb_open(&dbfile, temp, &fconfig);
1643    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1644    fdb_get_kvs_info(db, &kvs_info);
1645    TEST_CHK(kvs_info.last_seqnum == ndocs+nwrites);
1646    fdb_kvs_close(db);
1647    fdb_close(dbfile);
1648
1649    // shutdown
1650    fdb_shutdown();
1651
1652    memleak_end();
1653
1654    TEST_RESULT("multi thread test");
1655}
1656
1657void *multi_thread_client_shutdown(void *args)
1658{
1659
1660    TEST_INIT();
1661
1662    int i, r;
1663    int nclients;
1664    fdb_file_handle *tdbfile;
1665    fdb_status status;
1666    fdb_config fconfig;
1667    fdb_kvs_config kvs_config;
1668    thread_t *tid;
1669    void **thread_ret;
1670
1671    if (args == NULL)
1672    { // parent
1673        memleak_start();
1674
1675        r = system(SHELL_DEL" dummy* > errorlog.txt");
1676        (void)r;
1677        nclients = 2;
1678        tid = alca(thread_t, nclients);
1679        thread_ret = alca(void *, nclients);
1680        for (i=0;i<nclients;++i){
1681            thread_create(&tid[i], multi_thread_client_shutdown, (void *)&i);
1682        }
1683        for (i=0;i<nclients;++i){
1684            thread_join(tid[i], &thread_ret[i]);
1685        }
1686
1687        memleak_end();
1688        TEST_RESULT("multi thread client shutdown");
1689        return NULL;
1690    }
1691
1692    // threads enter here //
1693
1694    fconfig = fdb_get_default_config();
1695    kvs_config = fdb_get_default_kvs_config();
1696    fconfig.wal_threshold = 1024;
1697    fconfig.compaction_threshold = 0;
1698
1699    // open/close db
1700    status = fdb_open(&tdbfile, "./dummy1", &fconfig);
1701    TEST_CHK(status == FDB_RESULT_SUCCESS);
1702    TEST_CHK(fdb_close(tdbfile) == FDB_RESULT_SUCCESS);
1703
1704    // shutdown
1705    fdb_shutdown();
1706    thread_exit(0);
1707    return NULL;
1708}
1709
1710void *multi_thread_kvs_client(void *args)
1711{
1712
1713    TEST_INIT();
1714
1715    int i, j, r;
1716    int n = 50;
1717    int nclients = 20;
1718    int *tid_args = alca(int, nclients);
1719    char dbstr[256];
1720    char keybuf[256], metabuf[256], bodybuf[256];
1721    fdb_file_handle *dbfile;
1722    fdb_kvs_handle *tdb;
1723    fdb_kvs_handle **db = alca(fdb_kvs_handle*, nclients);
1724    fdb_doc **doc = alca(fdb_doc*, n);
1725    fdb_doc *rdoc = NULL;
1726    fdb_status status;
1727    fdb_config fconfig;
1728    fdb_kvs_config kvs_config;
1729    fdb_seqnum_t seqnum;
1730    thread_t *tid;
1731    void **thread_ret;
1732
1733    if (args == NULL)
1734    { // parent
1735        memleak_start();
1736
1737        r = system(SHELL_DEL" dummy* > errorlog.txt");
1738        (void)r;
1739
1740        // init dbfile
1741        fconfig = fdb_get_default_config();
1742        fconfig.buffercache_size = 0;
1743        fconfig.wal_threshold = 1024;
1744        fconfig.compaction_threshold = 0;
1745
1746        status = fdb_open(&dbfile, "./dummy1", &fconfig);
1747        TEST_CHK(status == FDB_RESULT_SUCCESS);
1748
1749        tid = alca(thread_t, nclients);
1750        thread_ret = alca(void *, nclients);
1751        for (i=0;i<nclients;++i){
1752            sprintf(dbstr, "db%d", i);
1753            kvs_config = fdb_get_default_kvs_config();
1754            status = fdb_kvs_open(dbfile, &db[i], dbstr, &kvs_config);
1755            TEST_CHK(status == FDB_RESULT_SUCCESS);
1756            status = fdb_kvs_close(db[i]);
1757            TEST_CHK(status == FDB_RESULT_SUCCESS);
1758        }
1759        for (i=0;i<nclients;++i){
1760            tid_args[i] = i;
1761            thread_create(&tid[i], multi_thread_kvs_client,
1762                          (void *)&tid_args[i]);
1763        }
1764        for (i=0;i<nclients;++i){
1765            thread_join(tid[i], &thread_ret[i]);
1766        }
1767
1768        status = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1769        TEST_CHK(status == FDB_RESULT_SUCCESS);
1770
1771        // check threads updated kvs
1772        for (i=0; i<nclients; i++){
1773            sprintf(dbstr, "db%d", i);
1774            kvs_config = fdb_get_default_kvs_config();
1775            status = fdb_kvs_open(dbfile, &db[i], dbstr, &kvs_config);
1776            TEST_CHK(status == FDB_RESULT_SUCCESS);
1777
1778            // verify seqnum
1779            status = fdb_get_kvs_seqnum(db[i], &seqnum);
1780            TEST_CHK(status == FDB_RESULT_SUCCESS);
1781            TEST_CHK(seqnum == (fdb_seqnum_t)n);
1782
1783            for (j=0; j<n; j++){
1784                sprintf(keybuf, "key%d", j);
1785                sprintf(metabuf, "meta%d", j);
1786                sprintf(bodybuf, "body%d", j);
1787                fdb_doc_create(&rdoc, keybuf, strlen(keybuf),
1788                                      NULL, 0, NULL, 0);
1789                status = fdb_get(db[i], rdoc);
1790                TEST_CHK(status == FDB_RESULT_SUCCESS);
1791                TEST_CHK(!memcmp(rdoc->key, keybuf, strlen(keybuf)));
1792                TEST_CHK(!memcmp(rdoc->meta, metabuf, rdoc->metalen));
1793                TEST_CHK(!memcmp(rdoc->body, bodybuf, rdoc->bodylen));
1794                fdb_doc_free(rdoc);
1795            }
1796            status = fdb_kvs_close(db[i]);
1797            TEST_CHK(status == FDB_RESULT_SUCCESS);
1798        }
1799
1800        status = fdb_close(dbfile);
1801        TEST_CHK(status == FDB_RESULT_SUCCESS);
1802        fdb_shutdown();
1803        memleak_end();
1804        TEST_RESULT("multi thread kvs client");
1805        return NULL;
1806    }
1807
1808    // threads enter here //
1809
1810    // open fhandle
1811    fconfig = fdb_get_default_config();
1812    status = fdb_open(&dbfile, "./dummy1", &fconfig);
1813    TEST_CHK(status == FDB_RESULT_SUCCESS);
1814
1815    // get kvs ID from args
1816    memcpy(&i, args, sizeof(int));
1817    sprintf(dbstr, "db%d", i);
1818    kvs_config = fdb_get_default_kvs_config();
1819    status = fdb_kvs_open(dbfile, &tdb, dbstr, &kvs_config);
1820    TEST_CHK(status == FDB_RESULT_SUCCESS);
1821
1822    // insert documents
1823    for (i=0;i<n;++i){
1824        sprintf(keybuf, "key%d", i);
1825        sprintf(metabuf, "meta%d", i);
1826        sprintf(bodybuf, "body%d", i);
1827        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1828                                (void*)metabuf, strlen(metabuf),
1829                                (void*)bodybuf, strlen(bodybuf));
1830        status = fdb_set(tdb, doc[i]);
1831        TEST_CHK(status == FDB_RESULT_SUCCESS);
1832        fdb_doc_free(doc[i]);
1833    }
1834    status = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1835    TEST_CHK(status == FDB_RESULT_SUCCESS);
1836
1837    fdb_close(dbfile);
1838    return NULL;
1839}
1840
1841void incomplete_block_test()
1842{
1843    TEST_INIT();
1844
1845    memleak_start();
1846
1847    int i, r;
1848    int n = 2;
1849    fdb_file_handle *dbfile;
1850    fdb_kvs_handle *db;
1851    fdb_doc **doc = alca(fdb_doc*, n);
1852    fdb_doc *rdoc = NULL;
1853    fdb_status status;
1854
1855    char keybuf[256], metabuf[256], bodybuf[256];
1856
1857    // remove previous dummy files
1858    r = system(SHELL_DEL" dummy* > errorlog.txt");
1859    (void)r;
1860
1861    fdb_config fconfig = fdb_get_default_config();
1862    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1863    fconfig.buffercache_size = 0;
1864    fconfig.wal_threshold = 1024;
1865    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1866    fconfig.compaction_threshold = 0;
1867
1868    // open db
1869    fdb_open(&dbfile, "./dummy1", &fconfig);
1870    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1871    status = fdb_set_log_callback(db, logCallbackFunc,
1872                                  (void *) "incomplete_block_test");
1873    TEST_CHK(status == FDB_RESULT_SUCCESS);
1874
1875    // insert documents
1876    for (i=0;i<n;++i){
1877        sprintf(keybuf, "key%d", i);
1878        sprintf(metabuf, "meta%d", i);
1879        sprintf(bodybuf, "body%d", i);
1880        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1881            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1882        fdb_set(db, doc[i]);
1883    }
1884
1885    // retrieve documents
1886    for (i=0;i<n;++i){
1887        // search by key
1888        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
1889        status = fdb_get(db, rdoc);
1890
1891        // updated documents
1892        TEST_CHK(status == FDB_RESULT_SUCCESS);
1893        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
1894        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
1895
1896        // free result document
1897        fdb_doc_free(rdoc);
1898        rdoc = NULL;
1899    }
1900
1901    // close db file
1902    fdb_kvs_close(db);
1903    fdb_close(dbfile);
1904
1905    // free all documents
1906    for (i=0;i<n;++i){
1907        fdb_doc_free(doc[i]);
1908    }
1909
1910    // free all resources
1911    fdb_shutdown();
1912
1913    memleak_end();
1914
1915    TEST_RESULT("incomplete block test");
1916}
1917
1918
1919static int _cmp_double(void *key1, size_t keylen1, void *key2, size_t keylen2)
1920{
1921    double aa, bb;
1922    aa = *(double *)key1;
1923    bb = *(double *)key2;
1924
1925    if (aa<bb) {
1926        return -1;
1927    } else if (aa>bb) {
1928        return 1;
1929    } else {
1930        return 0;
1931    }
1932}
1933
1934void custom_compare_primitive_test()
1935{
1936    TEST_INIT();
1937
1938    memleak_start();
1939
1940    int i, r;
1941    int n = 10;
1942    fdb_file_handle *dbfile;
1943    fdb_kvs_handle *db;
1944    fdb_doc **doc = alca(fdb_doc*, n);
1945    fdb_doc *rdoc = NULL;
1946    fdb_status status;
1947    fdb_iterator *iterator;
1948
1949    char keybuf[256], bodybuf[256];
1950    double key_double, key_double_prev;
1951
1952    // remove previous dummy files
1953    r = system(SHELL_DEL" dummy* > errorlog.txt");
1954    (void)r;
1955
1956    fdb_config fconfig = fdb_get_default_config();
1957    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1958    fconfig.buffercache_size = 0;
1959    fconfig.wal_threshold = 1024;
1960    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1961    fconfig.compaction_threshold = 0;
1962    fconfig.multi_kv_instances = true;
1963
1964    kvs_config.custom_cmp = _cmp_double;
1965
1966    // open db with custom compare function for double key type
1967    fdb_open(&dbfile, "./dummy1", &fconfig);
1968    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1969    status = fdb_set_log_callback(db, logCallbackFunc,
1970                                  (void *) "custom_compare_primitive_test");
1971    TEST_CHK(status == FDB_RESULT_SUCCESS);
1972
1973    for (i=0;i<n;++i){
1974        key_double = 10000/(i*11.0);
1975        memcpy(keybuf, &key_double, sizeof(key_double));
1976        sprintf(bodybuf, "value: %d, %f", i, key_double);
1977        fdb_doc_create(&doc[i], (void*)keybuf, sizeof(key_double), NULL, 0,
1978            (void*)bodybuf, strlen(bodybuf)+1);
1979        fdb_set(db, doc[i]);
1980    }
1981
1982    // range scan (before flushing WAL)
1983    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
1984    key_double_prev = -1;
1985    do {
1986        status = fdb_iterator_get(iterator, &rdoc);
1987        TEST_CHK(status == FDB_RESULT_SUCCESS);
1988        memcpy(&key_double, rdoc->key, rdoc->keylen);
1989        TEST_CHK(key_double > key_double_prev);
1990        key_double_prev = key_double;
1991        fdb_doc_free(rdoc);
1992        rdoc = NULL;
1993    } while(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
1994    fdb_iterator_close(iterator);
1995
1996    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1997
1998    // range scan (after flushing WAL)
1999    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2000    key_double_prev = -1;
2001    do {
2002        status = fdb_iterator_get(iterator, &rdoc);
2003        TEST_CHK(status == FDB_RESULT_SUCCESS);
2004        memcpy(&key_double, rdoc->key, rdoc->keylen);
2005        TEST_CHK(key_double > key_double_prev);
2006        key_double_prev = key_double;
2007        fdb_doc_free(rdoc);
2008        rdoc = NULL;
2009    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2010    fdb_iterator_close(iterator);
2011
2012    // do compaction
2013    fdb_compact(dbfile, (char *) "./dummy2");
2014
2015    // range scan (after compaction)
2016    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2017    key_double_prev = -1;
2018    do {
2019        status = fdb_iterator_get(iterator, &rdoc);
2020        TEST_CHK(status == FDB_RESULT_SUCCESS);
2021        memcpy(&key_double, rdoc->key, rdoc->keylen);
2022        TEST_CHK(key_double > key_double_prev);
2023        key_double_prev = key_double;
2024        fdb_doc_free(rdoc);
2025        rdoc = NULL;
2026    } while(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2027    fdb_iterator_close(iterator);
2028
2029    // close db file
2030    fdb_kvs_close(db);
2031    fdb_close(dbfile);
2032
2033    // free all documents
2034    for (i=0;i<n;++i){
2035        fdb_doc_free(doc[i]);
2036    }
2037
2038    // free all resources
2039    fdb_shutdown();
2040
2041    memleak_end();
2042
2043    TEST_RESULT("custom compare function for primitive key test");
2044}
2045
2046static int _cmp_variable(void *key1, size_t keylen1, void *key2, size_t keylen2)
2047{
2048    // compare only 3rd~8th bytes (ignore the others)
2049    return memcmp((uint8_t*)key1+2, (uint8_t*)key2+2, 6);
2050}
2051
2052void custom_compare_variable_test()
2053{
2054    TEST_INIT();
2055
2056    memleak_start();
2057
2058    int i, j, r;
2059    int n = 1000;
2060    int count;
2061    fdb_file_handle *dbfile;
2062    fdb_kvs_handle *db, *db2;
2063    fdb_doc **doc = alca(fdb_doc*, n);
2064    fdb_doc *rdoc = NULL;
2065    fdb_status status;
2066    fdb_iterator *iterator;
2067
2068    size_t keylen = 16;
2069    size_t prev_keylen;
2070    char keybuf[256], bodybuf[256];
2071    char prev_key[256];
2072
2073    // remove previous dummy files
2074    r = system(SHELL_DEL" dummy* > errorlog.txt");
2075    (void)r;
2076
2077    fdb_config fconfig = fdb_get_default_config();
2078    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2079    fconfig.buffercache_size = 0;
2080    fconfig.wal_threshold = 1024;
2081    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2082    fconfig.compaction_threshold = 0;
2083    fconfig.multi_kv_instances = true;
2084
2085    kvs_config.custom_cmp = _cmp_variable;
2086
2087    // open db with custom compare function for variable length key type
2088    //fdb_open_cmp_variable(&dbfile, "./dummy1", &fconfig);
2089    fdb_open(&dbfile, "./dummy1", &fconfig);
2090    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2091    status = fdb_set_log_callback(db, logCallbackFunc,
2092                                  (void *) "custom_compare_variable_test");
2093    TEST_CHK(status == FDB_RESULT_SUCCESS);
2094
2095    for (i=0;i<n;++i){
2096        for (j=0;j<2;++j){
2097            keybuf[j] = 'a' + rand()%('z'-'a');
2098        }
2099        sprintf(keybuf+2, "%06d", i);
2100        for (j=8;(size_t)j<keylen-1;++j){
2101            keybuf[j] = 'a' + rand()%('z'-'a');
2102        }
2103        keybuf[keylen-1] = 0;
2104        sprintf(bodybuf, "value: %d", i);
2105        fdb_doc_create(&doc[i], (void*)keybuf, keylen, NULL, 0,
2106            (void*)bodybuf, strlen(bodybuf)+1);
2107        fdb_set(db, doc[i]);
2108    }
2109
2110    // point query
2111    for (i=0;i<n;++i){
2112        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
2113        status = fdb_get(db, rdoc);
2114
2115        TEST_CHK(status == FDB_RESULT_SUCCESS);
2116        TEST_CHK(rdoc->bodylen == doc[i]->bodylen);
2117        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2118
2119        fdb_doc_free(rdoc);
2120        rdoc = NULL;
2121    }
2122
2123    // range scan (before flushing WAL)
2124    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2125    sprintf(prev_key, "%016d", 0);
2126    count = 0;
2127    prev_keylen = 16;
2128    do {
2129        status = fdb_iterator_get(iterator, &rdoc);
2130        TEST_CHK(status == FDB_RESULT_SUCCESS);
2131        TEST_CHK(_cmp_variable(prev_key, prev_keylen, rdoc->key, rdoc->keylen) <= 0);
2132        prev_keylen = rdoc->keylen;
2133        memcpy(prev_key, rdoc->key, rdoc->keylen);
2134        fdb_doc_free(rdoc);
2135        rdoc = NULL;
2136        count++;
2137    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2138    TEST_CHK(count == n);
2139    fdb_iterator_close(iterator);
2140
2141    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2142
2143    // range scan (after flushing WAL)
2144    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2145    sprintf(prev_key, "%016d", 0);
2146    count = 0;
2147    prev_keylen = 16;
2148    do {
2149        status = fdb_iterator_get(iterator, &rdoc);
2150        TEST_CHK(status == FDB_RESULT_SUCCESS);
2151        TEST_CHK(_cmp_variable(prev_key, prev_keylen, rdoc->key, rdoc->keylen)
2152                 <= 0);
2153        prev_keylen = rdoc->keylen;
2154        memcpy(prev_key, rdoc->key, rdoc->keylen);
2155        fdb_doc_free(rdoc);
2156        rdoc = NULL;
2157        count++;
2158    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2159    TEST_CHK(count == n);
2160    fdb_iterator_close(iterator);
2161
2162    // do compaction
2163    fdb_compact(dbfile, (char *) "./dummy2");
2164
2165    // range scan (after compaction)
2166    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2167    sprintf(prev_key, "%016d", 0);
2168    count = 0;
2169    prev_keylen = 16;
2170    do {
2171        status = fdb_iterator_get(iterator, &rdoc);
2172        TEST_CHK(status == FDB_RESULT_SUCCESS);
2173        TEST_CHK(_cmp_variable(prev_key, prev_keylen, rdoc->key, rdoc->keylen) <= 0);
2174        prev_keylen = rdoc->keylen;
2175        memcpy(prev_key, rdoc->key, rdoc->keylen);
2176        fdb_doc_free(rdoc);
2177        rdoc = NULL;
2178        count++;
2179    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2180    TEST_CHK(count == n);
2181    fdb_iterator_close(iterator);
2182
2183    // range scan by sequence
2184    fdb_iterator_sequence_init(db, &iterator, 0, 0, 0x0);
2185    count = 0;
2186    do { // forward
2187        status = fdb_iterator_get(iterator, &rdoc);
2188        TEST_CHK(status == FDB_RESULT_SUCCESS);
2189        fdb_doc_free(rdoc);
2190        rdoc = NULL;
2191        count++;
2192    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2193    TEST_CHK(count == n);
2194
2195    // Reverse direction
2196    for (; fdb_iterator_prev(iterator) != FDB_RESULT_ITERATOR_FAIL; --count) {
2197        status = fdb_iterator_get(iterator, &rdoc);
2198        TEST_CHK(status == FDB_RESULT_SUCCESS);
2199        fdb_doc_free(rdoc);
2200        rdoc = NULL;
2201    };
2202    TEST_CHK(count == 0);
2203    fdb_iterator_close(iterator);
2204
2205    // open another handle
2206    kvs_config.custom_cmp = NULL;
2207    fdb_kvs_open_default(dbfile, &db2, &kvs_config);
2208
2209    // point query
2210    for (i=0;i<n;++i){
2211        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
2212        status = fdb_get(db2, rdoc);
2213
2214        TEST_CHK(status == FDB_RESULT_SUCCESS);
2215        TEST_CHK(rdoc->bodylen == doc[i]->bodylen);
2216        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2217
2218        fdb_doc_free(rdoc);
2219        rdoc = NULL;
2220    }
2221
2222    // close db file
2223    fdb_kvs_close(db);
2224    fdb_kvs_close(db2);
2225    fdb_close(dbfile);
2226
2227    // free all documents
2228    for (i=0;i<n;++i){
2229        fdb_doc_free(doc[i]);
2230    }
2231
2232    // free all resources
2233    fdb_shutdown();
2234
2235    memleak_end();
2236
2237    TEST_RESULT("custom compare function for variable length key test");
2238}
2239
2240/*
2241 * custom compare test with commit and compact
2242 *    eqkeys:  boolean to toggle whether bytes in
2243 *             comparision range are equal
2244 */
2245void custom_compare_commit_compact(bool eqkeys)
2246{
2247    TEST_INIT();
2248
2249    memleak_start();
2250
2251    int i, j, r;
2252    int count;
2253    int n = 10;
2254    static const int len = 1024;
2255    char keybuf[len];
2256    static const char *achar = "a";
2257    fdb_doc *rdoc = NULL;
2258    fdb_status status;
2259    fdb_file_handle *dbfile;
2260    fdb_kvs_handle *db;
2261    fdb_iterator *iterator;
2262    fdb_config fconfig = fdb_get_default_config();
2263    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2264
2265    // remove previous dummy files
2266    r = system(SHELL_DEL" dummy* > errorlog.txt");
2267    (void)r;
2268
2269    fconfig.buffercache_size = 0;
2270    fconfig.wal_threshold = 1024;
2271    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2272    fconfig.compaction_threshold = 0;
2273    fconfig.multi_kv_instances = true;
2274
2275    kvs_config.custom_cmp = _cmp_variable;
2276
2277
2278
2279    r = system(SHELL_DEL" dummy* > errorlog.txt");
2280    (void)r;
2281
2282
2283    // open db
2284    status = fdb_open(&dbfile, "./dummy1", &fconfig);
2285    TEST_CHK(status == FDB_RESULT_SUCCESS);
2286    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
2287    TEST_CHK(status == FDB_RESULT_SUCCESS);
2288
2289
2290    for (i=0;i<n;++i){
2291        if(eqkeys){
2292            sprintf(keybuf, "%d", i);
2293            for (j=1;j<len;++j) {
2294                keybuf[j] = *achar;
2295            }
2296        } else {
2297            sprintf(keybuf, "000%d", i);
2298            for (j=4;j<len;++j) {
2299                keybuf[j] = *achar;
2300            }
2301        }
2302        keybuf[len-1] = '\0';
2303        // set kv
2304        status = fdb_set_kv(db, keybuf, strlen(keybuf), NULL, 0);
2305        TEST_CHK(status == FDB_RESULT_SUCCESS);
2306    }
2307
2308    // compact pre & post commit
2309    fdb_compact(dbfile, NULL);
2310    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2311    fdb_compact(dbfile, NULL);
2312
2313    // scan after flush
2314    count = 0;
2315    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2316    do {
2317        status = fdb_iterator_get(iterator, &rdoc);
2318        TEST_CHK(status == FDB_RESULT_SUCCESS);
2319        fdb_doc_free(rdoc);
2320        rdoc = NULL;
2321        count++;
2322    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2323
2324    if (eqkeys) {
2325        // since the custom cmp function compares only 3rd~8th bytes,
2326        // all keys are identified as the same key.
2327        TEST_CHK(count == 1);
2328    } else {
2329        TEST_CHK(count == n);
2330    }
2331
2332    fdb_iterator_close(iterator);
2333
2334    fdb_close(dbfile);
2335    fdb_shutdown();
2336
2337    memleak_end();
2338    TEST_RESULT("custom compare commit compact");
2339
2340}
2341
2342
2343void doc_compression_test()
2344{
2345    TEST_INIT();
2346
2347    memleak_start();
2348
2349    int i, r;
2350    int n = 10;
2351    int dummy_len = 32;
2352    fdb_file_handle *dbfile;
2353    fdb_kvs_handle *db;
2354    fdb_doc **doc = alca(fdb_doc*, n);
2355    fdb_doc *rdoc = NULL;
2356    fdb_status status;
2357
2358    char keybuf[256], metabuf[256], bodybuf[256], temp[256];
2359
2360    // remove previous dummy files
2361    r = system(SHELL_DEL" dummy* > errorlog.txt");
2362    (void)r;
2363
2364    fdb_config fconfig = fdb_get_default_config();
2365    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2366    fconfig.buffercache_size = 0;
2367    fconfig.wal_threshold = 1024;
2368    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2369    fconfig.compress_document_body = true;
2370    fconfig.compaction_threshold = 0;
2371
2372    // open db
2373    fdb_open(&dbfile, "./dummy1", &fconfig);
2374    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2375    status = fdb_set_log_callback(db, logCallbackFunc,
2376                                  (void *) "doc_compression_test");
2377    TEST_CHK(status == FDB_RESULT_SUCCESS);
2378
2379    // set dummy str
2380    memset(temp, 'a', dummy_len);
2381    temp[dummy_len]=0;
2382
2383    // insert documents
2384    for (i=0;i<n;++i){
2385        sprintf(keybuf, "key%d", i);
2386        sprintf(metabuf, "meta%d", i);
2387        sprintf(bodybuf, "body%d_%s", i, temp);
2388        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2389            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2390        fdb_set(db, doc[i]);
2391    }
2392
2393    // remove document #5
2394    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, doc[5]->meta, doc[5]->metalen, NULL, 0);
2395    status = fdb_del(db, rdoc);
2396    TEST_CHK(status == FDB_RESULT_SUCCESS);
2397    fdb_doc_free(rdoc);
2398    rdoc = NULL;
2399
2400    // commit
2401    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2402
2403    // close the db
2404    fdb_kvs_close(db);
2405    fdb_close(dbfile);
2406
2407    // reopen
2408    fdb_open(&dbfile, "./dummy1", &fconfig);
2409    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2410    status = fdb_set_log_callback(db, logCallbackFunc,
2411                                  (void *) "doc_compression_test");
2412    TEST_CHK(status == FDB_RESULT_SUCCESS);
2413
2414    // update dummy str
2415    dummy_len = 64;
2416    memset(temp, 'b', dummy_len);
2417    temp[dummy_len]=0;
2418
2419    // update document #0 and #1
2420    for (i=0;i<2;++i){
2421        sprintf(metabuf, "newmeta%d", i);
2422        sprintf(bodybuf, "newbody%d_%s", i, temp);
2423        fdb_doc_update(&doc[i], (void *)metabuf, strlen(metabuf),
2424            (void *)bodybuf, strlen(bodybuf));
2425        fdb_set(db, doc[i]);
2426    }
2427
2428    // commit
2429    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2430
2431    // retrieve documents
2432    for (i=0;i<n;++i){
2433        // search by key
2434        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
2435        status = fdb_get(db, rdoc);
2436
2437        if (i != 5) {
2438            // updated documents
2439            TEST_CHK(status == FDB_RESULT_SUCCESS);
2440            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2441            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2442        } else {
2443            // removed document
2444            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
2445        }
2446
2447        // free result document
2448        fdb_doc_free(rdoc);
2449        rdoc = NULL;
2450    }
2451
2452    // do compaction
2453    fdb_compact(dbfile, (char *) "./dummy2");
2454
2455    // retrieve documents after compaction
2456    for (i=0;i<n;++i){
2457        // search by key
2458        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
2459        status = fdb_get(db, rdoc);
2460
2461        if (i != 5) {
2462            // updated documents
2463            TEST_CHK(status == FDB_RESULT_SUCCESS);
2464            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2465            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2466        } else {
2467            // removed document
2468            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
2469        }
2470
2471        // free result document
2472        fdb_doc_free(rdoc);
2473        rdoc = NULL;
2474    }
2475
2476    // free all documents
2477    for (i=0;i<n;++i){
2478        fdb_doc_free(doc[i]);
2479    }
2480
2481    // close db file
2482    fdb_kvs_close(db);
2483    fdb_close(dbfile);
2484
2485    // free all resources
2486    fdb_shutdown();
2487
2488    memleak_end();
2489
2490    TEST_RESULT("document compression test");
2491}
2492
2493void read_doc_by_offset_test()
2494{
2495    TEST_INIT();
2496    memleak_start();
2497
2498    int i, r;
2499    int n = 100;
2500    fdb_file_handle *dbfile;
2501    fdb_kvs_handle *db;
2502    fdb_doc **doc = alca(fdb_doc*, n);
2503    fdb_doc *rdoc, *rdoc1;
2504    fdb_status status;
2505
2506    char keybuf[256], metabuf[256], bodybuf[256];
2507
2508    // remove previous dummy files
2509    r = system(SHELL_DEL" dummy* > errorlog.txt");
2510    (void)r;
2511
2512    fdb_config fconfig = fdb_get_default_config();
2513    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2514    fconfig.buffercache_size = 0;
2515    fconfig.wal_threshold = 1024;
2516    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2517    fconfig.purging_interval = 3600;
2518    fconfig.compaction_threshold = 0;
2519
2520    // open db
2521    fdb_open(&dbfile, "./dummy1", &fconfig);
2522    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2523    status = fdb_set_log_callback(db, logCallbackFunc,
2524                                  (void *) "read_doc_by_offset_test");
2525    TEST_CHK(status == FDB_RESULT_SUCCESS);
2526
2527    // insert documents
2528    for (i=0;i<n;++i){
2529        sprintf(keybuf, "key%d", i);
2530        sprintf(metabuf, "meta%d", i);
2531        sprintf(bodybuf, "body%d", i);
2532        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2533            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2534        fdb_set(db, doc[i]);
2535    }
2536
2537    // commit
2538    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2539
2540    // update documents from #0 to #49
2541    for (i=0;i<n/2;++i){
2542        sprintf(metabuf, "meta2%d", i);
2543        sprintf(bodybuf, "body2%d", i);
2544        fdb_doc_update(&doc[i], (void *)metabuf, strlen(metabuf),
2545            (void *)bodybuf, strlen(bodybuf));
2546        fdb_set(db, doc[i]);
2547    }
2548
2549    // remove document #50
2550    fdb_doc_create(&rdoc, doc[50]->key, doc[50]->keylen, doc[50]->meta,
2551                   doc[50]->metalen, NULL, 0);
2552    status = fdb_del(db, rdoc);
2553    TEST_CHK(status == FDB_RESULT_SUCCESS);
2554    fdb_doc_free(rdoc);
2555    rdoc = NULL;
2556
2557    // commit
2558    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2559
2560    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, NULL, 0, NULL, 0);
2561    status = fdb_get_metaonly(db, rdoc);
2562    TEST_CHK(status == FDB_RESULT_SUCCESS);
2563    TEST_CHK(rdoc->deleted == false);
2564    TEST_CMP(rdoc->meta, doc[5]->meta, rdoc->metalen);
2565    // Fetch #5 doc using its offset.
2566    status = fdb_get_byoffset(db, rdoc);
2567    TEST_CHK(status == FDB_RESULT_SUCCESS);
2568    TEST_CHK(rdoc->deleted == false);
2569    TEST_CMP(rdoc->meta, doc[5]->meta, rdoc->metalen);
2570    TEST_CMP(rdoc->body, doc[5]->body, rdoc->bodylen);
2571
2572    // MB-13095
2573    fdb_doc_create(&rdoc1, NULL, 0, NULL, 0, NULL, 0);
2574    rdoc1->offset = rdoc->offset;
2575    status = fdb_get_byoffset(db, rdoc1);
2576    TEST_CHK(status == FDB_RESULT_SUCCESS);
2577    TEST_CMP(rdoc1->key, doc[5]->key, rdoc1->keylen);
2578    TEST_CMP(rdoc1->meta, doc[5]->meta, rdoc1->metalen);
2579    TEST_CMP(rdoc1->body, doc[5]->body, rdoc1->bodylen);
2580
2581    fdb_doc_free(rdoc);
2582    rdoc = NULL;
2583    fdb_doc_free(rdoc1);
2584
2585    // do compaction
2586    fdb_compact(dbfile, (char *) "./dummy2");
2587
2588    fdb_doc_create(&rdoc, doc[50]->key, doc[50]->keylen, NULL, 0, NULL, 0);
2589    status = fdb_get_metaonly(db, rdoc);
2590    TEST_CHK(status == FDB_RESULT_SUCCESS);
2591    TEST_CHK(rdoc->deleted == true);
2592    TEST_CMP(rdoc->meta, doc[50]->meta, rdoc->metalen);
2593    // Fetch #50 doc using its offset.
2594    status = fdb_get_byoffset(db, rdoc);
2595    TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
2596    TEST_CHK(rdoc->deleted == true);
2597    fdb_doc_free(rdoc);
2598    rdoc = NULL;
2599
2600    // free all documents
2601    for (i=0;i<n;++i){
2602        fdb_doc_free(doc[i]);
2603    }
2604
2605    // close db file
2606    fdb_kvs_close(db);
2607    fdb_close(dbfile);
2608
2609    // free all resources
2610    fdb_shutdown();
2611
2612    memleak_end();
2613
2614    TEST_RESULT("read_doc_by_offset test");
2615}
2616
2617void purge_logically_deleted_doc_test()
2618{
2619    TEST_INIT();
2620
2621    memleak_start();
2622
2623    int i, r;
2624    int n = 10;
2625    fdb_file_handle *dbfile;
2626    fdb_kvs_handle *db;
2627    fdb_doc **doc = alca(fdb_doc*, n);
2628    fdb_doc *rdoc = NULL;
2629    fdb_status status;
2630
2631    char keybuf[256], metabuf[256], bodybuf[256];
2632
2633    // remove previous dummy files
2634    r = system(SHELL_DEL" dummy* fdb_test_config.json > errorlog.txt");
2635    (void)r;
2636
2637    fdb_config fconfig = fdb_get_default_config();
2638    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2639    fconfig.buffercache_size = 0;
2640    fconfig.wal_threshold = 1024;
2641    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2642    fconfig.purging_interval = 2;
2643    fconfig.compaction_threshold = 0;
2644
2645    // open db
2646    fdb_open(&dbfile, "./dummy1", &fconfig);
2647    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2648    status = fdb_set_log_callback(db, logCallbackFunc,
2649                                  (void *) "purge_logically_deleted_doc_test");
2650    TEST_CHK(status == FDB_RESULT_SUCCESS);
2651
2652    // insert documents
2653    for (i=0;i<n;++i){
2654        sprintf(keybuf, "key%d", i);
2655        sprintf(metabuf, "meta%d", i);
2656        sprintf(bodybuf, "body%d", i);
2657        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2658            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2659        fdb_set(db, doc[i]);
2660    }
2661
2662    // remove document #5
2663    fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, doc[5]->meta, doc[5]->metalen, NULL, 0);
2664    status = fdb_del(db, rdoc);
2665    TEST_CHK(status == FDB_RESULT_SUCCESS);
2666    fdb_doc_free(rdoc);
2667    rdoc = NULL;
2668
2669    // commit
2670    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2671
2672    // do compaction
2673    fdb_compact(dbfile, (char *) "./dummy2");
2674
2675    // retrieve documents after compaction
2676    for (i=0;i<n;++i){
2677        // search by key
2678        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
2679        status = fdb_get(db, rdoc);
2680
2681        if (i != 5) {
2682            // updated documents
2683            TEST_CHK(status == FDB_RESULT_SUCCESS);
2684            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2685            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2686        } else {
2687            // removed document
2688            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
2689        }
2690        // free result document
2691        fdb_doc_free(rdoc);
2692        rdoc = NULL;
2693
2694        // retrieve metadata
2695        // all documents including logically deleted document should exist
2696        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
2697        status = fdb_get_metaonly(db, rdoc);
2698        TEST_CHK(status == FDB_RESULT_SUCCESS);
2699        fdb_doc_free(rdoc);
2700        rdoc = NULL;
2701    }
2702
2703    printf("wait for 3 seconds..\n");
2704    sleep(3);
2705
2706    // do one more compaction
2707    fdb_compact(dbfile, (char *) "./dummy3");
2708
2709    // retrieve documents after compaction
2710    for (i=0;i<n;++i){
2711        // search by key
2712        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
2713        status = fdb_get(db, rdoc);
2714
2715        if (i != 5) {
2716            // updated documents
2717            TEST_CHK(status == FDB_RESULT_SUCCESS);
2718            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2719            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2720        } else {
2721            // removed document
2722            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
2723        }
2724        // free result document
2725        fdb_doc_free(rdoc);
2726        rdoc = NULL;
2727
2728        // retrieve metadata
2729        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
2730        status = fdb_get_metaonly(db, rdoc);
2731        if (i != 5) {
2732            TEST_CHK(status == FDB_RESULT_SUCCESS);
2733        } else {
2734            // logically deletec document must be purged during the compaction
2735            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
2736        }
2737        fdb_doc_free(rdoc);
2738        rdoc = NULL;
2739    }
2740
2741    // close db file
2742    fdb_kvs_close(db);
2743    fdb_close(dbfile);
2744
2745    // free all documents
2746    for (i=0;i<n;++i){
2747        fdb_doc_free(doc[i]);
2748    }
2749
2750    // free all resources
2751    fdb_shutdown();
2752
2753    memleak_end();
2754
2755    TEST_RESULT("purge logically deleted doc test");
2756}
2757
2758void api_wrapper_test()
2759{
2760    TEST_INIT();
2761
2762    memleak_start();
2763
2764    int i, r;
2765    int n = 10;
2766    size_t valuelen;
2767    void *value;
2768    fdb_file_handle *dbfile;
2769    fdb_kvs_handle *db;
2770    fdb_status status;
2771
2772    char keybuf[256], bodybuf[256], temp[256];
2773
2774    // remove previous dummy files
2775    r = system(SHELL_DEL" dummy* > errorlog.txt");
2776    (void)r;
2777
2778    fdb_config fconfig = fdb_get_default_config();
2779    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2780    fconfig.buffercache_size = 0;
2781    fconfig.wal_threshold = 1024;
2782    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2783    fconfig.purging_interval = 0;
2784    fconfig.compaction_threshold = 0;
2785
2786    // open db
2787    fdb_open(&dbfile, "./dummy1", &fconfig);
2788    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2789    status = fdb_set_log_callback(db, logCallbackFunc,
2790                                  (void *) "api_wrapper_test");
2791    TEST_CHK(status == FDB_RESULT_SUCCESS);
2792
2793    // error check
2794    status = fdb_set_kv(db, NULL, 0, NULL, 0);
2795    TEST_CHK(status == FDB_RESULT_INVALID_ARGS);
2796
2797    // insert key-value pairs
2798    for (i=0;i<n;++i){
2799        sprintf(keybuf, "key%d", i);
2800        sprintf(bodybuf, "body%d", i);
2801        status = fdb_set_kv(db, keybuf, strlen(keybuf), bodybuf, strlen(bodybuf));
2802        TEST_CHK(status == FDB_RESULT_SUCCESS);
2803    }
2804
2805    // remove key5
2806    sprintf(keybuf, "key%d", 5);
2807    status = fdb_del_kv(db, keybuf, strlen(keybuf));
2808    TEST_CHK(status == FDB_RESULT_SUCCESS);
2809
2810    // error check
2811    status = fdb_del_kv(db, NULL, 0);
2812    TEST_CHK(status == FDB_RESULT_INVALID_ARGS);
2813
2814    // retrieve key-value pairs
2815    for (i=0;i<n;++i){
2816        sprintf(keybuf, "key%d", i);
2817        status = fdb_get_kv(db, keybuf, strlen(keybuf), &value, &valuelen);
2818
2819        if (i != 5) {
2820            // updated documents
2821            TEST_CHK(status == FDB_RESULT_SUCCESS);
2822            sprintf(temp, "body%d", i);
2823            TEST_CMP(value, temp, valuelen);
2824            fdb_free_block(value);
2825        } else {
2826            // removed document
2827            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
2828        }
2829    }
2830
2831    // error check
2832    status = fdb_get_kv(db, NULL, 0, &value, &valuelen);
2833    TEST_CHK(status == FDB_RESULT_INVALID_ARGS);
2834
2835    status = fdb_get_kv(db, keybuf, strlen(keybuf), NULL, NULL);
2836    TEST_CHK(status == FDB_RESULT_INVALID_ARGS);
2837
2838    // close db file
2839    fdb_kvs_close(db);
2840    fdb_close(dbfile);
2841
2842    // free all resources
2843    fdb_shutdown();
2844
2845    memleak_end();
2846
2847    TEST_RESULT("API wrapper test");
2848}
2849
2850
2851void flush_before_commit_test()
2852{
2853    TEST_INIT();
2854
2855    memleak_start();
2856
2857    int i, r;
2858    int n = 30;
2859    fdb_file_handle *dbfile, *dbfile_txn;
2860    fdb_kvs_handle *db, *db_txn;
2861    fdb_doc **doc = alca(fdb_doc*, n);
2862    fdb_status status;
2863
2864    char keybuf[256], metabuf[256], bodybuf[256];
2865
2866    // remove previous dummy files
2867    r = system(SHELL_DEL" dummy* > errorlog.txt");
2868    (void)r;
2869
2870    fdb_config fconfig = fdb_get_default_config();
2871    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2872    fconfig.buffercache_size = 0;
2873    fconfig.wal_threshold = 5;
2874    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2875    fconfig.purging_interval = 0;
2876    fconfig.compaction_threshold = 0;
2877    fconfig.wal_flush_before_commit = true;
2878
2879    // open db
2880    fdb_open(&dbfile, "dummy1", &fconfig);
2881    fdb_open(&dbfile_txn, "dummy1", &fconfig);
2882    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2883    fdb_kvs_open_default(dbfile_txn, &db_txn, &kvs_config);
2884    status = fdb_set_log_callback(db_txn, logCallbackFunc,
2885                                  (void *) "flush_before_commit_test");
2886    TEST_CHK(status == FDB_RESULT_SUCCESS);
2887
2888    // create docs
2889    for (i=0;i<n;++i){
2890        sprintf(keybuf, "key%d", i);
2891        sprintf(metabuf, "meta%d", i);
2892        sprintf(bodybuf, "body%d", i);
2893        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2894                                (void*)metabuf, strlen(metabuf),
2895                                (void*)bodybuf, strlen(bodybuf));
2896    }
2897
2898    // non-transactional commit first, transactional commit next
2899    fdb_begin_transaction(dbfile_txn, FDB_ISOLATION_READ_COMMITTED);
2900    for (i=0;i<2;++i){
2901        fdb_set(db, doc[i]);
2902    }
2903    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2904    for (i=0;i<2;++i){
2905        fdb_set(db_txn, doc[i]);
2906    }
2907    fdb_end_transaction(dbfile_txn, FDB_COMMIT_NORMAL);
2908    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2909
2910    // transactional commit first, non-transactional commit next
2911    fdb_begin_transaction(dbfile_txn, FDB_ISOLATION_READ_COMMITTED);
2912    for (i=0;i<2;++i){
2913        fdb_set(db_txn, doc[i]);
2914    }
2915    fdb_end_transaction(dbfile_txn, FDB_COMMIT_NORMAL);
2916    for (i=0;i<2;++i){
2917        fdb_set(db, doc[i]);
2918    }
2919    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2920    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2921
2922    // concurrent update (non-txn commit first, txn commit next)
2923    fdb_begin_transaction(dbfile_txn, FDB_ISOLATION_READ_COMMITTED);
2924    for (i=0;i<2;++i){
2925        fdb_set(db_txn, doc[i]);
2926    }
2927    for (i=0;i<2;++i){
2928        fdb_set(db, doc[i]);
2929    }
2930    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2931    fdb_end_transaction(dbfile_txn, FDB_COMMIT_NORMAL);
2932    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2933
2934    // concurrent update (txn commit first, non-txn commit next)
2935    fdb_begin_transaction(dbfile_txn, FDB_ISOLATION_READ_COMMITTED);
2936    for (i=0;i<2;++i){
2937        fdb_set(db, doc[i]);
2938    }
2939    for (i=0;i<2;++i){
2940        fdb_set(db_txn, doc[i]);
2941    }
2942    fdb_end_transaction(dbfile_txn, FDB_COMMIT_NORMAL);
2943    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2944    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2945
2946    // begin transaction
2947    fdb_begin_transaction(dbfile_txn, FDB_ISOLATION_READ_COMMITTED);
2948
2949    // insert docs using transaction
2950    for (i=0;i<10;++i){
2951        fdb_set(db_txn, doc[i]);
2952    }
2953
2954    // insert docs without transaction
2955    for (i=10;i<20;++i){
2956        fdb_set(db, doc[i]);
2957    }
2958
2959    // do compaction
2960    fdb_compact(dbfile, "dummy2");
2961
2962    for (i=20;i<25;++i){
2963        fdb_set(db_txn, doc[i]);
2964    }
2965    // end transaction
2966    fdb_end_transaction(dbfile_txn, FDB_COMMIT_NORMAL);
2967
2968    for (i=25;i<30;++i){
2969        fdb_set(db, doc[i]);
2970    }
2971
2972    // close db file
2973    fdb_close(dbfile);
2974    fdb_close(dbfile_txn);
2975
2976    // free all documents
2977    for (i=0;i<n;++i){
2978        fdb_doc_free(doc[i]);
2979    }
2980
2981    // free all resources
2982    fdb_shutdown();
2983
2984    memleak_end();
2985
2986    TEST_RESULT("flush before commit test");
2987}
2988
2989void flush_before_commit_multi_writers_test()
2990{
2991    TEST_INIT();
2992
2993    memleak_start();
2994
2995    int i, r;
2996    int n = 10;
2997    fdb_file_handle *dbfile1, *dbfile2;
2998    fdb_kvs_handle *db1, *db2;
2999    fdb_doc **doc = alca(fdb_doc*, n);
3000    fdb_doc *rdoc = NULL;
3001    fdb_status status;
3002    fdb_config fconfig;
3003    fdb_kvs_config kvs_config;
3004
3005    char keybuf[256], metabuf[256], bodybuf[256];
3006
3007    // remove previous dummy files
3008    r = system(SHELL_DEL" dummy* > errorlog.txt");
3009    (void)r;
3010
3011    fconfig = fdb_get_default_config();
3012    fconfig.buffercache_size = 0;
3013    fconfig.wal_threshold = 8;
3014    fconfig.flags = FDB_OPEN_FLAG_CREATE;
3015    fconfig.purging_interval = 0;
3016    fconfig.compaction_threshold = 0;
3017    fconfig.wal_flush_before_commit = true;
3018
3019    kvs_config = fdb_get_default_kvs_config();
3020
3021    // open db
3022    fdb_open(&dbfile1, "dummy1", &fconfig);
3023    fdb_kvs_open(dbfile1, &db1, NULL, &kvs_config);
3024
3025    // create & insert docs
3026    for (i=0;i<n;++i){
3027        sprintf(keybuf, "key%d", i);
3028        sprintf(metabuf, "meta%d", i);
3029        sprintf(bodybuf, "body%d", i);
3030        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
3031                                (void*)metabuf, strlen(metabuf),
3032                                (void*)bodybuf, strlen(bodybuf));
3033        fdb_set(db1, doc[i]);
3034    }
3035    fdb_commit(dbfile1, FDB_COMMIT_MANUAL_WAL_FLUSH);
3036
3037    // open second writer
3038    fdb_open(&dbfile2, "dummy1", &fconfig);
3039    fdb_kvs_open(dbfile2, &db2, NULL, &kvs_config);
3040
3041    for (i=0;i<n/2;++i){
3042        sprintf(metabuf, "meta2%d", i);
3043        sprintf(bodybuf, "body2%d(db2)", i);
3044        fdb_doc_update(&doc[i], (void *)metabuf, strlen(metabuf),
3045            (void *)bodybuf, strlen(bodybuf));
3046        fdb_set(db2, doc[i]);
3047    }
3048    for (i=n/2;i<n;++i){
3049        sprintf(metabuf, "meta2%d", i);
3050        sprintf(bodybuf, "body2%d(db1)", i);
3051        fdb_doc_update(&doc[i], (void *)metabuf, strlen(metabuf),
3052            (void *)bodybuf, strlen(bodybuf));
3053        fdb_set(db1, doc[i]);
3054    }
3055
3056    // retrieve before commit
3057    for (i=0;i<n;++i){
3058        sprintf(keybuf, "key%d", i);
3059        sprintf(metabuf, "meta2%d", i);
3060        if (i < n/2) {
3061            sprintf(bodybuf, "body2%d(db2)", i);
3062        } else {
3063            sprintf(bodybuf, "body2%d(db1)", i);
3064        }
3065        // retrieve through db1
3066        fdb_doc_create(&rdoc, (void*)keybuf, strlen(keybuf),
3067                                NULL, 0, NULL, 0);
3068        status = fdb_get(db1, rdoc);
3069        TEST_CHK(status == FDB_RESULT_SUCCESS);
3070        TEST_CMP(metabuf, rdoc->meta, rdoc->metalen);
3071        TEST_CMP(bodybuf, rdoc->body, rdoc->bodylen);
3072        fdb_doc_free(rdoc);
3073        rdoc = NULL;
3074
3075        // retrieve through db2
3076        fdb_doc_create(&rdoc, (void*)keybuf, strlen(keybuf),
3077                                NULL, 0, NULL, 0);
3078        status = fdb_get(db2, rdoc);
3079        TEST_CHK(status == FDB_RESULT_SUCCESS);
3080        TEST_CMP(metabuf, rdoc->meta, rdoc->metalen);
3081        TEST_CMP(bodybuf, rdoc->body, rdoc->bodylen);
3082        fdb_doc_free(rdoc);
3083        rdoc = NULL;
3084    }
3085
3086    fdb_commit(dbfile1, FDB_COMMIT_NORMAL);
3087    fdb_commit(dbfile2, FDB_COMMIT_NORMAL);
3088
3089    // retrieve after commit
3090    for (i=0;i<n;++i){
3091        sprintf(keybuf, "key%d", i);
3092        sprintf(metabuf, "meta2%d", i);
3093        if (i < n/2) {
3094            sprintf(bodybuf, "body2%d(db2)", i);
3095        } else {
3096            sprintf(bodybuf, "body2%d(db1)", i);
3097        }
3098        // retrieve through db1
3099        fdb_doc_create(&rdoc, (void*)keybuf, strlen(keybuf),
3100                                NULL, 0, NULL, 0);
3101        status = fdb_get(db1, rdoc);
3102        TEST_CHK(status == FDB_RESULT_SUCCESS);
3103        TEST_CMP(metabuf, rdoc->meta, rdoc->metalen);
3104        TEST_CMP(bodybuf, rdoc->body, rdoc->bodylen);
3105        fdb_doc_free(rdoc);
3106        rdoc = NULL;
3107
3108        // retrieve through db2
3109        fdb_doc_create(&rdoc, (void*)keybuf, strlen(keybuf),
3110                                NULL, 0, NULL, 0);
3111        status = fdb_get(db2, rdoc);
3112        TEST_CHK(status == FDB_RESULT_SUCCESS);
3113        TEST_CMP(metabuf, rdoc->meta, rdoc->metalen);
3114        TEST_CMP(bodybuf, rdoc->body, rdoc->bodylen);
3115        fdb_doc_free(rdoc);
3116        rdoc = NULL;
3117    }
3118
3119    // close db file
3120    fdb_close(dbfile1);
3121    fdb_close(dbfile2);
3122
3123    // free all documents
3124    for (i=0;i<n;++i){
3125        fdb_doc_free(doc[i]);
3126    }
3127
3128    // free all resources
3129    fdb_shutdown();
3130
3131    memleak_end();
3132
3133    TEST_RESULT("flush before commit with multi writers test");
3134}
3135
3136void auto_commit_test()
3137{
3138    TEST_INIT();
3139
3140    memleak_start();
3141
3142    int i, r;
3143    int n = 5000;
3144    fdb_file_handle *dbfile;
3145    fdb_kvs_handle *db;
3146    fdb_status status;
3147
3148    char key[256], value[256];
3149    void *value_out;
3150    size_t valuelen;
3151
3152    // remove previous dummy files
3153    r = system(SHELL_DEL" dummy* > errorlog.txt");
3154    (void)r;
3155
3156    fdb_config fconfig = fdb_get_default_config();
3157    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
3158    fconfig.buffercache_size = 0;
3159    fconfig.wal_threshold = 4096;
3160    fconfig.flags = FDB_OPEN_FLAG_CREATE;
3161    fconfig.durability_opt = FDB_DRB_ASYNC;
3162    fconfig.auto_commit = true;
3163
3164    // open db
3165    status = fdb_open(&dbfile, "dummy1", &fconfig);
3166    TEST_CHK(status == FDB_RESULT_SUCCESS);
3167    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
3168    TEST_CHK(status == FDB_RESULT_SUCCESS);
3169
3170    // insert docs
3171    for (i=0;i<n;++i){
3172        sprintf(key, "key%d", i);
3173        sprintf(value, "body%d", i);
3174        status = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
3175        TEST_CHK(status == FDB_RESULT_SUCCESS);
3176    }
3177
3178    // retrieve check before close
3179    for (i=0;i<n;++i){
3180        sprintf(key, "key%d", i);
3181        sprintf(value, "body%d", i);
3182        status = fdb_get_kv(db, key, strlen(key)+1, &value_out, &valuelen);
3183        TEST_CHK(status == FDB_RESULT_SUCCESS);
3184        TEST_CMP(value_out, value, valuelen);
3185        fdb_free_block(value_out);
3186    }
3187
3188    // close & reopen
3189    status = fdb_close(dbfile);
3190    TEST_CHK(status == FDB_RESULT_SUCCESS);
3191    status = fdb_open(&dbfile, "dummy1", &fconfig);
3192    TEST_CHK(status == FDB_RESULT_SUCCESS);
3193    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
3194    TEST_CHK(status == FDB_RESULT_SUCCESS);
3195
3196    // retrieve check again
3197    for (i=0;i<n;++i){
3198        sprintf(key, "key%d", i);
3199        sprintf(value, "body%d", i);
3200        status = fdb_get_kv(db, key, strlen(key)+1, &value_out, &valuelen);
3201        TEST_CHK(status == FDB_RESULT_SUCCESS);
3202        TEST_CMP(value_out, value, valuelen);
3203        fdb_free_block(value_out);
3204    }
3205
3206    // free all resources
3207    status = fdb_close(dbfile);
3208    TEST_CHK(status == FDB_RESULT_SUCCESS);
3209    fdb_shutdown();
3210
3211    memleak_end();
3212
3213    TEST_RESULT("auto commit test");
3214}
3215
3216void last_wal_flush_header_test()
3217{
3218    TEST_INIT();
3219
3220    memleak_start();
3221
3222    int i, r;
3223    int n = 30;
3224    fdb_file_handle *dbfile, *dbfile_txn1, *dbfile_txn2;
3225    fdb_kvs_handle *db, *db_txn1, *db_txn2;
3226    fdb_doc **doc = alca(fdb_doc*, n);
3227    fdb_doc *rdoc = NULL;
3228    fdb_status status;
3229
3230    char keybuf[256], metabuf[256], bodybuf[256];
3231
3232    // remove previous dummy files
3233    r = system(SHELL_DEL" dummy* > errorlog.txt");
3234    (void)r;
3235
3236    fdb_config fconfig = fdb_get_default_config();
3237    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
3238    fconfig.buffercache_size = 0;
3239    fconfig.flags = FDB_OPEN_FLAG_CREATE;
3240    fconfig.purging_interval = 0;
3241    fconfig.compaction_threshold = 0;
3242
3243    // open db
3244    fdb_open(&dbfile, "dummy1", &fconfig);
3245    fdb_open(&dbfile_txn1, "dummy1", &fconfig);
3246    fdb_kvs_open_default(dbfile, &db, &kvs_config);
3247    fdb_kvs_open_default(dbfile_txn1, &db_txn1, &kvs_config);
3248
3249    // create docs
3250    for (i=0;i<n;++i){
3251        sprintf(keybuf, "key%d", i);
3252        sprintf(metabuf, "meta%d", i);
3253        sprintf(bodybuf, "body%d", i);
3254        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
3255                                (void*)metabuf, strlen(metabuf),
3256                                (void*)bodybuf, strlen(bodybuf));
3257    }
3258
3259    // insert docs without transaction
3260    for (i=0;i<2;++i) {
3261        fdb_set(db, doc[i]);
3262    }
3263    // insert docs using transaction
3264    fdb_begin_transaction(dbfile_txn1, FDB_ISOLATION_READ_COMMITTED);
3265    for (i=2;i<4;++i){
3266        fdb_set(db_txn1, doc[i]);
3267    }
3268    // commit without transaction
3269    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
3270
3271    // close & reopen db
3272    fdb_close(dbfile);
3273    fdb_close(dbfile_txn1);
3274    fdb_open(&dbfile, "dummy1", &fconfig);
3275    fdb_open(&dbfile_txn1, "dummy1", &fconfig);
3276    fdb_kvs_open_default(dbfile, &db, &kvs_config);
3277    fdb_kvs_open_default(dbfile_txn1, &db_txn1, &kvs_config);
3278
3279    // retrieve check
3280    for (i=0;i<4;++i){
3281        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
3282        status = fdb_get(db, rdoc);
3283        if (i<2) {
3284            TEST_CHK(status == FDB_RESULT_SUCCESS);
3285        } else {
3286            TEST_CHK(status != FDB_RESULT_SUCCESS);
3287        }
3288        fdb_doc_free(rdoc);
3289        rdoc = NULL;
3290    }
3291
3292    // insert docs using transaction
3293    fdb_begin_transaction(dbfile_txn1, FDB_ISOLATION_READ_COMMITTED);
3294    for (i=2;i<4;++i){
3295        fdb_set(db_txn1, doc[i]);
3296    }
3297    // insert docs without transaction
3298    for (i=4;i<6;++i){
3299        fdb_set(db, doc[i]);
3300    }
3301    fdb_end_transaction(dbfile_txn1, FDB_COMMIT_MANUAL_WAL_FLUSH);
3302
3303    // close & reopen db
3304    fdb_close(dbfile);
3305    fdb_close(dbfile_txn1);
3306    fdb_open(&dbfile, "dummy1", &fconfig);
3307    fdb_open(&dbfile_txn1, "dummy1", &fconfig);
3308    fdb_kvs_open_default(dbfile, &db, &kvs_config);
3309    fdb_kvs_open_default(dbfile_txn1, &db_txn1, &kvs_config);
3310
3311    // retrieve check
3312    for (i=0;i<6;++i){
3313        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
3314        status = fdb_get(db, rdoc);
3315        if (i<4) {
3316            TEST_CHK(status == FDB_RESULT_SUCCESS);
3317        } else {
3318            // doesn't matter
3319        }
3320        fdb_doc_free(rdoc);
3321        rdoc = NULL;
3322    }
3323
3324    // insert docs without transaction
3325    for (i=4;i<6;++i) {
3326        fdb_set(db, doc[i]);
3327    }
3328    fdb_begin_transaction(dbfile_txn1, FDB_ISOLATION_READ_COMMITTED);
3329    for (i=6;i<8;++i) {
3330        fdb_set(db_txn1, doc[i]);
3331    }
3332    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
3333
3334    // begin another transaction
3335    fdb_open(&dbfile_txn2, "dummy1", &fconfig);
3336    fdb_kvs_open_default(dbfile_txn2, &db_txn2, &kvs_config);
3337    fdb_begin_transaction(dbfile_txn2, FDB_ISOLATION_READ_COMMITTED);
3338    for (i=8;i<10;++i){
3339        fdb_set(db_txn2, doc[i]);
3340    }
3341    fdb_end_transaction(dbfile_txn2, FDB_COMMIT_MANUAL_WAL_FLUSH);
3342
3343    // close & reopen db
3344    fdb_close(dbfile);
3345    fdb_close(dbfile_txn1);
3346    fdb_close(dbfile_txn2);
3347    fdb_open(&dbfile, "dummy1", &fconfig);
3348    fdb_kvs_open_default(dbfile, &db, &kvs_config);
3349
3350    // retrieve check
3351    for (i=0;i<10;++i){
3352        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
3353        status = fdb_get(db, rdoc);
3354        if (i<6 || i>=8) {
3355            TEST_CHK(status == FDB_RESULT_SUCCESS);
3356        } else {
3357            TEST_CHK(status != FDB_RESULT_SUCCESS);
3358        }
3359        fdb_doc_free(rdoc);
3360        rdoc = NULL;
3361    }
3362
3363    fdb_open(&dbfile_txn1, "dummy1", &fconfig);
3364    fdb_open(&dbfile_txn2, "dummy1", &fconfig);
3365    fdb_kvs_open_default(dbfile_txn1, &db_txn1, &kvs_config);
3366    fdb_kvs_open_default(dbfile_txn2, &db_txn2, &kvs_config);
3367    fdb_begin_transaction(dbfile_txn1, FDB_ISOLATION_READ_COMMITTED);
3368
3369    fdb_set(db, doc[10]);
3370    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
3371
3372    fdb_set(db, doc[11]);
3373    fdb_set(db_txn1, doc[12]);
3374    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
3375
3376    fdb_set(db_txn1, doc[13]);
3377    fdb_begin_transaction(dbfile_txn2, FDB_ISOLATION_READ_COMMITTED);
3378    fdb_set(db_txn2, doc[14]);
3379    fdb_end_transaction(dbfile_txn1, FDB_COMMIT_MANUAL_WAL_FLUSH);
3380
3381    fdb_set(db_txn2, doc[15]);
3382    fdb_set(db, doc[16]);
3383    fdb_end_transaction(dbfile_txn2, FDB_COMMIT_MANUAL_WAL_FLUSH);
3384
3385    fdb_set(db, doc[17]);
3386    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
3387
3388    fdb_close(dbfile);
3389    fdb_close(dbfile_txn1);
3390    fdb_close(dbfile_txn2);
3391    fdb_open(&dbfile, "dummy1", &fconfig);
3392    fdb_kvs_open_default(dbfile, &db, &kvs_config);
3393
3394    // retrieve check
3395    for (i=10;i<18;++i){
3396        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
3397        status = fdb_get(db, rdoc);
3398        TEST_CHK(status == FDB_RESULT_SUCCESS);
3399        fdb_doc_free(rdoc);
3400        rdoc = NULL;
3401    }
3402
3403    fdb_open(&dbfile_txn1, "dummy1", &fconfig);
3404    fdb_kvs_open_default(dbfile_txn1, &db_txn1, &kvs_config);
3405    fdb_begin_transaction(dbfile_txn1, FDB_ISOLATION_READ_COMMITTED);
3406    fdb_set(db_txn1, doc[20]);
3407
3408    fdb_compact(dbfile, "dummy2");
3409
3410    fdb_end_transaction(dbfile_txn1, FDB_COMMIT_MANUAL_WAL_FLUSH);
3411    fdb_close(dbfile);
3412    fdb_close(dbfile_txn1);
3413
3414    // free all documents
3415    for (i=0;i<n;++i){
3416        fdb_doc_free(doc[i]);
3417    }
3418
3419    // free all resources
3420    fdb_shutdown();
3421
3422    memleak_end();
3423
3424    TEST_RESULT("last wal flush header test");
3425}
3426
3427void long_key_test()
3428{
3429    TEST_INIT();
3430    memleak_start();
3431
3432    int i, j, k, idx, r;
3433    int l=3, n=100, m=10;// l: # length groups, n: # prefixes, m: # postfixes
3434    int keylen_limit;
3435    fdb_file_handle *dbfile;
3436    fdb_kvs_handle *db;
3437    fdb_doc **doc = alca(fdb_doc*, l*n*m);
3438    fdb_doc *rdoc = NULL;
3439    fdb_status status;
3440    fdb_file_info info;
3441
3442    char *keybuf;
3443    char metabuf[256], bodybuf[256], temp[256];
3444
3445    // remove previous dummy files
3446    r = system(SHELL_DEL " dummy* > errorlog.txt");
3447    (void)r;
3448
3449    fdb_config fconfig = fdb_get_default_config();
3450    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
3451    fconfig.flags = FDB_OPEN_FLAG_CREATE;
3452    fconfig.purging_interval = 0;
3453    fconfig.compaction_threshold = 0;
3454    fconfig.durability_opt = FDB_DRB_ASYNC;
3455
3456    keybuf = alca(char, FDB_MAX_KEYLEN);
3457
3458    // open db
3459    fdb_open(&dbfile, "dummy1", &fconfig);
3460    fdb_kvs_open_default(dbfile, &db, &kvs_config);
3461
3462    // key structure:
3463    // <------------ <keylen_limit> bytes ------------->
3464    // <-- 8 bytes -->             <-- 8 bytes  -->< 1 >
3465    // [prefix number]____ ... ____[postfix number][ \0]
3466    // e.g.)
3467    // 00000001____ ... ____00000013[\0]
3468
3469    // create docs
3470    for (k=0; k<l; ++k) {
3471        if (k == 0) {
3472            keylen_limit = 32768; // mid-length key
3473        } else if (k == 1) {
3474            keylen_limit = 8192; // short-length key
3475        } else {
3476            keylen_limit = FDB_MAX_KEYLEN; // max-length key
3477        }
3478
3479        memset(keybuf, '_', keylen_limit-1);
3480        keybuf[keylen_limit-1] = 0;
3481
3482        for (i=0;i<n;++i){
3483            // set prefix
3484            sprintf(temp, "%08d", i);
3485            memcpy(keybuf, temp, 8);
3486            for (j=0;j<m;++j){
3487                idx = k*n*m + i*m + j;
3488                // set postfix
3489                sprintf(temp, "%08d", j);
3490                memcpy(keybuf + (keylen_limit-1) - 8, temp, 8);
3491                sprintf(metabuf, "meta%d", idx);
3492                sprintf(bodybuf, "body%d", idx);
3493                fdb_doc_create(&doc[idx], (void*)keybuf, strlen(keybuf)+1,
3494                                          (void*)metabuf, strlen(metabuf)+1,
3495                                          (void*)bodybuf, strlen(bodybuf)+1);
3496            }
3497        }
3498    }
3499
3500    // insert docs
3501    for (i=0;i<l*n*m;++i) {
3502        fdb_set(db, doc[i]);
3503    }
3504    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
3505
3506    // doc count check
3507    fdb_get_file_info(dbfile, &info);
3508    TEST_CHK(info.doc_count == (size_t)l*n*m);
3509
3510    // retrieval check
3511    for (i=0;i<l*n*m;++i){
3512        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
3513        status = fdb_get(db, rdoc);
3514        TEST_CHK(status == FDB_RESULT_SUCCESS);
3515        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
3516        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
3517        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
3518        fdb_doc_free(rdoc);
3519        rdoc = NULL;
3520    }
3521
3522    fdb_close(dbfile);
3523
3524    // free all documents
3525    for (i=0;i<l*n*m;++i){
3526        fdb_doc_free(doc[i]);
3527    }
3528
3529    // free all resources
3530    fdb_shutdown();
3531
3532    memleak_end();
3533
3534    TEST_RESULT("long key test");
3535}
3536
3537void open_multi_files_kvs_test()
3538{
3539    TEST_INIT();
3540    memleak_start();
3541
3542    int i, j, r;
3543    int vb;
3544    int n = 10;
3545    int n_files = 8;
3546    int n_kvs = 128;
3547    char keybuf[256], bodybuf[256];
3548    char fname[256];
3549
3550    fdb_file_handle **