1 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3  *     Copyright 2010 Couchbase, Inc
4  *
5  *   Licensed under the Apache License, Version 2.0 (the "License");
6  *   you may not use this file except in compliance with the License.
7  *   You may obtain a copy of the License at
8  *
9  *       http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *   Unless required by applicable law or agreed to in writing, software
12  *   distributed under the License is distributed on an "AS IS" BASIS,
13  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *   See the License for the specific language governing permissions and
15  *   limitations under the License.
16  */
17 
18 #include <stdio.h>
19 #include <stdlib.h>
20 #include <string.h>
21 #include <stdint.h>
22 #include <time.h>
23 #if !defined(WIN32) && !defined(_WIN32)
24 #include <unistd.h>
25 #endif
26 
27 #include "libforestdb/forestdb.h"
28 #include "test.h"
29 #include "internal_types.h"
30 #include "functional_util.h"
31 
basic_test()32 void basic_test()
33 {
34     TEST_INIT();
35 
36     memleak_start();
37 
38     int i, r;
39     int n = 10;
40     fdb_file_handle *dbfile, *dbfile_rdonly;
41     fdb_kvs_handle *db;
42     fdb_kvs_handle *db_rdonly;
43     fdb_doc **doc = alca(fdb_doc*, n);
44     fdb_doc *rdoc = NULL;
45     fdb_status status;
46 
47     char keybuf[256], metabuf[256], bodybuf[256];
48 
49     // remove previous dummy test files
50     r = system(SHELL_DEL" dummy* > errorlog.txt");
51     (void)r;
52 
53     // Get the ForestDB version
54     const char *version = fdb_get_lib_version();
55     TEST_CHK(version != NULL && strlen(version) > 0);
56 
57     fdb_config fconfig = fdb_get_default_config();
58     fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
59     fconfig.wal_threshold = 1024;
60     fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
61     fconfig.compaction_threshold = 0;
62     fconfig.purging_interval = 1;
63 
64     // Read-Write mode test without a create flag.
65     fconfig.flags = 0;
66     status = fdb_open(&dbfile, "./dummy1", &fconfig);
67     TEST_CHK(status == FDB_RESULT_NO_SUCH_FILE);
68     TEST_CHK(!strcmp(fdb_error_msg(status), "no such file"));
69 
70     // Read-Only mode test: Must not create new file.
71     fconfig.flags = FDB_OPEN_FLAG_RDONLY;
72     status = fdb_open(&dbfile, "./dummy1", &fconfig);
73     TEST_CHK(status == FDB_RESULT_NO_SUCH_FILE);
74 
75     // Read-Only and Create mode: Must not create a new file.
76     fconfig.flags = FDB_OPEN_FLAG_RDONLY | FDB_OPEN_FLAG_CREATE;
77     status = fdb_open(&dbfile, "./dummy1", &fconfig);
78     TEST_CHK(status == FDB_RESULT_INVALID_CONFIG);
79     TEST_CHK(!strcmp(fdb_error_msg(status), "invalid configuration"));
80 
81     // open and close db with a create flag.
82     fconfig.flags = FDB_OPEN_FLAG_CREATE;
83     status = fdb_open(&dbfile, "./dummy1", &fconfig);
84     TEST_CHK(status == FDB_RESULT_SUCCESS);
85     TEST_CHK(!strcmp(fdb_error_msg(status), "success"));
86     const char *file_version = fdb_get_file_version(dbfile);
87     TEST_CHK(file_version != NULL && strlen(file_version) > 0);
88     fdb_close(dbfile);
89 
90     // reopen db
91     r = system(SHELL_DEL" dummy* > errorlog.txt");
92     (void)r;
93     fdb_open(&dbfile, "./dummy1",&fconfig);
94     fdb_kvs_open_default(dbfile, &db, &kvs_config);
95     status = fdb_set_log_callback(db, logCallbackFunc, (void *) "basic_test");
96     TEST_CHK(status == FDB_RESULT_SUCCESS);
97 
98     // insert documents
99     for (i=0;i<n;++i){
100         sprintf(keybuf, "key%d", i);
101         sprintf(metabuf, "meta%d", i);
102         sprintf(bodybuf, "body%d", i);
103         fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
104             (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
105         fdb_set(db, doc[i]);
106     }
107 
108     // remove document #5
109     fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, doc[5]->meta,
110                    doc[5]->metalen, NULL, 0);
111     status = fdb_del(db, rdoc);
112     TEST_CHK(status == FDB_RESULT_SUCCESS);
113     fdb_doc_free(rdoc);
114     rdoc = NULL;
115 
116     // commit
117     fdb_commit(dbfile, FDB_COMMIT_NORMAL);
118 
119     // check the file info
120     fdb_file_info info;
121     fdb_get_file_info(dbfile, &info);
122     TEST_CHK(info.doc_count == 9);
123     TEST_CHK(info.deleted_count == 1);
124     TEST_CHK(info.space_used > 0);
125     TEST_CHK(info.num_kv_stores == 1);
126 
127     fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, NULL, 0, NULL, 0);
128     status = fdb_get_metaonly(db, rdoc);
129     TEST_CHK(status == FDB_RESULT_SUCCESS);
130     TEST_CHK(rdoc->deleted == true);
131     TEST_CMP(rdoc->meta, doc[5]->meta, rdoc->metalen);
132     fdb_doc_free(rdoc);
133     rdoc = NULL;
134 
135     // close the db
136     fdb_kvs_close(db);
137     fdb_close(dbfile);
138 
139     // reopen
140     fdb_open(&dbfile, "./dummy1", &fconfig);
141     fdb_kvs_open_default(dbfile, &db, &kvs_config);
142     status = fdb_set_log_callback(db, logCallbackFunc, (void *) "basic_test");
143     TEST_CHK(status == FDB_RESULT_SUCCESS);
144 
145     // update document #0 and #1
146     for (i=0;i<2;++i){
147         sprintf(metabuf, "meta2%d", i);
148         sprintf(bodybuf, "body2%d", i);
149         fdb_doc_update(&doc[i], (void *)metabuf, strlen(metabuf),
150             (void *)bodybuf, strlen(bodybuf));
151         fdb_set(db, doc[i]);
152     }
153 
154     // commit
155     fdb_commit(dbfile, FDB_COMMIT_NORMAL);
156 
157     // retrieve documents
158     for (i=0;i<n;++i){
159         // search by key
160         fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
161         status = fdb_get(db, rdoc);
162 
163         if (i != 5) {
164             // updated documents
165             TEST_CHK(status == FDB_RESULT_SUCCESS);
166             TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
167             TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
168         } else {
169             // removed document
170             TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
171             TEST_CHK(!strcmp(fdb_error_msg(status), "key not found"));
172         }
173 
174         // free result document
175         fdb_doc_free(rdoc);
176         rdoc = NULL;
177     }
178 
179     // do compaction
180     fdb_compact(dbfile, (char *) "./dummy2");
181 
182     // retrieve documents after compaction
183     for (i=0;i<n;++i){
184         // search by key
185         fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
186         status = fdb_get(db, rdoc);
187 
188         if (i != 5) {
189             // updated documents
190             TEST_CHK(status == FDB_RESULT_SUCCESS);
191             TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
192             TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
193         } else {
194             // removed document
195             TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
196         }
197 
198         // free result document
199         fdb_doc_free(rdoc);
200         rdoc = NULL;
201     }
202 
203     // retrieve documents by sequence number
204     for (i=0; i < n+3; ++i){
205         // search by seq
206         fdb_doc_create(&rdoc, NULL, 0, NULL, 0, NULL, 0);
207         rdoc->seqnum = i + 1;
208         status = fdb_get_byseq(db, rdoc);
209         if ( (i>=2 && i<=4) || (i>=6 && i<=9) || (i>=11 && i<=12)) {
210             // updated documents
211             TEST_CHK(status == FDB_RESULT_SUCCESS);
212         } else {
213             // removed document
214             TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
215         }
216 
217         // free result document
218         fdb_doc_free(rdoc);
219         rdoc = NULL;
220     }
221 
222     // update document #5 with an empty doc body.
223     fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, doc[5]->meta,
224                    doc[5]->metalen, NULL, 0);
225     status = fdb_set(db, rdoc);
226     TEST_CHK(status == FDB_RESULT_SUCCESS);
227     fdb_doc_free(rdoc);
228     rdoc = NULL;
229     fdb_commit(dbfile, FDB_COMMIT_NORMAL);
230 
231     // Check document #5 with respect to metadata and doc body.
232     fdb_doc_create(&rdoc, doc[5]->key, doc[5]->keylen, NULL, 0, NULL, 0);
233     status = fdb_get(db, rdoc);
234     TEST_CHK(status == FDB_RESULT_SUCCESS);
235     TEST_CHK(memcmp(rdoc->meta, doc[5]->meta, rdoc->metalen) == 0);
236     TEST_CHK(rdoc->body == NULL);
237     TEST_CHK(rdoc->bodylen == 0);
238     fdb_doc_free(rdoc);
239     rdoc = NULL;
240 
241     // Read-Only mode test: Open succeeds if file exists, but disallow writes
242     fconfig.flags = FDB_OPEN_FLAG_RDONLY;
243     status = fdb_open(&dbfile_rdonly, "./dummy2", &fconfig);
244     TEST_CHK(status == FDB_RESULT_SUCCESS);
245     status = fdb_kvs_open_default(dbfile_rdonly, &db_rdonly, &kvs_config);
246     TEST_CHK(status == FDB_RESULT_SUCCESS);
247 
248     fdb_doc_create(&rdoc, doc[0]->key, doc[0]->keylen, NULL, 0, NULL, 0);
249     status = fdb_get(db_rdonly, rdoc);
250     TEST_CHK(status == FDB_RESULT_SUCCESS);
251 
252     status = fdb_set_log_callback(db_rdonly, logCallbackFunc,
253                                   (void *) "basic_test");
254     TEST_CHK(status == FDB_RESULT_SUCCESS);
255 
256     status = fdb_set(db_rdonly, doc[i]);
257     TEST_CHK(status == FDB_RESULT_RONLY_VIOLATION);
258     TEST_CHK(!strcmp(fdb_error_msg(status), "database is read-only"));
259 
260     status = fdb_commit(dbfile_rdonly, FDB_COMMIT_NORMAL);
261     TEST_CHK(status == FDB_RESULT_RONLY_VIOLATION);
262 
263     fdb_doc_free(rdoc);
264     rdoc = NULL;
265     fdb_kvs_close(db_rdonly);
266     fdb_close(dbfile_rdonly);
267 
268     // free all documents
269     for (i=0;i<n;++i){
270         fdb_doc_free(doc[i]);
271     }
272 
273     // do one more compaction
274     fdb_compact(dbfile, (char *) "./dummy3");
275 
276     // close db file
277     fdb_kvs_close(db);
278     fdb_close(dbfile);
279 
280     // free all resources
281     fdb_shutdown();
282 
283     memleak_end();
284 
285     TEST_RESULT("basic test");
286 }
287 
init_test()288 void init_test()
289 {
290     TEST_INIT();
291     memleak_start();
292 
293     int r;
294     fdb_status status;
295     fdb_file_handle *dbfile;
296     fdb_kvs_handle *db;
297     fdb_config fconfig = fdb_get_default_config();
298     fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
299 
300     r = system(SHELL_DEL" dummy* > errorlog.txt");
301     (void)r;
302 
303     status = fdb_init(&fconfig);
304     TEST_CHK(status == FDB_RESULT_SUCCESS);
305 
306     status = fdb_open(&dbfile, "./dummy1", &fconfig);
307     TEST_CHK(status == FDB_RESULT_SUCCESS);
308     status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
309     TEST_CHK(status == FDB_RESULT_SUCCESS);
310 
311     status = fdb_close(dbfile);
312     TEST_CHK(status == FDB_RESULT_SUCCESS);
313     status = fdb_shutdown();
314     TEST_CHK(status == FDB_RESULT_SUCCESS);
315 
316     memleak_end();
317     TEST_RESULT("init test");
318 }
319 
set_get_max_keylen()320 void set_get_max_keylen()
321 {
322     TEST_INIT();
323     memleak_start();
324 
325     int r;
326     static const int len = FDB_MAX_KEYLEN;
327     char keybuf[len];
328     void *rvalue;
329     size_t rvalue_len;
330     static const char *achar = "a";
331 
332     fdb_status status;
333     fdb_file_handle *dbfile;
334     fdb_kvs_handle *db;
335     fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
336     fdb_config fconfig = fdb_get_default_config();
337     fconfig.chunksize = 16;
338 
339 
340     r = system(SHELL_DEL" dummy* > errorlog.txt");
341     (void)r;
342 
343     for (int i = 0; i < len; ++i) {
344         keybuf[i] = *achar;
345     }
346     keybuf[len-1] = '\0';
347 
348     // open db
349     status = fdb_open(&dbfile, "./dummy1", &fconfig);
350     TEST_CHK(status == FDB_RESULT_SUCCESS);
351     status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
352     TEST_CHK(status == FDB_RESULT_SUCCESS);
353 
354     // set kv
355     status = fdb_set_kv(db, keybuf, strlen(keybuf), NULL, 0);
356     TEST_CHK(status == FDB_RESULT_SUCCESS);
357 
358     // get NULL pointer
359     status = fdb_get(db, NULL);
360     TEST_CHK(status == FDB_RESULT_INVALID_ARGS);
361 
362     // get kv
363     status = fdb_get_kv(db, keybuf, strlen(keybuf), &rvalue, &rvalue_len);
364     TEST_CHK(status == FDB_RESULT_SUCCESS);
365 
366     fdb_close(dbfile);
367     fdb_shutdown();
368 
369     memleak_end();
370     TEST_RESULT("set get max keylen");
371 }
372 
config_test()373 void config_test()
374 {
375     TEST_INIT();
376 
377     memleak_start();
378 
379     fdb_file_handle *dbfile;
380     fdb_kvs_handle *db;
381     fdb_status status;
382     fdb_config fconfig;
383     fdb_kvs_config kvs_config;
384     int nfiles = 4;
385     int i;
386     size_t bcache_space_used;
387     char fname[256];
388 
389     // remove previous dummy test files
390     int r = system(SHELL_DEL" dummy* > errorlog.txt");
391     (void)r;
392 
393     bcache_space_used = fdb_get_buffer_cache_used();
394     TEST_CHK(bcache_space_used == 0);
395 
396     fconfig = fdb_get_default_config();
397     fconfig.buffercache_size= (uint64_t) -1;
398     status = fdb_open(&dbfile, "./dummy1", &fconfig);
399     TEST_CHK(status == FDB_RESULT_TOO_BIG_BUFFER_CACHE);
400 
401     fconfig = fdb_get_default_config();
402     fconfig.max_writer_lock_prob = 120;
403     status = fdb_open(&dbfile, "./dummy1", &fconfig);
404     TEST_CHK(status == FDB_RESULT_INVALID_CONFIG);
405 
406     fconfig = fdb_get_default_config();
407     kvs_config = fdb_get_default_kvs_config();
408     for (i = nfiles; i; --i) {
409         sprintf(fname, "dummy%d", i);
410         status = fdb_open(&dbfile, fname, &fconfig);
411         TEST_CHK(status == FDB_RESULT_SUCCESS);
412         status = fdb_kvs_open(dbfile, &db, "justonekv", &kvs_config);
413         TEST_CHK(status == FDB_RESULT_SUCCESS);
414 
415         bcache_space_used = fdb_get_buffer_cache_used();
416 
417         fdb_file_info finfo;
418         status = fdb_get_file_info(dbfile, &finfo);
419         TEST_CHK(status == FDB_RESULT_SUCCESS);
420         // Since V3 magic number, 7 blocks are used:
421         // 4 superblocks + KV name header + Stale-tree root node + DB header
422         TEST_CHK(finfo.file_size == fconfig.blocksize * 7);
423         // Buffercache must only have KV name header + stale-tree root
424         TEST_CHK(bcache_space_used == fconfig.blocksize * 2);
425 
426         status = fdb_close(dbfile);
427         TEST_CHK(status == FDB_RESULT_SUCCESS);
428     }
429 
430     status = fdb_open(&dbfile, fname, &fconfig);
431     TEST_CHK(status == FDB_RESULT_SUCCESS);
432     status = fdb_kvs_open(dbfile, &db, "justonekv", &kvs_config);
433     TEST_CHK(status == FDB_RESULT_SUCCESS);
434     status = fdb_set_kv(db, (void*)"key", 3, (void*)"body", 5);
435     TEST_CHK(status == FDB_RESULT_SUCCESS);
436 
437     bcache_space_used = fdb_get_buffer_cache_used();
438 
439     // Since V3 magic number, 8 blocks are used:
440     // 7 blocks created eariler + document block for KV pair
441     TEST_CHK(bcache_space_used == fconfig.blocksize * 8);
442 
443     fdb_close(dbfile);
444 
445     fdb_shutdown();
446 
447     memleak_end();
448     TEST_RESULT("forestdb config test");
449 }
450 
delete_reopen_test()451 void delete_reopen_test()
452 {
453     TEST_INIT();
454     memleak_start();
455 
456     int r;
457     fdb_file_handle *fh;
458     fdb_kvs_handle *db;
459     fdb_status status;
460     fdb_config fconfig;
461 
462     r = system(SHELL_DEL " dummy* > errorlog.txt");
463     (void)r;
464 
465     fconfig = fdb_get_default_config();
466     fconfig.buffercache_size = 0;
467     fconfig.num_compactor_threads = 1;
468     status = fdb_open(&fh, "./dummy3", &fconfig);
469     TEST_CHK(status == FDB_RESULT_SUCCESS);
470 
471     status = fdb_kvs_open_default(fh, &db, NULL);
472     TEST_CHK(status == FDB_RESULT_SUCCESS);
473 
474     status = fdb_begin_transaction(fh, FDB_ISOLATION_READ_COMMITTED);
475     TEST_CHK(status == FDB_RESULT_SUCCESS);
476 
477     status = fdb_set_kv(db, (void *) "foo", 3, (void *)"value", 5);
478     TEST_CHK(status == FDB_RESULT_SUCCESS);
479 
480     status = fdb_end_transaction(fh, FDB_COMMIT_NORMAL);
481     TEST_CHK(status == FDB_RESULT_SUCCESS);
482 
483     void *value;
484     size_t valueSize;
485     status = fdb_get_kv(db, (void*)"foo", 3, &value, &valueSize);
486     TEST_CHK(status == FDB_RESULT_SUCCESS);
487 
488     TEST_CHK(valueSize == 5);
489     TEST_CMP(value, "value", 5);
490     fdb_free_block(value);
491 
492     status = fdb_begin_transaction(fh, FDB_ISOLATION_READ_COMMITTED);
493     TEST_CHK(status == FDB_RESULT_SUCCESS);
494 
495     status = fdb_del_kv(db, "foo", 3);
496     TEST_CHK(status == FDB_RESULT_SUCCESS);
497 
498     status = fdb_end_transaction(fh, FDB_COMMIT_NORMAL);
499     TEST_CHK(status == FDB_RESULT_SUCCESS);
500 
501     status = fdb_get_kv(db, "foo", 3, &value, &valueSize);
502     TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
503 
504     status = fdb_close(fh);
505     TEST_CHK(status == FDB_RESULT_SUCCESS);
506 
507     // Reopen:
508     status = fdb_open(&fh, "./dummy3", &fconfig);
509     TEST_CHK(status == FDB_RESULT_SUCCESS);
510 
511     status = fdb_kvs_open_default(fh, &db, NULL);
512     TEST_CHK(status == FDB_RESULT_SUCCESS);
513 
514     status = fdb_get_kv(db, "foo", 3, &value, &valueSize);
515     TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
516 
517     status = fdb_close(fh);
518     TEST_CHK(status == FDB_RESULT_SUCCESS);
519 
520     fdb_shutdown();
521 
522     memleak_end();
523     TEST_RESULT("end trans delete & reopen passed");
524 }
525 
deleted_doc_get_api_test()526 void deleted_doc_get_api_test()
527 {
528     TEST_INIT();
529     memleak_start();
530 
531     int r;
532     fdb_file_handle *dbfile;
533     fdb_kvs_handle *db;
534     fdb_doc _doc;
535     fdb_doc *doc = &_doc;
536     fdb_doc *rdoc;
537     fdb_status status;
538     fdb_config fconfig;
539     fdb_kvs_config kvs_config;
540     char keybuf[256], bodybuf[256];
541 
542     r = system(SHELL_DEL " dummy* > errorlog.txt");
543     (void)r;
544 
545     memset(doc, 0, sizeof(fdb_doc));
546     doc->key = &keybuf[0];
547     doc->body = &bodybuf[0];
548     doc->seqnum = SEQNUM_NOT_USED;
549 
550     // open dbfile
551     fconfig = fdb_get_default_config();
552     fconfig.purging_interval = 1;
553     fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
554     kvs_config = fdb_get_default_kvs_config();
555     status = fdb_open(&dbfile, "./dummy1", &fconfig);
556     TEST_CHK(status == FDB_RESULT_SUCCESS);
557     status = fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
558     TEST_CHK(status == FDB_RESULT_SUCCESS);
559 
560     sprintf(keybuf, "key");
561     sprintf(bodybuf, "body");
562     doc->keylen = strlen(keybuf);
563     doc->bodylen = strlen(bodybuf);
564     status = fdb_set(db, doc);
565     TEST_CHK(status == FDB_RESULT_SUCCESS);
566 
567     // Commit the doc so it goes into main index
568     status = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
569     TEST_CHK(status == FDB_RESULT_SUCCESS);
570 
571     // Delete the doc
572     status = fdb_del(db, doc);
573     TEST_CHK(status == FDB_RESULT_SUCCESS);
574 
575     // Commit the doc with wal flush so the delete is appended into the file
576     status = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
577     TEST_CHK(status == FDB_RESULT_SUCCESS);
578 
579 
580     fdb_doc_create(&rdoc, keybuf, doc->keylen, NULL, 0, NULL, 0);
581 
582     // Deleted document should be accessible via fdb_get_metaonly()
583     status = fdb_get_metaonly(db, rdoc);
584     TEST_CHK(status == FDB_RESULT_SUCCESS);
585     TEST_CHK(rdoc->deleted);
586     rdoc->deleted = false;
587 
588     // Deleted document should be accessible via fdb_get_metaonly_byseq()
589     status = fdb_get_metaonly_byseq(db, rdoc);
590     TEST_CHK(status == FDB_RESULT_SUCCESS);
591     TEST_CHK(rdoc->deleted);
592     rdoc->deleted = false;
593 
594     // Deleted document should be accessible via fdb_get_byoffset()
595     // But the return code must be FDB_RESULT_KEY_NOT_FOUND!
596     status = fdb_get_byoffset(db, rdoc);
597     TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
598     TEST_CHK(rdoc->deleted);
599     rdoc->deleted = false;
600 
601     // Deleted document should NOT be accessible via fdb_get()
602     status = fdb_get(db, rdoc);
603     TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
604     TEST_CHK(!rdoc->deleted);
605     rdoc->deleted = false;
606 
607     status = fdb_get_byseq(db, rdoc);
608     TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
609     TEST_CHK(!rdoc->deleted);
610 
611     fdb_doc_free(rdoc);
612     // close without commit
613     status = fdb_kvs_close(db);
614     TEST_CHK(status == FDB_RESULT_SUCCESS);
615     status = fdb_close(dbfile);
616     TEST_CHK(status == FDB_RESULT_SUCCESS);
617     fdb_shutdown();
618     memleak_end();
619     TEST_RESULT("deleted doc get api test");
620 }
621 
deleted_doc_stat_test()622 void deleted_doc_stat_test()
623 {
624     TEST_INIT();
625     memleak_start();
626 
627     int r;
628     fdb_file_handle *dbfile;
629     fdb_kvs_handle *db;
630     fdb_doc _doc;
631     fdb_doc *doc = &_doc;
632     fdb_doc *rdoc;
633     fdb_status status;
634     fdb_config fconfig;
635     fdb_file_info info;
636     fdb_kvs_config kvs_config;
637     char keybuf[256], bodybuf[256];
638 
639     r = system(SHELL_DEL " dummy* > errorlog.txt");
640     (void)r;
641 
642     memset(doc, 0, sizeof(fdb_doc));
643     doc->key = &keybuf[0];
644     doc->body = &bodybuf[0];
645     doc->seqnum = SEQNUM_NOT_USED;
646 
647     // open dbfile
648     fconfig = fdb_get_default_config();
649     fconfig.purging_interval = 0;
650     kvs_config = fdb_get_default_kvs_config();
651     status = fdb_open(&dbfile, "./dummy1", &fconfig);
652     TEST_CHK(status == FDB_RESULT_SUCCESS);
653     status = fdb_kvs_open(dbfile, &db, "main", &kvs_config);
654     TEST_CHK(status == FDB_RESULT_SUCCESS);
655 
656     sprintf(keybuf, "K"); // This is necessary to set keysize to 2 bytes so
657     sprintf(bodybuf, "body"); // it matches KV_header doc's keysize of 10
658     doc->keylen = strlen(keybuf) + 1; // in multi-kv mode and hits MB-16491
659     doc->bodylen = strlen(bodybuf) + 1;
660     status = fdb_set(db, doc);
661     TEST_CHK(status == FDB_RESULT_SUCCESS);
662 
663     // Delete the doc
664     status = fdb_del(db, doc);
665     TEST_CHK(status == FDB_RESULT_SUCCESS);
666 
667     // Fetch the doc back
668     fdb_doc_create(&rdoc, doc->key, doc->keylen, NULL, 0, NULL, 0);
669     status = fdb_get(db, rdoc);
670     TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
671     fdb_doc_free(rdoc);
672 
673     // check the file info
674     fdb_get_file_info(dbfile, &info);
675     TEST_CHK(info.doc_count == 0);
676 
677     status = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
678     TEST_CHK(status == FDB_RESULT_SUCCESS);
679     // check the file info again after commit..
680     fdb_get_file_info(dbfile, &info);
681     TEST_CHK(info.doc_count == 0);
682 
683     status = fdb_kvs_close(db);
684     TEST_CHK(status == FDB_RESULT_SUCCESS);
685     status = fdb_close(dbfile);
686     TEST_CHK(status == FDB_RESULT_SUCCESS);
687     fdb_shutdown();
688     memleak_end();
689     TEST_RESULT("deleted doc stat test");
690 }
691 
692 // MB-16312
complete_delete_test()693 void complete_delete_test()
694 {
695     TEST_INIT();
696 
697     int i, r, n = 1000;
698     fdb_file_handle *dbfile;
699     fdb_kvs_handle *db;
700     fdb_iterator *fit;
701     fdb_config config;
702     fdb_kvs_config kvs_config;
703     fdb_status s; (void)s;
704     char path[256];
705     char keybuf[256], valuebuf[256];
706 
707     memleak_start();
708 
709     sprintf(path, "./dummy1");
710 
711     r = system(SHELL_DEL " dummy* > errorlog.txt");
712     (void)r;
713 
714     config = fdb_get_default_config();
715     config.buffercache_size = 0;
716     kvs_config = fdb_get_default_kvs_config();
717 
718     fdb_open(&dbfile, path, &config);
719     s = fdb_kvs_open(dbfile, &db, "db1", &kvs_config);
720     TEST_CHK(s == FDB_RESULT_SUCCESS);
721 
722     for (i=0;i<n;++i){
723         sprintf(keybuf, "key%05d", i);
724         sprintf(valuebuf, "value%05d", i);
725         s = fdb_set_kv(db, keybuf, strlen(keybuf)+1, valuebuf, strlen(valuebuf)+1);
726         TEST_CHK(s == FDB_RESULT_SUCCESS);
727     }
728     fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
729 
730     for (i=0;i<n;++i){
731         sprintf(keybuf, "key%05d", i);
732         sprintf(valuebuf, "value%05d", i);
733         s = fdb_del_kv(db, keybuf, strlen(keybuf)+1);
734         TEST_CHK(s == FDB_RESULT_SUCCESS);
735     }
736     fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
737 
738     s = fdb_iterator_init(db, &fit, NULL, 0, NULL, 0, FDB_ITR_NO_DELETES);
739     TEST_CHK(s == FDB_RESULT_SUCCESS);
740     s = fdb_iterator_close(fit);
741     TEST_CHK(s == FDB_RESULT_SUCCESS);
742 
743     fdb_close(dbfile);
744     fdb_shutdown();
745 
746     memleak_end();
747     TEST_RESULT("complete delete");
748 }
749 
large_batch_write_no_commit_test()750 void large_batch_write_no_commit_test()
751 {
752     TEST_INIT();
753     memleak_start();
754 
755     int i, r;
756     int n = 500000;
757     fdb_file_handle *dbfile;
758     fdb_kvs_handle *db;
759     fdb_doc **doc = (fdb_doc **) malloc(sizeof(fdb_doc *) * n);
760     fdb_status status;
761     fdb_config fconfig;
762     fdb_kvs_config kvs_config;
763     char keybuf[256], metabuf[256], bodybuf[256];
764 
765     r = system(SHELL_DEL " dummy* > errorlog.txt");
766     (void)r;
767 
768     // open dbfile
769     fconfig = fdb_get_default_config();
770     kvs_config = fdb_get_default_kvs_config();
771     status = fdb_open(&dbfile, "./dummy1", &fconfig);
772     TEST_CHK(status == FDB_RESULT_SUCCESS);
773     status = fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
774     TEST_CHK(status == FDB_RESULT_SUCCESS);
775 
776     // Write 500K docs to eject and flush some dirty pages into disk.
777     for (i=0;i<n;++i){
778         sprintf(keybuf, "key%128d", i);
779         sprintf(metabuf, "meta%128d", i);
780         sprintf(bodybuf, "body%128d", i);
781         fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
782             (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
783         fdb_set(db, doc[i]);
784         fdb_doc_free(doc[i]);
785     }
786 
787     // close without commit
788     status = fdb_kvs_close(db);
789     TEST_CHK(status == FDB_RESULT_SUCCESS);
790     status = fdb_close(dbfile);
791     TEST_CHK(status == FDB_RESULT_SUCCESS);
792 
793     status = fdb_open(&dbfile, "./dummy1", &fconfig);
794     TEST_CHK(status == FDB_RESULT_NO_DB_HEADERS ||
795              status == FDB_RESULT_SUCCESS); // No dirty pages are flushed into disk.
796     if (status == FDB_RESULT_SUCCESS) {
797         status = fdb_close(dbfile);
798         TEST_CHK(status == FDB_RESULT_SUCCESS);
799     }
800 
801     free(doc);
802     fdb_shutdown();
803     memleak_end();
804     TEST_RESULT("large batch write test with no commits");
805 }
806 
set_get_meta_test()807 void set_get_meta_test()
808 {
809     TEST_INIT();
810     memleak_start();
811 
812     int r;
813     char keybuf[256];
814     fdb_file_handle *dbfile;
815     fdb_kvs_handle *db;
816     fdb_doc *rdoc;
817     fdb_status status;
818     fdb_config fconfig = fdb_get_default_config();
819     fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
820     fconfig.wal_threshold = 1024;
821     fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
822     fconfig.purging_interval = 1;
823     fconfig.flags = FDB_OPEN_FLAG_CREATE;
824 
825     // remove previous dummy files
826     r = system(SHELL_DEL" dummy* > errorlog.txt");
827     (void)r;
828 
829     // open db
830     fdb_open(&dbfile, "./dummy1", &fconfig);
831     fdb_kvs_open(dbfile, &db, "db1", &kvs_config);
832 
833     sprintf(keybuf, "key%d", 0);
834     fdb_doc_create(&rdoc, keybuf, strlen(keybuf), NULL, 0, NULL, 0);
835     fdb_set(db, rdoc);
836     status = fdb_get(db, rdoc);
837     assert(status == FDB_RESULT_SUCCESS);
838     status = fdb_get_byoffset(db, rdoc);
839     TEST_CHK(status == FDB_RESULT_SUCCESS);
840     status = fdb_get_metaonly(db, rdoc);
841     assert(status == FDB_RESULT_SUCCESS);
842     status = fdb_get_metaonly_byseq(db, rdoc);
843     TEST_CHK(status == FDB_RESULT_SUCCESS);
844 
845     fdb_del(db, rdoc);
846     status = fdb_get(db, rdoc);
847     assert(status == FDB_RESULT_KEY_NOT_FOUND);
848     assert(rdoc->deleted == true);
849 
850     status = fdb_get_metaonly(db, rdoc);
851     assert(status == FDB_RESULT_SUCCESS);
852     assert(rdoc->deleted == true);
853 
854     status = fdb_get_metaonly_byseq(db, rdoc);
855     TEST_CHK(status == FDB_RESULT_SUCCESS);
856     assert(rdoc->deleted == true);
857 
858     status = fdb_get_byoffset(db, rdoc);
859     TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
860     assert(rdoc->deleted == true);
861 
862 
863     fdb_doc_free(rdoc);
864     fdb_kvs_close(db);
865     fdb_close(dbfile);
866     fdb_shutdown();
867 
868     memleak_end();
869     TEST_RESULT("set get meta test");
870 }
871 
long_filename_test()872 void long_filename_test()
873 {
874     TEST_INIT();
875     memleak_start();
876 
877     int i, j, r;
878     int n=15, m=1000;
879     char keyword[] = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
880     char filename[4096], cmd[4096], temp[4096];
881     fdb_file_handle *dbfile;
882     fdb_kvs_handle *db;
883     fdb_config config;
884     fdb_kvs_config kvs_config;
885     fdb_status s;
886     size_t rvalue_len;
887     char key[256], value[256];
888     void *rvalue;
889 
890     config = fdb_get_default_config();
891     kvs_config = fdb_get_default_kvs_config();
892     sprintf(temp, SHELL_DMT"%s", keyword);
893 
894     // filename longer than 1024 bytes
895     sprintf(filename, "%s", keyword);
896     while (strlen(filename) < 1024) {
897         strcat(filename, keyword);
898     }
899     s = fdb_open(&dbfile, filename, &config);
900     TEST_CHK(s == FDB_RESULT_TOO_LONG_FILENAME);
901 
902     // make nested directories for long path
903     // but shorter than 1024 bytes (windows: 256 bytes)
904     sprintf(cmd, SHELL_RMDIR" %s", keyword);
905     r = system(cmd);
906     (void)r;
907     for (i=0;i<n;++i) {
908         sprintf(cmd, SHELL_MKDIR" %s", keyword);
909         for (j=0;j<i;++j){
910             strcat(cmd, temp);
911         }
912         if (strlen(cmd) > SHELL_MAX_PATHLEN) break;
913         r = system(cmd);
914         (void)r;
915     }
916 
917     // create DB file
918     sprintf(filename, "%s", keyword);
919     for (j=0;j<i-1;++j){
920         strcat(filename, temp);
921     }
922     strcat(filename, SHELL_DMT"dbfile");
923     s = fdb_open(&dbfile, filename, &config);
924     TEST_CHK(s == FDB_RESULT_SUCCESS);
925     s = fdb_kvs_open_default(dbfile, &db, &kvs_config);
926     TEST_CHK(s == FDB_RESULT_SUCCESS);
927 
928     // === write ===
929     for (i=0;i<m;++i){
930         sprintf(key, "key%08d", i);
931         sprintf(value, "value%08d", i);
932         s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
933         TEST_CHK(s == FDB_RESULT_SUCCESS);
934     }
935     s = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
936     TEST_CHK(s == FDB_RESULT_SUCCESS);
937 
938     // === read ===
939     for (i=0;i<m;++i){
940         sprintf(key, "key%08d", i);
941         s = fdb_get_kv(db, key, strlen(key)+1, &rvalue, &rvalue_len);
942         TEST_CHK(s == FDB_RESULT_SUCCESS);
943         fdb_free_block(rvalue);
944     }
945 
946     s = fdb_kvs_close(db);
947     TEST_CHK(s == FDB_RESULT_SUCCESS);
948     s = fdb_close(dbfile);
949     TEST_CHK(s == FDB_RESULT_SUCCESS);
950     s = fdb_shutdown();
951     TEST_CHK(s == FDB_RESULT_SUCCESS);
952 
953     sprintf(cmd, SHELL_RMDIR" %s", keyword);
954     r = system(cmd);
955     (void)r;
956 
957     memleak_end();
958     TEST_RESULT("long filename test");
959 }
960 
error_to_str_test()961 void error_to_str_test()
962 {
963     TEST_INIT();
964     memleak_start();
965     int i;
966     const char *err_msg;
967 
968     for (i = FDB_RESULT_SUCCESS; i >= FDB_RESULT_LAST; --i) {
969         err_msg = fdb_error_msg((fdb_status)i);
970         // Verify that all error codes have corresponding error messages
971         TEST_CHK(strcmp(err_msg, "unknown error"));
972     }
973 
974     err_msg = fdb_error_msg((fdb_status)i);
975     // Verify that the last error code has been checked
976     TEST_CHK(!strcmp(err_msg, "unknown error"));
977 
978     memleak_end();
979     TEST_RESULT("error to string message test");
980 }
981 
seq_tree_exception_test()982 void seq_tree_exception_test()
983 {
984     TEST_INIT();
985 
986     memleak_start();
987 
988     int i, r;
989     int n = 10;
990     fdb_file_handle *dbfile;
991     fdb_kvs_handle *db;
992     fdb_doc **doc = alca(fdb_doc*, n);
993     fdb_doc *rdoc = NULL;
994     fdb_status status;
995     fdb_iterator *it;
996 
997     char keybuf[256], metabuf[256], bodybuf[256];
998 
999     // remove previous dummy files
1000     r = system(SHELL_DEL" dummy* > errorlog.txt");
1001     (void)r;
1002 
1003     fdb_config fconfig = fdb_get_default_config();
1004     fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1005     fconfig.seqtree_opt = FDB_SEQTREE_NOT_USE;
1006 
1007     // open db
1008     fdb_open(&dbfile, "./dummy1", &fconfig);
1009     fdb_kvs_open_default(dbfile, &db, &kvs_config);
1010     status = fdb_set_log_callback(db, logCallbackFunc,
1011                                   (void *) "seq_tree_exception_test");
1012     TEST_CHK(status == FDB_RESULT_SUCCESS);
1013 
1014     // insert documents
1015     for (i=0;i<n;++i){
1016         sprintf(keybuf, "key%d", i);
1017         sprintf(metabuf, "meta%d", i);
1018         sprintf(bodybuf, "body%d", i);
1019         fdb_doc_create(&doc[i],
1020                        (void *)keybuf,  strlen(keybuf),
1021                        (void *)metabuf, strlen(metabuf),
1022                        (void *)bodybuf, strlen(bodybuf));
1023         fdb_set(db, doc[i]);
1024     }
1025 
1026     // commit
1027     fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1028 
1029     status = fdb_compact(dbfile, NULL);
1030     TEST_CHK(status == FDB_RESULT_SUCCESS);
1031 
1032     // close the db
1033     fdb_kvs_close(db);
1034     fdb_close(dbfile);
1035 
1036     // reopen with seq tree option
1037     fconfig.seqtree_opt = FDB_SEQTREE_USE;
1038     status = fdb_open(&dbfile, "./dummy1", &fconfig);
1039     // must succeed
1040     TEST_CHK(status == FDB_RESULT_SUCCESS);
1041     status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
1042     TEST_CHK(status == FDB_RESULT_SUCCESS);
1043     status = fdb_set_log_callback(db, logCallbackFunc,
1044                                   (void *) "seq_tree_exception_test");
1045     TEST_CHK(status == FDB_RESULT_SUCCESS);
1046 
1047     // search by seq
1048     fdb_doc_create(&rdoc, NULL, 0, NULL, 0, NULL, 0);
1049     rdoc->seqnum = 1;
1050     status = fdb_get_byseq(db, rdoc);
1051     // must fail
1052     TEST_CHK(status != FDB_RESULT_SUCCESS);
1053 
1054     // search meta by seq
1055     status = fdb_get_metaonly_byseq(db, rdoc);
1056     // must fail
1057     TEST_CHK(status != FDB_RESULT_SUCCESS);
1058 
1059     // init iterator by seq
1060     status = fdb_iterator_sequence_init(db , &it, 0, 0, FDB_ITR_NONE);
1061     // must fail
1062     TEST_CHK(status != FDB_RESULT_SUCCESS);
1063 
1064     // close db file
1065     fdb_kvs_close(db);
1066     fdb_close(dbfile);
1067 
1068     // free all documents
1069     free(rdoc);
1070     for (i=0;i<n;++i){
1071         fdb_doc_free(doc[i]);
1072     }
1073 
1074     // remove previous dummy files
1075     r = system(SHELL_DEL" dummy* > errorlog.txt");
1076     (void)r;
1077 
1078     // open db
1079     fconfig.seqtree_opt = FDB_SEQTREE_USE;
1080     fdb_open(&dbfile, "./dummy1", &fconfig);
1081     fdb_kvs_open_default(dbfile, &db, &kvs_config);
1082     status = fdb_set_log_callback(db, logCallbackFunc,
1083                                   (void *) "seq_tree_exception_test");
1084     TEST_CHK(status == FDB_RESULT_SUCCESS);
1085 
1086     // insert documents
1087     for (i=0;i<n;++i){
1088         sprintf(keybuf, "key%d", i);
1089         sprintf(metabuf, "meta%d", i);
1090         sprintf(bodybuf, "body%d", i);
1091         fdb_doc_create(&doc[i],
1092                        (void *)keybuf,  strlen(keybuf),
1093                        (void *)metabuf, strlen(metabuf),
1094                        (void *)bodybuf, strlen(bodybuf));
1095         fdb_set(db, doc[i]);
1096     }
1097 
1098     // commit
1099     fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1100 
1101     // close the db
1102     fdb_kvs_close(db);
1103     fdb_close(dbfile);
1104 
1105     // reopen with an option disabling seq tree
1106     fconfig.seqtree_opt = FDB_SEQTREE_NOT_USE;
1107     status = fdb_open(&dbfile, "./dummy1", &fconfig);
1108     // must succeed
1109     TEST_CHK(status == FDB_RESULT_SUCCESS);
1110     status = fdb_close(dbfile);
1111     TEST_CHK(status == FDB_RESULT_SUCCESS);
1112 
1113     // free all documents
1114     for (i=0;i<n;++i){
1115         fdb_doc_free(doc[i]);
1116     }
1117 
1118     // free all resources
1119     fdb_shutdown();
1120 
1121     memleak_end();
1122 
1123     TEST_RESULT("sequence tree exception test");
1124 }
1125 
wal_commit_test()1126 void wal_commit_test()
1127 {
1128     TEST_INIT();
1129 
1130     memleak_start();
1131 
1132     int i, r;
1133     int n = 10;
1134     fdb_file_handle *dbfile;
1135     fdb_kvs_handle *db;
1136     fdb_doc **doc = alca(fdb_doc*, n);
1137     fdb_doc *rdoc = NULL;
1138     fdb_status status;
1139 
1140     char keybuf[256], metabuf[256], bodybuf[256];
1141 
1142     // remove previous dummy files
1143     r = system(SHELL_DEL" dummy* > errorlog.txt");
1144     (void)r;
1145 
1146     fdb_config fconfig = fdb_get_default_config();
1147     fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1148     fconfig.buffercache_size = 0;
1149     fconfig.wal_threshold = 1024;
1150     fconfig.flags = FDB_OPEN_FLAG_CREATE;
1151     fconfig.compaction_threshold = 0;
1152 
1153     // open db
1154     fdb_open(&dbfile, "./dummy1", &fconfig);
1155     fdb_kvs_open_default(dbfile, &db, &kvs_config);
1156     status = fdb_set_log_callback(db, logCallbackFunc, (void *) "wal_commit_test");
1157     TEST_CHK(status == FDB_RESULT_SUCCESS);
1158 
1159     // insert half documents
1160     for (i=0;i<n/2;++i){
1161         sprintf(keybuf, "key%d", i);
1162         sprintf(metabuf, "meta%d", i);
1163         sprintf(bodybuf, "body%d", i);
1164         fdb_doc_create(&doc[i], (void *)keybuf, strlen(keybuf),
1165             (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1166         fdb_set(db, doc[i]);
1167     }
1168 
1169     // commit
1170     fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1171 
1172     // insert the other half documents
1173     for (i=n/2;i<n;++i){
1174         sprintf(keybuf, "key%d", i);
1175         sprintf(metabuf, "meta%d", i);
1176         sprintf(bodybuf, "body%d", i);
1177         fdb_doc_create(&doc[i], (void *)keybuf, strlen(keybuf),
1178             (void *)metabuf, strlen(metabuf), (void *)bodybuf, strlen(bodybuf));
1179         fdb_set(db, doc[i]);
1180     }
1181 
1182     // close the db
1183     fdb_kvs_close(db);
1184     fdb_close(dbfile);
1185 
1186     // reopen
1187     fdb_open(&dbfile, "./dummy1", &fconfig);
1188     fdb_kvs_open_default(dbfile, &db, &kvs_config);
1189     status = fdb_set_log_callback(db, logCallbackFunc, (void *) "wal_commit_test");
1190     TEST_CHK(status == FDB_RESULT_SUCCESS);
1191 
1192     // retrieve documents
1193     for (i=0;i<n;++i){
1194         // search by key
1195         fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
1196         status = fdb_get(db, rdoc);
1197 
1198         if (i < n/2) {
1199             // committed documents
1200             TEST_CHK(status == FDB_RESULT_SUCCESS);
1201             TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
1202             TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
1203         } else {
1204             // not committed document
1205             TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
1206         }
1207 
1208         // free result document
1209         fdb_doc_free(rdoc);
1210         rdoc = NULL;
1211     }
1212 
1213     // free all documents
1214     for (i=0;i<n;++i){
1215         fdb_doc_free(doc[i]);
1216     }
1217 
1218     // close db file
1219     fdb_kvs_close(db);
1220     fdb_close(dbfile);
1221 
1222     // free all resources
1223     fdb_shutdown();
1224 
1225     memleak_end();
1226 
1227     TEST_RESULT("WAL commit test");
1228 }
1229 
db_close_and_remove()1230 void db_close_and_remove()
1231 {
1232 
1233     TEST_INIT();
1234     memleak_start();
1235 
1236     int i, r;
1237     int n = 10;
1238     fdb_file_handle *dbfile;
1239     fdb_kvs_handle *db;
1240     fdb_doc **doc = alca(fdb_doc *, n);
1241     fdb_status status;
1242     fdb_config fconfig;
1243     fdb_kvs_config kvs_config;
1244     char keybuf[256], metabuf[256], bodybuf[256];
1245 
1246     r = system(SHELL_DEL " dummy* > errorlog.txt");
1247     (void)r;
1248 
1249     // open dbfile
1250     fconfig = fdb_get_default_config();
1251     kvs_config = fdb_get_default_kvs_config();
1252     fconfig.cleanup_cache_onclose = false;
1253     fdb_open(&dbfile, "./dummy1", &fconfig);
1254     fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
1255 
1256     // write to db
1257     for (i=0;i<n;++i){
1258         sprintf(keybuf, "key%d", i);
1259         sprintf(metabuf, "meta%d", i);
1260         sprintf(bodybuf, "body%d", i);
1261         fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1262             (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1263         fdb_set(db, doc[i]);
1264         fdb_doc_free(doc[i]);
1265     }
1266     status = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1267     TEST_CHK(status == FDB_RESULT_SUCCESS);
1268 
1269     // close
1270     fdb_kvs_close(db);
1271     fdb_close(dbfile);
1272 
1273     // remove dbfile
1274     r = system(SHELL_DEL " dummy* > errorlog.txt");
1275     (void)r;
1276 
1277     // re-open read-only
1278     fconfig.flags = FDB_OPEN_FLAG_RDONLY;
1279     status = fdb_open(&dbfile, "./dummy1", &fconfig);
1280     TEST_CHK(status == FDB_RESULT_NO_SUCH_FILE);
1281 
1282     fdb_shutdown();
1283     memleak_end();
1284     TEST_RESULT("db close and remove");
1285 }
1286 
db_drop_test()1287 void db_drop_test()
1288 {
1289     TEST_INIT();
1290 
1291     memleak_start();
1292 
1293     int i, r;
1294     int n = 3;
1295     fdb_file_handle *dbfile;
1296     fdb_kvs_handle *db;
1297     fdb_doc **doc = alca(fdb_doc *, n);
1298     fdb_doc *rdoc = NULL;
1299     fdb_status status;
1300 
1301     char keybuf[256], metabuf[256], bodybuf[256];
1302 
1303     // remove previous dummy files
1304     r = system(SHELL_DEL " dummy* > errorlog.txt");
1305     (void)r;
1306 
1307     fdb_config fconfig = fdb_get_default_config();
1308     fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1309     fconfig.buffercache_size = 16777216;
1310     fconfig.wal_threshold = 1024;
1311     fconfig.flags = FDB_OPEN_FLAG_CREATE;
1312     fconfig.compaction_threshold = 0;
1313 
1314     // open db
1315     fdb_open(&dbfile, "./dummy1", &fconfig);
1316     fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
1317     status = fdb_set_log_callback(db, logCallbackFunc,
1318                                   (void *) "db_drop_test");
1319     TEST_CHK(status == FDB_RESULT_SUCCESS);
1320 
1321     // insert first two documents
1322     for (i=0;i<2;++i){
1323         sprintf(keybuf, "key%d", i);
1324         sprintf(metabuf, "meta%d", i);
1325         sprintf(bodybuf, "body%d", i);
1326         fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1327             (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1328         fdb_set(db, doc[i]);
1329     }
1330 
1331     // commit
1332     fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1333     fdb_kvs_close(db);
1334     fdb_close(dbfile);
1335 
1336     // Remove the database file manually.
1337     r = system(SHELL_DEL " dummy1 > errorlog.txt");
1338     (void)r;
1339 
1340     // Open the empty db with the same name.
1341     fdb_open(&dbfile, "./dummy1", &fconfig);
1342     fdb_kvs_open_default(dbfile, &db, &kvs_config);
1343     status = fdb_set_log_callback(db, logCallbackFunc,
1344                                   (void *) "db_drop_test");
1345     TEST_CHK(status == FDB_RESULT_SUCCESS);
1346 
1347     // now insert a new doc.
1348     sprintf(keybuf, "key%d", 0);
1349     sprintf(metabuf, "meta%d", 0);
1350     sprintf(bodybuf, "body%d", 0);
1351     fdb_doc_free(doc[0]);
1352     fdb_doc_create(&doc[0], (void*)keybuf, strlen(keybuf),
1353         (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1354     fdb_set(db, doc[0]);
1355 
1356     // commit
1357     fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1358 
1359     // search by key
1360     fdb_doc_create(&rdoc, doc[0]->key, doc[0]->keylen, NULL, 0, NULL, 0);
1361     status = fdb_get(db, rdoc);
1362     TEST_CHK(status == FDB_RESULT_SUCCESS);
1363     // Make sure that a doc seqnum starts with one.
1364     TEST_CHK(rdoc->seqnum == 1);
1365 
1366     fdb_kvs_close(db);
1367     fdb_close(dbfile);
1368 
1369     // free all documents
1370     fdb_doc_free(rdoc);
1371     rdoc = NULL;
1372     for (i=0;i<2;++i){
1373         fdb_doc_free(doc[i]);
1374     }
1375 
1376     // free all resources
1377     fdb_shutdown();
1378 
1379     memleak_end();
1380 
1381     TEST_RESULT("Database drop test");
1382 }
1383 
db_destroy_test()1384 void db_destroy_test()
1385 {
1386     TEST_INIT();
1387 
1388     memleak_start();
1389 
1390     int i, r;
1391     int n = 30;
1392     fdb_file_handle *dbfile, *dbfile2;
1393     fdb_kvs_handle *db, *db2;
1394     fdb_doc **doc = alca(fdb_doc *, n);
1395     fdb_doc *rdoc = NULL;
1396     fdb_status status;
1397     fdb_config fconfig;
1398     fdb_kvs_config kvs_config;
1399 
1400     char keybuf[256], metabuf[256], bodybuf[256];
1401 
1402     // remove previous dummy files
1403     r = system(SHELL_DEL " dummy* > errorlog.txt");
1404     (void)r;
1405 
1406     fconfig = fdb_get_default_config();
1407     kvs_config = fdb_get_default_kvs_config();
1408     fconfig.buffercache_size = 16777216;
1409     fconfig.wal_threshold = 1024;
1410     fconfig.flags = FDB_OPEN_FLAG_CREATE;
1411     fconfig.compaction_threshold = 0;
1412 
1413     // open db
1414     fdb_open(&dbfile, "./dummy1", &fconfig);
1415     fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
1416     status = fdb_set_log_callback(db, logCallbackFunc,
1417                                   (void *) "db_destroy_test");
1418     TEST_CHK(status == FDB_RESULT_SUCCESS);
1419 
1420     // insert 30 documents
1421     for (i=0;i<n;++i){
1422         sprintf(keybuf, "key%d", i);
1423         sprintf(metabuf, "meta%d", i);
1424         sprintf(bodybuf, "body%d", i);
1425         fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1426             (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1427         fdb_set(db, doc[i]);
1428     }
1429 
1430     // commit
1431     fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1432 
1433     // Open the empty db with the same name.
1434     fdb_open(&dbfile2, "./dummy2", &fconfig);
1435     fdb_kvs_open(dbfile2, &db2, NULL, &kvs_config);
1436     status = fdb_set_log_callback(db2, logCallbackFunc,
1437                                   (void *) "db_destroy_test");
1438     TEST_CHK(status == FDB_RESULT_SUCCESS);
1439     // insert 30 documents
1440     for (i=0;i<n;++i){
1441         fdb_set(db2, doc[i]);
1442     }
1443 
1444     // commit
1445     fdb_commit(dbfile2, FDB_COMMIT_NORMAL);
1446 
1447     // Only close db not db2 and try to destroy
1448     fdb_close(dbfile);
1449 
1450     status = fdb_destroy("./dummy2", &fconfig);
1451     TEST_CHK(status == FDB_RESULT_FILE_IS_BUSY);
1452 
1453     //Now close the open db file
1454     fdb_close(dbfile2);
1455 
1456     status = fdb_destroy("./dummy1", &fconfig);
1457     TEST_CHK(status == FDB_RESULT_SUCCESS);
1458 
1459     // Open the same db with the same names.
1460     fdb_open(&dbfile, "./dummy1", &fconfig);
1461     fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
1462     status = fdb_set_log_callback(db, logCallbackFunc,
1463                                   (void *) "db_destroy_test");
1464     TEST_CHK(status == FDB_RESULT_SUCCESS);
1465 
1466     // search by key
1467     fdb_doc_create(&rdoc, doc[0]->key, doc[0]->keylen, NULL, 0, NULL, 0);
1468     status = fdb_get(db, rdoc);
1469     TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
1470     fdb_close(dbfile);
1471 
1472     // free all documents
1473     fdb_doc_free(rdoc);
1474     rdoc = NULL;
1475     for (i=0;i<n;++i){
1476         fdb_doc_free(doc[i]);
1477     }
1478 
1479     // free all resources
1480     fdb_shutdown();
1481 
1482     memleak_end();
1483 
1484     TEST_RESULT("Database destroy test");
1485 }
1486 
1487 // Test for MB-16348
db_destroy_test_full_path()1488 void db_destroy_test_full_path()
1489 {
1490     TEST_INIT();
1491 
1492     memleak_start();
1493     randomize();
1494 
1495     int r;
1496     fdb_file_handle *dbfile;
1497     fdb_config config;
1498     fdb_status s;
1499     char path[256];
1500     char cmd[256];
1501 
1502     sprintf(path, "/tmp/fdb_destroy_test_%d", random(10000));
1503 
1504     sprintf(cmd, "rm -rf %s*", path);
1505     r = system(cmd); (void)r;
1506 
1507     config = fdb_get_default_config();
1508     config.compaction_mode = FDB_COMPACTION_AUTO;
1509 
1510     fdb_open(&dbfile, path, &config);
1511     fdb_close(dbfile);
1512 
1513     s = fdb_destroy(path, &config);
1514     TEST_CHK(s == FDB_RESULT_SUCCESS);
1515 
1516     fdb_shutdown();
1517 
1518     memleak_end();
1519 
1520     TEST_RESULT("Database destroy (full path) test");
1521 }
1522 
operational_stats_test(bool multi_kv)1523 void operational_stats_test(bool multi_kv)
1524 {
1525     TEST_INIT();
1526 
1527     memleak_start();
1528 
1529     int i, r;
1530     int n = 10;
1531     int num_kv = 4;
1532     fdb_file_handle *dbfile;
1533     fdb_kvs_handle **db = alca(fdb_kvs_handle*, num_kv);
1534     fdb_doc **doc = alca(fdb_doc*, n);
1535     fdb_doc real_doc;
1536     fdb_doc *rdoc = &real_doc;
1537     fdb_status status;
1538     fdb_iterator *iterator;
1539     fdb_kvs_ops_info info, rinfo;
1540 
1541     char keybuf[256], bodybuf[256];
1542     memset(&info, 0, sizeof(fdb_kvs_ops_info));
1543     memset(&real_doc, 0, sizeof(fdb_doc));
1544     real_doc.key = &keybuf;
1545     real_doc.body = &bodybuf;
1546 
1547     // remove previous dummy files
1548     r = system(SHELL_DEL" dummy* > errorlog.txt");
1549     (void)r;
1550 
1551     fdb_config fconfig = fdb_get_default_config();
1552     fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1553 
1554     fconfig.buffercache_size = 0;
1555     fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
1556     fconfig.wal_threshold = 1024;
1557     fconfig.flags = FDB_OPEN_FLAG_CREATE;
1558     fconfig.compaction_threshold = 0;
1559     fconfig.multi_kv_instances = multi_kv;
1560     r = 0;
1561 
1562     fdb_open(&dbfile, "./dummy1", &fconfig);
1563     if (multi_kv) {
1564         num_kv = 4;
1565         for (r = num_kv - 1; r >= 0; --r) {
1566             char tmp[16];
1567             sprintf(tmp, "kv%d", r);
1568             status = fdb_kvs_open(dbfile, &db[r], tmp, &kvs_config);
1569             TEST_CHK(status == FDB_RESULT_SUCCESS);
1570             status = fdb_set_log_callback(db[r], logCallbackFunc,
1571                                           (void *) "operational_stats_test");
1572             TEST_CHK(status == FDB_RESULT_SUCCESS);
1573         }
1574     } else {
1575         num_kv = 1;
1576         status = fdb_kvs_open_default(dbfile, &db[r], &kvs_config);
1577         TEST_CHK(status == FDB_RESULT_SUCCESS);
1578         status = fdb_set_log_callback(db[r], logCallbackFunc,
1579                 (void *) "operational_stats_test");
1580         TEST_CHK(status == FDB_RESULT_SUCCESS);
1581     }
1582 
1583     for (i = 0; i < n; ++i){
1584         sprintf(keybuf, "key%d", i);
1585         sprintf(bodybuf, "body%d", i);
1586         fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf) + 1, NULL, 0,
1587             (void*)bodybuf, strlen(bodybuf)+1);
1588         for (r = num_kv - 1; r >= 0; --r) {
1589             status = fdb_set(db[r], doc[i]);
1590             TEST_CHK(status == FDB_RESULT_SUCCESS);
1591             status = fdb_get_kvs_ops_info(db[r], &rinfo);
1592             TEST_CHK(status == FDB_RESULT_SUCCESS);
1593             info.num_sets = i + 1;
1594             TEST_CMP(&rinfo, &info, sizeof(fdb_kvs_ops_info));
1595         }
1596     }
1597 
1598     for (r = num_kv - 1; r >= 0; --r) {
1599         // range scan (before flushing WAL)
1600         fdb_iterator_init(db[r], &iterator, NULL, 0, NULL, 0, 0x0);
1601         i = 0;
1602         do {
1603             status = fdb_iterator_get(iterator, &rdoc);
1604             TEST_CHK(status == FDB_RESULT_SUCCESS);
1605             TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
1606             TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
1607             status = fdb_get_kvs_ops_info(db[r], &rinfo);
1608             TEST_CHK(status == FDB_RESULT_SUCCESS);
1609             ++info.num_iterator_gets;
1610             ++info.num_iterator_moves;
1611             TEST_CMP(&rinfo, &info, sizeof(fdb_kvs_ops_info));
1612             ++i;
1613         } while(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
1614         ++info.num_iterator_moves; // account for the last move that failed
1615         fdb_iterator_close(iterator);
1616 
1617         fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1618         ++info.num_commits;
1619 
1620         status = fdb_get_kvs_ops_info(db[r], &rinfo);
1621         TEST_CHK(status == FDB_RESULT_SUCCESS);
1622         TEST_CMP(&rinfo, &info, sizeof(fdb_kvs_ops_info));
1623 
1624         if (r) {
1625             info.num_iterator_gets = 0;
1626             info.num_iterator_moves = 0;
1627         }
1628     }
1629 
1630     ++info.num_compacts;
1631     // do compaction
1632     fdb_compact(dbfile, (char *) "./dummy2");
1633 
1634     status = fdb_get_kvs_ops_info(db[0], &rinfo);
1635     TEST_CHK(status == FDB_RESULT_SUCCESS);
1636     TEST_CMP(&rinfo, &info, sizeof(fdb_kvs_ops_info));
1637 
1638     for (i = 0; i < n; ++i){
1639         sprintf(keybuf, "key%d", i);
1640         for (r = num_kv - 1; r >= 0; --r) {
1641             if (i % 2 == 0) {
1642                 if (i % 4 == 0) {
1643                     status = fdb_get_metaonly(db[r], rdoc);
1644                 } else {
1645                     rdoc->seqnum = i + 1;
1646                     status = fdb_get_byseq(db[r], rdoc);
1647                 }
1648             } else {
1649                 status = fdb_get(db[r], rdoc);
1650             }
1651             TEST_CHK(status == FDB_RESULT_SUCCESS);
1652             status = fdb_get_kvs_ops_info(db[r], &rinfo);
1653             TEST_CHK(status == FDB_RESULT_SUCCESS);
1654             info.num_gets = i + 1;
1655             TEST_CMP(&rinfo, &info, sizeof(fdb_kvs_ops_info));
1656         }
1657     }
1658     // also get latency stats..
1659     for (int i = 0; i < FDB_LATENCY_NUM_STATS; ++i) {
1660         fdb_latency_stat stat;
1661         memset(&stat, 0, sizeof(fdb_latency_stat));
1662         status = fdb_get_latency_stats(dbfile, &stat, i);
1663         TEST_CHK(status == FDB_RESULT_SUCCESS);
1664         fprintf(stderr, "%d:\t%u\t%u\t%u\t%" _F64 "\n", i,
1665                 stat.lat_max, stat.lat_avg, stat.lat_max, stat.lat_count);
1666     }
1667 
1668     fdb_close(dbfile);
1669 
1670     // free all documents
1671     for (i=0;i<n;++i){
1672         fdb_doc_free(doc[i]);
1673     }
1674 
1675     // free all resources
1676     fdb_shutdown();
1677 
1678     memleak_end();
1679 
1680     sprintf(bodybuf,"Operational stats test %s", multi_kv ?
1681             "multiple kv instances" : "single kv instance");
1682     TEST_RESULT(bodybuf);
1683 }
1684 
1685 struct work_thread_args{
1686     int tid;
1687     size_t nthreads;
1688     size_t ndocs;
1689     size_t writer;
1690     fdb_doc **doc;
1691     size_t time_sec;
1692     size_t nbatch;
1693     size_t compact_term;
1694     int *n_opened;
1695     int *filename_count;
1696     spin_t *filename_count_lock;
1697     size_t nops;
1698     fdb_config *config;
1699     fdb_kvs_config *kvs_config;
1700 };
1701 
1702 //#define FILENAME "./hdd/dummy"
1703 #define FILENAME "dummy"
1704 
1705 #define KSIZE (100)
1706 #define VSIZE (100)
1707 #define IDX_DIGIT (7)
1708 #define IDX_DIGIT_STR "7"
1709 
_worker_thread(void * voidargs)1710 void *_worker_thread(void *voidargs)
1711 {
1712     TEST_INIT();
1713 
1714     struct work_thread_args *args = (struct work_thread_args *)voidargs;
1715     int i, c, commit_count, filename_count;
1716     struct timeval ts_begin, ts_cur, ts_gap;
1717     fdb_file_handle *dbfile;
1718     fdb_kvs_handle *db;
1719     fdb_status status;
1720     fdb_doc *rdoc = NULL;
1721     char temp[1024];
1722 
1723     char cnt_str[IDX_DIGIT+1];
1724     int cnt_int;
1725 
1726     filename_count = *args->filename_count;
1727     sprintf(temp, FILENAME"%d", filename_count);
1728     fdb_open(&dbfile, temp, args->config);
1729     fdb_kvs_open_default(dbfile, &db, args->kvs_config);
1730     status = fdb_set_log_callback(db, logCallbackFunc,
1731                                   (void *) "worker_thread");
1732     TEST_CHK(status == FDB_RESULT_SUCCESS);
1733 
1734     // wait until all other threads open the DB file.
1735     // (to avoid performing compaction before opening the file)
1736     spin_lock(args->filename_count_lock);
1737     *args->n_opened += 1;
1738     spin_unlock(args->filename_count_lock);
1739     do {
1740         spin_lock(args->filename_count_lock);
1741         if ((size_t)(*args->n_opened) == args->nthreads) {
1742             // all threads open the DB file
1743             spin_unlock(args->filename_count_lock);
1744             break;
1745         }
1746         spin_unlock(args->filename_count_lock);
1747         // sleep 1 sec
1748         sleep(1);
1749     } while (1);
1750 
1751     gettimeofday(&ts_begin, NULL);
1752 
1753     c = cnt_int = commit_count = 0;
1754     cnt_str[IDX_DIGIT] = 0;
1755 
1756     while (1){
1757         i = rand() % args->ndocs;
1758         fdb_doc_create(&rdoc, args->doc[i]->key, args->doc[i]->keylen, NULL, 0, NULL, 0);
1759         status = fdb_get(db, rdoc);
1760 
1761         TEST_CHK(status == FDB_RESULT_SUCCESS);
1762         TEST_CMP(rdoc->body, args->doc[i]->body, (IDX_DIGIT+1));
1763 
1764         if (args->writer) {
1765             // if writer,
1766             // copy and parse the counter in body
1767             memcpy(cnt_str, (uint8_t *)rdoc->body + (IDX_DIGIT+1), IDX_DIGIT);
1768             cnt_int = atoi(cnt_str);
1769 
1770             // increase and rephrase
1771             sprintf(cnt_str, "%0 " IDX_DIGIT_STR "d", ++cnt_int);
1772             memcpy((uint8_t *)rdoc->body + (IDX_DIGIT+1), cnt_str, IDX_DIGIT);
1773 
1774             // update and commit
1775             status = fdb_set(db, rdoc);
1776             TEST_CHK(status == FDB_RESULT_SUCCESS);
1777 
1778             if (args->nbatch > 0) {
1779                 if (c % args->nbatch == 0) {
1780                     // commit for every NBATCH
1781                     fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1782                     commit_count++;
1783                     fdb_file_info info;
1784                     fdb_get_file_info(dbfile, &info);
1785                     if (args->compact_term == (size_t)commit_count &&
1786                         args->compact_term > 0 &&
1787                         info.new_filename == NULL &&
1788                         args->tid == 0) {
1789                         // do compaction for every COMPACT_TERM batch
1790                         spin_lock(args->filename_count_lock);
1791                         *args->filename_count += 1;
1792                         filename_count = *args->filename_count;
1793                         spin_unlock(args->filename_count_lock);
1794 
1795                         sprintf(temp, FILENAME"%d", filename_count);
1796 
1797                         status = fdb_compact(dbfile, temp);
1798                         if (status != FDB_RESULT_SUCCESS) {
1799                             spin_lock(args->filename_count_lock);
1800                             *args->filename_count -= 1;
1801                             spin_unlock(args->filename_count_lock);
1802                         }
1803 
1804                         commit_count = 0;
1805                     }
1806                 }
1807             }
1808         }
1809         fdb_doc_free(rdoc);
1810         rdoc = NULL;
1811         c++;
1812 
1813         gettimeofday(&ts_cur, NULL);
1814         ts_gap = _utime_gap(ts_begin, ts_cur);
1815         if ((size_t)ts_gap.tv_sec >= args->time_sec) break;
1816     }
1817 
1818     DBG("Thread #%d (%s) %d ops / %d seconds\n",
1819         args->tid, (args->writer)?("writer"):("reader"), c, (int)args->time_sec);
1820     args->nops = c;
1821 
1822     fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1823 
1824     fdb_kvs_close(db);
1825     fdb_close(dbfile);
1826     thread_exit(0);
1827     return NULL;
1828 }
1829 
multi_thread_test(size_t ndocs,size_t wal_threshold,size_t time_sec,size_t nbatch,size_t compact_term,size_t nwriters,size_t nreaders)1830 void multi_thread_test(
1831     size_t ndocs, size_t wal_threshold, size_t time_sec,
1832     size_t nbatch, size_t compact_term, size_t nwriters, size_t nreaders)
1833 {
1834     TEST_INIT();
1835 
1836     size_t nwrites, nreads;
1837     int i, r;
1838     int n = nwriters + nreaders;;
1839     thread_t *tid = alca(thread_t, n);
1840     void **thread_ret = alca(void *, n);
1841     struct work_thread_args *args = alca(struct work_thread_args, n);
1842     struct timeval ts_begin, ts_cur, ts_gap;
1843     fdb_file_handle *dbfile;
1844     fdb_kvs_handle *db;
1845     fdb_doc **doc = alca(fdb_doc*, ndocs);
1846     fdb_status status;
1847     fdb_kvs_info kvs_info;
1848 
1849     int filename_count = 1;
1850     int n_opened = 0;
1851     spin_t filename_count_lock;
1852     spin_init(&filename_count_lock);
1853 
1854     char keybuf[1024], metabuf[1024], bodybuf[1024], temp[1024];
1855 
1856     // remove previous dummy files
1857     r = system(SHELL_DEL" " FILENAME "* > errorlog.txt");
1858     (void)r;
1859 
1860     fdb_config fconfig = fdb_get_default_config();
1861     fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1862     fconfig.buffercache_size = 16777216;
1863     fconfig.wal_threshold = 1024;
1864     fconfig.flags = FDB_OPEN_FLAG_CREATE;
1865     fconfig.compaction_threshold = 0;
1866 
1867     memleak_start();
1868 
1869     // initial population ===
1870     DBG("Initialize..\n");
1871 
1872     // open db
1873     sprintf(temp, FILENAME"%d", filename_count);
1874     fdb_open(&dbfile, temp, &fconfig);
1875     fdb_kvs_open_default(dbfile, &db, &kvs_config);
1876     status = fdb_set_log_callback(db, logCallbackFunc,
1877                                   (void *) "multi_thread_test");
1878     TEST_CHK(status == FDB_RESULT_SUCCESS);
1879 
1880     gettimeofday(&ts_begin, NULL);
1881 
1882     // insert documents
1883     for (i = 0; (size_t)i < ndocs; ++i){
1884         _set_random_string_smallabt(temp, KSIZE - (IDX_DIGIT+1));
1885         sprintf(keybuf, "k%0" IDX_DIGIT_STR "d%s", i, temp);
1886 
1887         sprintf(metabuf, "m%0" IDX_DIGIT_STR "d", i);
1888 
1889         _set_random_string_smallabt(temp, VSIZE-(IDX_DIGIT*2+1));
1890         sprintf(bodybuf, "b%0" IDX_DIGIT_STR "d%0" IDX_DIGIT_STR "d%s", i, 0, temp);
1891 
1892         fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1893             (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1894         fdb_set(db, doc[i]);
1895     }
1896 
1897     fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1898 
1899     gettimeofday(&ts_cur, NULL);
1900     ts_gap = _utime_gap(ts_begin, ts_cur);
1901     //DBG("%d.%09d seconds elapsed\n", (int)ts_gap.tv_sec, (int)ts_gap.tv_nsec);
1902 
1903     fdb_kvs_close(db);
1904     fdb_close(dbfile);
1905     // end of population ===
1906 
1907     // drop OS's page cache
1908     //r = system("free && sync && echo 3 > /proc/sys/vm/drop_caches && free");
1909 
1910     // create workers
1911     for (i=0;i<n;++i){
1912         args[i].tid = i;
1913         args[i].nthreads = n;
1914         args[i].writer = (((size_t)i<nwriters)?(1):(0));
1915         args[i].ndocs = ndocs;
1916         args[i].doc = doc;
1917         args[i].time_sec = time_sec;
1918         args[i].nbatch = nbatch;
1919         args[i].compact_term = compact_term;
1920         args[i].n_opened = &n_opened;
1921         args[i].filename_count = &filename_count;
1922         args[i].filename_count_lock = &filename_count_lock;
1923         args[i].config = &fconfig;
1924         args[i].kvs_config = &kvs_config;
1925         thread_create(&tid[i], _worker_thread, &args[i]);
1926     }
1927 
1928     printf("wait for %d seconds..\n", (int)time_sec);
1929 
1930     // wait for thread termination
1931     for (i=0;i<n;++i){
1932         thread_join(tid[i], &thread_ret[i]);
1933     }
1934 
1935     // free all documents
1936     for (i=0;(size_t)i<ndocs;++i){
1937         fdb_doc_free(doc[i]);
1938     }
1939 
1940     nwrites = nreads = 0;
1941     for (i=0;i<n;++i){
1942         if (args[i].writer) {
1943             nwrites += args[i].nops;
1944         } else {
1945             nreads += args[i].nops;
1946         }
1947     }
1948     printf("read: %.1f ops/sec\n", (double)nreads/time_sec);
1949     printf("write: %.1f ops/sec\n", (double)nwrites/time_sec);
1950 
1951     // check sequence number
1952     sprintf(temp, FILENAME"%d", filename_count);
1953     fdb_open(&dbfile, temp, &fconfig);
1954     fdb_kvs_open_default(dbfile, &db, &kvs_config);
1955     fdb_get_kvs_info(db, &kvs_info);
1956     TEST_CHK(kvs_info.last_seqnum == ndocs+nwrites);
1957     fdb_kvs_close(db);
1958     fdb_close(dbfile);
1959 
1960     // shutdown
1961     fdb_shutdown();
1962 
1963     memleak_end();
1964 
1965     TEST_RESULT("multi thread test");
1966 }
1967 
multi_thread_client_shutdown(void * args)1968 void *multi_thread_client_shutdown(void *args)
1969 {
1970 
1971     TEST_INIT();
1972 
1973     int i, r;
1974     int nclients;
1975     fdb_file_handle *tdbfile;
1976     fdb_status status;
1977     fdb_config fconfig;
1978     fdb_kvs_config kvs_config;
1979     thread_t *tid;
1980     void **thread_ret;
1981 
1982     if (args == NULL)
1983     { // parent
1984         memleak_start();
1985 
1986         r = system(SHELL_DEL" dummy* > errorlog.txt");
1987         (void)r;
1988         nclients = 2;
1989         tid = alca(thread_t, nclients);
1990         thread_ret = alca(void *, nclients);
1991         for (i=0;i<nclients;++i){
1992             thread_create(&tid[i], multi_thread_client_shutdown, (void *)&i);
1993         }
1994         for (i=0;i<nclients;++i){
1995             thread_join(tid[i], &thread_ret[i]);
1996         }
1997 
1998         memleak_end();
1999         TEST_RESULT("multi thread client shutdown");
2000         return NULL;
2001     }
2002 
2003     // threads enter here //
2004 
2005     fconfig = fdb_get_default_config();
2006     kvs_config = fdb_get_default_kvs_config();
2007     fconfig.wal_threshold = 1024;
2008     fconfig.compaction_threshold = 0;
2009 
2010     // open/close db
2011     status = fdb_open(&tdbfile, "./dummy1", &fconfig);
2012     TEST_CHK(status == FDB_RESULT_SUCCESS);
2013     TEST_CHK(fdb_close(tdbfile) == FDB_RESULT_SUCCESS);
2014 
2015     // shutdown
2016     fdb_shutdown();
2017     thread_exit(0);
2018     return NULL;
2019 }
2020 
multi_thread_kvs_client(void * args)2021 void *multi_thread_kvs_client(void *args)
2022 {
2023 
2024     TEST_INIT();
2025 
2026     int i, j, r;
2027     int n = 50;
2028     int nclients = 20;
2029     int *tid_args = alca(int, nclients);
2030     char dbstr[256];
2031     char keybuf[256], metabuf[256], bodybuf[256];
2032     fdb_file_handle *dbfile;
2033     fdb_kvs_handle *tdb;
2034     fdb_kvs_handle **db = alca(fdb_kvs_handle*, nclients);
2035     fdb_doc **doc = alca(fdb_doc*, n);
2036     fdb_doc *rdoc = NULL;
2037     fdb_status status;
2038     fdb_config fconfig;
2039     fdb_kvs_config kvs_config;
2040     fdb_seqnum_t seqnum;
2041     thread_t *tid;
2042     void **thread_ret;
2043 
2044     if (args == NULL)
2045     { // parent
2046         memleak_start();
2047 
2048         r = system(SHELL_DEL" dummy* > errorlog.txt");
2049         (void)r;
2050 
2051         // init dbfile
2052         fconfig = fdb_get_default_config();
2053         fconfig.buffercache_size = 0;
2054         fconfig.wal_threshold = 1024;
2055         fconfig.compaction_threshold = 0;
2056 
2057         status = fdb_open(&dbfile, "./dummy1", &fconfig);
2058         TEST_CHK(status == FDB_RESULT_SUCCESS);
2059 
2060         tid = alca(thread_t, nclients);
2061         thread_ret = alca(void *, nclients);
2062         for (i=0;i<nclients;++i){
2063             sprintf(dbstr, "db%d", i);
2064             kvs_config = fdb_get_default_kvs_config();
2065             status = fdb_kvs_open(dbfile, &db[i], dbstr, &kvs_config);
2066             TEST_CHK(status == FDB_RESULT_SUCCESS);
2067             status = fdb_kvs_close(db[i]);
2068             TEST_CHK(status == FDB_RESULT_SUCCESS);
2069         }
2070         for (i=0;i<nclients;++i){
2071             tid_args[i] = i;
2072             thread_create(&tid[i], multi_thread_kvs_client,
2073                           (void *)&tid_args[i]);
2074         }
2075         for (i=0;i<nclients;++i){
2076             thread_join(tid[i], &thread_ret[i]);
2077         }
2078 
2079         status = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2080         TEST_CHK(status == FDB_RESULT_SUCCESS);
2081 
2082         // check threads updated kvs
2083         for (i=0; i<nclients; i++){
2084             sprintf(dbstr, "db%d", i);
2085             kvs_config = fdb_get_default_kvs_config();
2086             status = fdb_kvs_open(dbfile, &db[i], dbstr, &kvs_config);
2087             TEST_CHK(status == FDB_RESULT_SUCCESS);
2088 
2089             // verify seqnum
2090             status = fdb_get_kvs_seqnum(db[i], &seqnum);
2091             TEST_CHK(status == FDB_RESULT_SUCCESS);
2092             TEST_CHK(seqnum == (fdb_seqnum_t)n);
2093 
2094             for (j=0; j<n; j++){
2095                 sprintf(keybuf, "key%d", j);
2096                 sprintf(metabuf, "meta%d", j);
2097                 sprintf(bodybuf, "body%d", j);
2098                 fdb_doc_create(&rdoc, keybuf, strlen(keybuf),
2099                                       NULL, 0, NULL, 0);
2100                 status = fdb_get(db[i], rdoc);
2101                 TEST_CHK(status == FDB_RESULT_SUCCESS);
2102                 TEST_CHK(!memcmp(rdoc->key, keybuf, strlen(keybuf)));
2103                 TEST_CHK(!memcmp(rdoc->meta, metabuf, rdoc->metalen));
2104                 TEST_CHK(!memcmp(rdoc->body, bodybuf, rdoc->bodylen));
2105                 fdb_doc_free(rdoc);
2106             }
2107             status = fdb_kvs_close(db[i]);
2108             TEST_CHK(status == FDB_RESULT_SUCCESS);
2109         }
2110 
2111         status = fdb_close(dbfile);
2112         TEST_CHK(status == FDB_RESULT_SUCCESS);
2113         fdb_shutdown();
2114         memleak_end();
2115         TEST_RESULT("multi thread kvs client");
2116         return NULL;
2117     }
2118 
2119     // threads enter here //
2120 
2121     // open fhandle
2122     fconfig = fdb_get_default_config();
2123     status = fdb_open(&dbfile, "./dummy1", &fconfig);
2124     TEST_CHK(status == FDB_RESULT_SUCCESS);
2125 
2126     // get kvs ID from args
2127     memcpy(&i, args, sizeof(int));
2128     sprintf(dbstr, "db%d", i);
2129     kvs_config = fdb_get_default_kvs_config();
2130     status = fdb_kvs_open(dbfile, &tdb, dbstr, &kvs_config);
2131     TEST_CHK(status == FDB_RESULT_SUCCESS);
2132 
2133     // insert documents
2134     for (i=0;i<n;++i){
2135         sprintf(keybuf, "key%d", i);
2136         sprintf(metabuf, "meta%d", i);
2137         sprintf(bodybuf, "body%d", i);
2138         fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2139                                 (void*)metabuf, strlen(metabuf),
2140                                 (void*)bodybuf, strlen(bodybuf));
2141         status = fdb_set(tdb, doc[i]);
2142         TEST_CHK(status == FDB_RESULT_SUCCESS);
2143         fdb_doc_free(doc[i]);
2144     }
2145     status = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2146     TEST_CHK(status == FDB_RESULT_SUCCESS);
2147 
2148     fdb_close(dbfile);
2149     return NULL;
2150 }
2151 
multi_thread_fhandle_share(void * args)2152 void *multi_thread_fhandle_share(void *args)
2153 {
2154     TEST_INIT();
2155     fdb_status status;
2156     int n = 2000;
2157     int i, r;
2158     char tmpbuf[32];
2159     typedef struct {
2160         fdb_file_handle *dbfile;
2161         fdb_kvs_handle *def;
2162         fdb_kvs_handle *main;
2163         fdb_kvs_handle *back;
2164         bool isWriter;
2165         std::atomic<bool> shutdown;
2166     } thread_data_t;
2167 
2168     if (args == NULL) { // MAIN THREAD..
2169         int nthreads = 2; // Half of these are reader and half are writers
2170         int nwriters = nthreads / 2;
2171         thread_t *tid = (thread_t *)malloc(nthreads * sizeof(thread_t *));
2172         thread_data_t *tdata = (thread_data_t *) malloc(nthreads
2173                                                * sizeof(thread_data_t));
2174         void **thread_ret = (void **)malloc(nthreads * sizeof (void *));
2175         fdb_kvs_config kvs_config;
2176         fdb_config fconfig;
2177 
2178         r = system(SHELL_DEL" func_test* > errorlog.txt");
2179         (void)r;
2180 
2181         // Shared File Handle data...
2182         fconfig = fdb_get_default_config();
2183         fconfig.buffercache_size = 0;
2184         fconfig.compaction_threshold = 0;
2185         fconfig.num_compactor_threads = 1;
2186         kvs_config = fdb_get_default_kvs_config();
2187         for (i=0; i < nwriters; ++i) {
2188             // Let Readers share same file handle as writers..
2189             fdb_file_handle *dbfile;
2190             sprintf(tmpbuf, "./func_test_pt.%d", i);
2191             status = fdb_open(&dbfile, tmpbuf, &fconfig);
2192             TEST_CHK(status == FDB_RESULT_SUCCESS);
2193             tdata[i].dbfile = dbfile;
2194             int ridx = i+nwriters; // reader index
2195             tdata[ridx].dbfile = dbfile;
2196             tdata[i].isWriter = true;
2197             // Open separate KVS Handles for Readers..
2198             tdata[ridx].isWriter = false; // Set for readers
2199             status = fdb_kvs_open_default(dbfile, &tdata[ridx].def, &kvs_config);
2200             TEST_CHK(status == FDB_RESULT_SUCCESS);
2201             status = fdb_kvs_open(dbfile, &tdata[ridx].main, "main", &kvs_config);
2202             TEST_CHK(status == FDB_RESULT_SUCCESS);
2203             status = fdb_kvs_open(dbfile, &tdata[ridx].back, "back", &kvs_config);
2204             TEST_CHK(status == FDB_RESULT_SUCCESS);
2205             // Open Separate KVS Handle for Writers..
2206             status = fdb_kvs_open_default(dbfile, &tdata[i].def, &kvs_config);
2207             TEST_CHK(status == FDB_RESULT_SUCCESS);
2208             status = fdb_kvs_open(dbfile, &tdata[i].main, "main", &kvs_config);
2209             TEST_CHK(status == FDB_RESULT_SUCCESS);
2210             status = fdb_kvs_open(dbfile, &tdata[i].back, "back", &kvs_config);
2211             TEST_CHK(status == FDB_RESULT_SUCCESS);
2212         }
2213         printf("Creating %d writers+readers over %d docs..\n", nwriters, n);
2214         for (i=nthreads - 1;i>=0;--i){
2215             tdata[i].shutdown = false;
2216             thread_create(&tid[i], multi_thread_fhandle_share,
2217                           reinterpret_cast<void *>(&tdata[i]));
2218         }
2219         for (i=0; i < nwriters; ++i) { // first wait for writers..
2220             thread_join(tid[i], &thread_ret[i]);
2221             printf("Writer %d done\n", i);
2222             tdata[i+nwriters].shutdown = true; // tell reader to shutdown
2223         }
2224         for (;i<nthreads;++i){ // now wait for readers..
2225             thread_join(tid[i], &thread_ret[i]);
2226         }
2227 
2228         for (i=0; i<nwriters;++i) {
2229             status = fdb_close(tdata[i].dbfile);
2230             TEST_CHK(status == FDB_RESULT_SUCCESS);
2231         }
2232 
2233         free(tid);
2234         free(tdata);
2235         free(thread_ret);
2236         fdb_shutdown();
2237         TEST_RESULT("multi thread file handle share test");
2238         return NULL;
2239     }
2240     // threads enter here ----
2241     thread_data_t *tdata = reinterpret_cast<thread_data_t *>(args);
2242     if (tdata->isWriter) { // Writer Threads Run this...
2243         for (i=0; i < n; ++i) {
2244             sprintf(tmpbuf, "key%03d", i);
2245             status = fdb_set_kv(tdata->main, &tmpbuf, 7, nullptr, 0);
2246             TEST_CHK(status == FDB_RESULT_SUCCESS);
2247             status = fdb_set_kv(tdata->back, &tmpbuf, 7, nullptr, 0);
2248             TEST_CHK(status == FDB_RESULT_SUCCESS);
2249             status = fdb_set_kv(tdata->def, &tmpbuf, 7, nullptr, 0);
2250             TEST_CHK(status == FDB_RESULT_SUCCESS);
2251             if (n % 100 == 0) {
2252                 status = fdb_commit(tdata->dbfile,
2253                                     FDB_COMMIT_MANUAL_WAL_FLUSH);
2254                 TEST_CHK(status != FDB_RESULT_HANDLE_BUSY);
2255             }
2256         }
2257         return NULL;
2258     } // else  Reader Threads Run this ...
2259     while (!tdata->shutdown) {
2260         for (i=0; i < n; ++i) {
2261             void *value = nullptr;
2262             size_t valuelen;
2263             sprintf(tmpbuf, "key%03d", i);
2264             status = fdb_get_kv(tdata->main, &tmpbuf, 7, &value, &valuelen);
2265             TEST_CHK(status != FDB_RESULT_HANDLE_BUSY);
2266             status = fdb_get_kv(tdata->back, &tmpbuf, 7, &value, &valuelen);
2267             TEST_CHK(status != FDB_RESULT_HANDLE_BUSY);
2268             status = fdb_get_kv(tdata->def, &tmpbuf, 7, &value, &valuelen);
2269             TEST_CHK(status != FDB_RESULT_HANDLE_BUSY);
2270         }
2271     }
2272 
2273     return NULL;
2274 }
2275 
incomplete_block_test()2276 void incomplete_block_test()
2277 {
2278     TEST_INIT();
2279 
2280     memleak_start();
2281 
2282     int i, r;
2283     int n = 2;
2284     fdb_file_handle *dbfile;
2285     fdb_kvs_handle *db;
2286     fdb_doc **doc = alca(fdb_doc*, n);
2287     fdb_doc *rdoc = NULL;
2288     fdb_status status;
2289 
2290     char keybuf[256], metabuf[256], bodybuf[256];
2291 
2292     // remove previous dummy files
2293     r = system(SHELL_DEL" dummy* > errorlog.txt");
2294     (void)r;
2295 
2296     fdb_config fconfig = fdb_get_default_config();
2297     fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2298     fconfig.buffercache_size = 0;
2299     fconfig.wal_threshold = 1024;
2300     fconfig.flags = FDB_OPEN_FLAG_CREATE;
2301     fconfig.compaction_threshold = 0;
2302 
2303     // open db
2304     fdb_open(&dbfile, "./dummy1", &fconfig);
2305     fdb_kvs_open_default(dbfile, &db, &kvs_config);
2306     status = fdb_set_log_callback(db, logCallbackFunc,
2307                                   (void *) "incomplete_block_test");
2308     TEST_CHK(status == FDB_RESULT_SUCCESS);
2309 
2310     // insert documents
2311     for (i=0;i<n;++i){
2312         sprintf(keybuf, "key%d", i);
2313         sprintf(metabuf, "meta%d", i);
2314         sprintf(bodybuf, "body%d", i);
2315         fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2316             (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2317         fdb_set(db, doc[i]);
2318     }
2319 
2320     // retrieve documents
2321     for (i=0;i<n;++i){
2322         // search by key
2323         fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
2324         status = fdb_get(db, rdoc);
2325 
2326         // updated documents
2327         TEST_CHK(status == FDB_RESULT_SUCCESS);
2328         TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2329         TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2330 
2331         // free result document
2332         fdb_doc_free(rdoc);
2333         rdoc = NULL;
2334     }
2335 
2336     // close db file
2337     fdb_kvs_close(db);
2338     fdb_close(dbfile);
2339 
2340     // free all documents
2341     for (i=0;i<n;++i){
2342         fdb_doc_free(doc[i]);
2343     }
2344 
2345     // free all resources
2346     fdb_shutdown();
2347 
2348     memleak_end();
2349 
2350     TEST_RESULT("incomplete block test");
2351 }
2352 
2353 
_cmp_double(void * key1,size_t keylen1,void * key2,size_t keylen2)2354 static int _cmp_double(void *key1, size_t keylen1, void *key2, size_t keylen2)
2355 {
2356     double aa, bb;
2357 
2358     if (!keylen1) {
2359         // key1 not set
2360         return -1;
2361     }
2362     if (!keylen2) {
2363         // key2 not set
2364         return 1;
2365     }
2366 
2367     aa = *(double *)key1;
2368     bb = *(double *)key2;
2369 
2370     if (aa<bb) {
2371         return -1;
2372     } else if (aa>bb) {
2373         return 1;
2374     } else {
2375         return 0;
2376     }
2377 }
2378 
custom_compare_primitive_test()2379 void custom_compare_primitive_test()
2380 {
2381     TEST_INIT();
2382 
2383     memleak_start();
2384 
2385     int i, r;
2386     int n = 10;
2387     fdb_file_handle *dbfile;
2388     fdb_kvs_handle *db;
2389     fdb_doc **doc = alca(fdb_doc*, n);
2390     fdb_doc *rdoc = NULL;
2391     fdb_status status;
2392     fdb_iterator *iterator;
2393 
2394     char keybuf[256], bodybuf[256];
2395     double key_double, key_double_prev;
2396 
2397     // remove previous dummy files
2398     r = system(SHELL_DEL" dummy* > errorlog.txt");
2399     (void)r;
2400 
2401     fdb_config fconfig = fdb_get_default_config();
2402     fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2403     fconfig.buffercache_size = 0;
2404     fconfig.wal_threshold = 1024;
2405     fconfig.flags = FDB_OPEN_FLAG_CREATE;
2406     fconfig.compaction_threshold = 0;
2407     fconfig.multi_kv_instances = true;
2408 
2409     kvs_config.custom_cmp = _cmp_double;
2410 
2411     // open db with custom compare function for double key type
2412     fdb_open(&dbfile, "./dummy1", &fconfig);
2413     fdb_kvs_open_default(dbfile, &db, &kvs_config);
2414     status = fdb_set_log_callback(db, logCallbackFunc,
2415                                   (void *) "custom_compare_primitive_test");
2416     TEST_CHK(status == FDB_RESULT_SUCCESS);
2417 
2418     for (i=0;i<n;++i){
2419         key_double = 10000/(i*11.0);
2420         memcpy(keybuf, &key_double, sizeof(key_double));
2421         sprintf(bodybuf, "value: %d, %f", i, key_double);
2422         fdb_doc_create(&doc[i], (void*)keybuf, sizeof(key_double), NULL, 0,
2423             (void*)bodybuf, strlen(bodybuf)+1);
2424         fdb_set(db, doc[i]);
2425     }
2426 
2427     // range scan (before flushing WAL)
2428     fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2429     key_double_prev = -1;
2430     do {
2431         status = fdb_iterator_get(iterator, &rdoc);
2432         TEST_CHK(status == FDB_RESULT_SUCCESS);
2433         memcpy(&key_double, rdoc->key, rdoc->keylen);
2434         TEST_CHK(key_double > key_double_prev);
2435         key_double_prev = key_double;
2436         fdb_doc_free(rdoc);
2437         rdoc = NULL;
2438     } while(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2439     fdb_iterator_close(iterator);
2440 
2441     fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2442 
2443     // range scan (after flushing WAL)
2444     fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2445     key_double_prev = -1;
2446     do {
2447         status = fdb_iterator_get(iterator, &rdoc);
2448         TEST_CHK(status == FDB_RESULT_SUCCESS);
2449         memcpy(&key_double, rdoc->key, rdoc->keylen);
2450         TEST_CHK(key_double > key_double_prev);
2451         key_double_prev = key_double;
2452         fdb_doc_free(rdoc);
2453         rdoc = NULL;
2454     } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2455     fdb_iterator_close(iterator);
2456 
2457     // do compaction
2458     fdb_compact(dbfile, (char *) "./dummy2");
2459 
2460     // range scan (after compaction)
2461     fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2462     key_double_prev = -1;
2463     do {
2464         status = fdb_iterator_get(iterator, &rdoc);
2465         TEST_CHK(status == FDB_RESULT_SUCCESS);
2466         memcpy(&key_double, rdoc->key, rdoc->keylen);
2467         TEST_CHK(key_double > key_double_prev);
2468         key_double_prev = key_double;
2469         fdb_doc_free(rdoc);
2470         rdoc = NULL;
2471     } while(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2472     fdb_iterator_close(iterator);
2473 
2474     // close db file
2475     fdb_kvs_close(db);
2476     fdb_close(dbfile);
2477 
2478     // free all documents
2479     for (i=0;i<n;++i){
2480         fdb_doc_free(doc[i]);
2481     }
2482 
2483     // free all resources
2484     fdb_shutdown();
2485 
2486     memleak_end();
2487 
2488     TEST_RESULT("custom compare function for primitive key test");
2489 }
2490 
_cmp_variable(void * key1,size_t keylen1,void * key2,size_t keylen2)2491 static int _cmp_variable(void *key1, size_t keylen1, void *key2, size_t keylen2)
2492 {
2493     if (keylen1 < 6 || keylen2 < 6) {
2494         return (keylen1 - keylen2);
2495     }
2496     // compare only 3rd~8th bytes (ignore the others)
2497     return memcmp((uint8_t*)key1+2, (uint8_t*)key2+2, 6);
2498 }
2499 
custom_compare_variable_test()2500 void custom_compare_variable_test()
2501 {
2502     TEST_INIT();
2503 
2504     memleak_start();
2505 
2506     int i, j, r;
2507     int n = 1000;
2508     int count;
2509     fdb_file_handle *dbfile;
2510     fdb_kvs_handle *db, *db2;
2511     fdb_doc **doc = alca(fdb_doc*, n);
2512     fdb_doc *rdoc = NULL;
2513     fdb_status status;
2514     fdb_iterator *iterator;
2515 
2516     size_t keylen = 16;
2517     size_t prev_keylen;
2518     char keybuf[256], bodybuf[256];
2519     char prev_key[256];
2520 
2521     // remove previous dummy files
2522     r = system(SHELL_DEL" dummy* > errorlog.txt");
2523     (void)r;
2524 
2525     fdb_config fconfig = fdb_get_default_config();
2526     fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2527     fconfig.seqtree_opt = FDB_SEQTREE_USE;
2528     fconfig.buffercache_size = 0;
2529     fconfig.wal_threshold = 1024;
2530     fconfig.flags = FDB_OPEN_FLAG_CREATE;
2531     fconfig.compaction_threshold = 0;
2532     fconfig.multi_kv_instances = true;
2533 
2534     kvs_config.custom_cmp = _cmp_variable;
2535 
2536     // open db with custom compare function for variable length key type
2537     //fdb_open_cmp_variable(&dbfile, "./dummy1", &fconfig);
2538     fdb_open(&dbfile, "./dummy1", &fconfig);
2539     fdb_kvs_open_default(dbfile, &db, &kvs_config);
2540     status = fdb_set_log_callback(db, logCallbackFunc,
2541                                   (void *) "custom_compare_variable_test");
2542     TEST_CHK(status == FDB_RESULT_SUCCESS);
2543 
2544     for (i=0;i<n;++i){
2545         for (j=0;j<2;++j){
2546             keybuf[j] = 'a' + rand()%('z'-'a');
2547         }
2548         sprintf(keybuf+2, "%06d", i);
2549         for (j=8;(size_t)j<keylen-1;++j){
2550             keybuf[j] = 'a' + rand()%('z'-'a');
2551         }
2552         keybuf[keylen-1] = 0;
2553         sprintf(bodybuf, "value: %d", i);
2554         fdb_doc_create(&doc[i], (void*)keybuf, keylen, NULL, 0,
2555             (void*)bodybuf, strlen(bodybuf)+1);
2556         fdb_set(db, doc[i]);
2557     }
2558 
2559     // point query
2560     for (i=0;i<n;++i){
2561         fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
2562         status = fdb_get(db, rdoc);
2563 
2564         TEST_CHK(status == FDB_RESULT_SUCCESS);
2565         TEST_CHK(rdoc->bodylen == doc[i]->bodylen);
2566         TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2567 
2568         fdb_doc_free(rdoc);
2569         rdoc = NULL;
2570     }
2571 
2572     // range scan (before flushing WAL)
2573     fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2574     sprintf(prev_key, "%016d", 0);
2575     count = 0;
2576     prev_keylen = 16;
2577     do {
2578         status = fdb_iterator_get(iterator, &rdoc);
2579         TEST_CHK(status == FDB_RESULT_SUCCESS);
2580         TEST_CHK(_cmp_variable(prev_key, prev_keylen, rdoc->key, rdoc->keylen) <= 0);
2581         prev_keylen = rdoc->keylen;
2582         memcpy(prev_key, rdoc->key, rdoc->keylen);
2583         fdb_doc_free(rdoc);
2584         rdoc = NULL;
2585         count++;
2586     } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2587     TEST_CHK(count == n);
2588     fdb_iterator_close(iterator);
2589 
2590     fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2591 
2592     // range scan (after flushing WAL)
2593     fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2594     sprintf(prev_key, "%016d", 0);
2595     count = 0;
2596     prev_keylen = 16;
2597     do {
2598         status = fdb_iterator_get(iterator, &rdoc);
2599         TEST_CHK(status == FDB_RESULT_SUCCESS);
2600         TEST_CHK(_cmp_variable(prev_key, prev_keylen, rdoc->key, rdoc->keylen)
2601                  <= 0);
2602         prev_keylen = rdoc->keylen;
2603         memcpy(prev_key, rdoc->key, rdoc->keylen);
2604         fdb_doc_free(rdoc);
2605         rdoc = NULL;
2606         count++;
2607     } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2608     TEST_CHK(count == n);
2609     fdb_iterator_close(iterator);
2610 
2611     // do compaction
2612     fdb_compact(dbfile, (char *) "./dummy2");
2613 
2614     // range scan (after compaction)
2615     fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2616     sprintf(prev_key, "%016d", 0);
2617     count = 0;
2618     prev_keylen = 16;
2619     do {
2620         status = fdb_iterator_get(iterator, &rdoc);
2621         TEST_CHK(status == FDB_RESULT_SUCCESS);
2622         TEST_CHK(_cmp_variable(prev_key, prev_keylen, rdoc->key, rdoc->keylen) <= 0);
2623         prev_keylen = rdoc->keylen;
2624         memcpy(prev_key, rdoc->key, rdoc->keylen);
2625         fdb_doc_free(rdoc);
2626         rdoc = NULL;
2627         count++;
2628     } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2629     TEST_CHK(count == n);
2630     fdb_iterator_close(iterator);
2631 
2632     // range scan by sequence
2633     fdb_iterator_sequence_init(db, &iterator, 0, 0, 0x0);
2634     count = 0;
2635     do { // forward
2636         status = fdb_iterator_get(iterator, &rdoc);
2637         TEST_CHK(status == FDB_RESULT_SUCCESS);
2638         fdb_doc_free(rdoc);
2639         rdoc = NULL;
2640         count++;
2641     } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2642     TEST_CHK(count == n);
2643 
2644     // Reverse direction
2645     for (; fdb_iterator_prev(iterator) != FDB_RESULT_ITERATOR_FAIL; --count) {
2646         status = fdb_iterator_get(iterator, &rdoc);
2647         TEST_CHK(status == FDB_RESULT_SUCCESS);
2648         fdb_doc_free(rdoc);
2649         rdoc = NULL;
2650     };
2651     TEST_CHK(count == 0);
2652     fdb_iterator_close(iterator);
2653 
2654     // open another handle
2655     kvs_config.custom_cmp = NULL;
2656     fdb_kvs_open_default(dbfile, &db2, &kvs_config);
2657 
2658     // point query
2659     for (i=0;i<n;++i){
2660         fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
2661         status = fdb_get(db2, rdoc);
2662 
2663         TEST_CHK(status == FDB_RESULT_SUCCESS);
2664         TEST_CHK(rdoc->bodylen == doc[i]->bodylen);
2665         TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2666 
2667         fdb_doc_free(rdoc);
2668         rdoc = NULL;
2669     }
2670 
2671     // close db file
2672     fdb_kvs_close(db);
2673     fdb_kvs_close(db2);
2674     fdb_close(dbfile);
2675 
2676     // free all documents
2677     for (i=0;i<n;++i){
2678         fdb_doc_free(doc[i]);
2679     }
2680 
2681     // free all resources
2682     fdb_shutdown();
2683 
2684     memleak_end();
2685 
2686     TEST_RESULT("custom compare function for variable length key test");
2687 }
2688 
2689 /*
2690  * custom compare test with commit and compact
2691  *    eqkeys:  boolean to toggle whether bytes in
2692  *             comparision range are equal
2693  */
custom_compare_commit_compact(bool eqkeys)2694 void custom_compare_commit_compact(bool eqkeys)
2695 {
2696     TEST_INIT();
2697 
2698     memleak_start();
2699 
2700     int i, j, r;
2701     int count;
2702     int n = 10;
2703     static const int len = 1024;
2704     char keybuf[len];
2705     static const char *achar = "a";
2706     fdb_doc *rdoc = NULL;
2707     fdb_status status;
2708     fdb_file_handle *dbfile;
2709     fdb_kvs_handle *db;
2710     fdb_iterator *iterator;
2711     fdb_config fconfig = fdb_get_default_config();
2712     fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2713 
2714     // remove previous dummy files
2715     r = system(SHELL_DEL" dummy* > errorlog.txt");
2716     (void)r;
2717 
2718     fconfig.buffercache_size = 0;
2719     fconfig.wal_threshold = 1024;
2720     fconfig.flags = FDB_OPEN_FLAG_CREATE;
2721     fconfig.compaction_threshold = 0;
2722     fconfig.multi_kv_instances = true;
2723 
2724     kvs_config.custom_cmp = _cmp_variable;
2725 
2726 
2727 
2728     r = system(SHELL_DEL" dummy* > errorlog.txt");
2729     (void)r;
2730 
2731 
2732     // open db
2733     status = fdb_open(&dbfile, "./dummy1", &fconfig);
2734     TEST_CHK(status == FDB_RESULT_SUCCESS);
2735     status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
2736     TEST_CHK(status == FDB_RESULT_SUCCESS);
2737 
2738 
2739     for (i=0;i<n;++i){
2740         if(eqkeys){
2741             sprintf(keybuf, "%d", i);
2742             for (j=1;j<len;++j) {
2743                 keybuf[j] = *achar;
2744             }
2745         } else {
2746             sprintf(keybuf, "000%d", i);
2747             for (j=4;j<len;++j) {
2748                 keybuf[j] = *achar;
2749             }
2750         }
2751         keybuf[len-1] = '\0';
2752         // set kv
2753         status = fdb_set_kv(db, keybuf, strlen(keybuf), NULL, 0);
2754         TEST_CHK(status == FDB_RESULT_SUCCESS);
2755     }
2756 
2757     // compact pre & post commit
2758     fdb_compact(dbfile, NULL);
2759     fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2760     fdb_compact(dbfile, NULL);
2761 
2762     // scan after flush
2763     count = 0;
2764     fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, 0x0);
2765     do {
2766         status = fdb_iterator_get(iterator, &rdoc);
2767         TEST_CHK(status == FDB_RESULT_SUCCESS);
2768         fdb_doc_free(rdoc);
2769         rdoc = NULL;
2770         count++;
2771     } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2772 
2773     if (eqkeys) {
2774         // since the custom cmp function compares only 3rd~8th bytes,
2775         // all keys are identified as the same key.
2776         TEST_CHK(count == 1);
2777     } else {
2778         TEST_CHK(count == n);
2779     }
2780 
2781     fdb_iterator_close(iterator);
2782 
2783     fdb_close(dbfile);
2784     fdb_shutdown();
2785 
2786     memleak_end();
2787     TEST_RESULT("custom compare commit compact");
2788 
2789 }
2790 
custom_seqnum_test(bool multi_kv)2791 void custom_seqnum_test(bool multi_kv)
2792 {
2793     TEST_INIT();
2794