1/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/*
3 *     Copyright 2010 Couchbase, Inc
4 *
5 *   Licensed under the Apache License, Version 2.0 (the "License");
6 *   you may not use this file except in compliance with the License.
7 *   You may obtain a copy of the License at
8 *
9 *       http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *   Unless required by applicable law or agreed to in writing, software
12 *   distributed under the License is distributed on an "AS IS" BASIS,
13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *   See the License for the specific language governing permissions and
15 *   limitations under the License.
16 */
17
18#include <stdio.h>
19#include <stdlib.h>
20#include <string.h>
21#include <stdint.h>
22#include <time.h>
23#if !defined(WIN32) && !defined(_WIN32)
24#include <unistd.h>
25#include <sys/types.h>
26#include <sys/stat.h>
27#endif
28
29#include "libforestdb/forestdb.h"
30#include "test.h"
31
32#include "internal_types.h"
33#include "wal.h"
34#include "functional_util.h"
35
36#undef THREAD_SANITIZER
37#if __clang__
38#   if defined(__has_feature) && __has_feature(thread_sanitizer)
39#define THREAD_SANITIZER
40#   endif
41#endif
42
43struct cb_args {
44    int n_moved_docs;
45    int n_batch_move;
46    bool begin;
47    bool end;
48    bool wal_flush;
49    fdb_kvs_handle *handle;
50};
51
52static fdb_compact_decision compaction_cb(fdb_file_handle *fhandle,
53                            fdb_compaction_status status, const char *kv_name,
54                            fdb_doc *doc, uint64_t old_offset,
55                            uint64_t new_offset,
56                            void *ctx)
57{
58    TEST_INIT();
59    fdb_doc *rdoc;
60    fdb_status s;
61    fdb_compact_decision ret = FDB_CS_KEEP_DOC;
62    struct cb_args *args = (struct cb_args *)ctx;
63
64    (void) doc;
65    (void) new_offset;
66
67    if (status == FDB_CS_BEGIN) {
68        args->begin = true;
69    } else if (status == FDB_CS_END) {
70        args->end = true;
71    } else if (status == FDB_CS_FLUSH_WAL) {
72        args->wal_flush = true;
73    } else if (status == FDB_CS_MOVE_DOC) {
74        if (doc->deleted) {
75            ret = FDB_CS_DROP_DOC;
76        }
77
78        args->n_moved_docs++;
79        fdb_doc_create(&rdoc, NULL, 0, NULL, 0, NULL, 0);
80        rdoc->offset = old_offset;
81        s = fdb_get_byoffset(args->handle, rdoc);
82        TEST_CHK(s == FDB_RESULT_SUCCESS);
83        fdb_doc_free(rdoc);
84
85        if (fhandle->root->config.multi_kv_instances) {
86            TEST_CMP(kv_name, "db", 2);
87        } else {
88            TEST_CMP(kv_name, "default", 7);
89        }
90    } else { // FDB_CS_BATCH_MOVE
91        args->n_batch_move++;
92        fdb_doc_create(&rdoc, NULL, 0, NULL, 0, NULL, 0);
93        rdoc->offset = old_offset;
94        s = fdb_get_byoffset(args->handle, rdoc);
95        TEST_CHK (s == FDB_RESULT_SUCCESS);
96        fdb_doc_free(rdoc);
97    }
98    return ret;
99}
100
101void compaction_callback_test(bool multi_kv)
102{
103    TEST_INIT();
104    memleak_start();
105
106    int i, r;
107    int n = 1000;
108    char keybuf[256], bodybuf[256];
109    fdb_file_handle *dbfile;
110    fdb_kvs_handle *db;
111    fdb_status s;
112    fdb_config fconfig = fdb_get_default_config();
113    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
114    struct cb_args cb_args;
115
116    memset(&cb_args, 0x0, sizeof(struct cb_args));
117    fconfig.wal_threshold = 1024;
118    fconfig.flags = FDB_OPEN_FLAG_CREATE;
119    fconfig.compaction_cb = compaction_cb;
120    fconfig.compaction_cb_ctx = &cb_args;
121    fconfig.compaction_cb_mask = FDB_CS_BEGIN |
122                                 FDB_CS_MOVE_DOC |
123                                 FDB_CS_FLUSH_WAL |
124                                 FDB_CS_END;
125    fconfig.multi_kv_instances = multi_kv;
126
127    // remove all previous compact_test files
128    r = system(SHELL_DEL" compact_test* > errorlog.txt");
129    (void)r;
130
131    // open db
132    fdb_open(&dbfile, "./compact_test1", &fconfig);
133    if (multi_kv) {
134        fdb_kvs_open(dbfile, &db, "db", &kvs_config);
135    } else {
136        fdb_kvs_open_default(dbfile, &db, &kvs_config);
137    }
138    cb_args.handle = db;
139
140    // write docs
141    for (i=0;i<n;++i){
142        sprintf(keybuf, "key%04d", i);
143        sprintf(bodybuf, "body%04d", i);
144        s = fdb_set_kv(db, keybuf, strlen(keybuf), bodybuf, strlen(bodybuf));
145        TEST_CHK(s == FDB_RESULT_SUCCESS);
146    }
147    s = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
148    TEST_CHK(s == FDB_RESULT_SUCCESS);
149    s = fdb_compact(dbfile, "./compact_test2");
150    TEST_CHK(s == FDB_RESULT_SUCCESS);
151
152    TEST_CHK(cb_args.n_moved_docs == n);
153    TEST_CHK(cb_args.begin);
154    TEST_CHK(cb_args.end);
155    TEST_CHK(cb_args.wal_flush);
156    fdb_close(dbfile);
157
158    // open db without move doc
159    memset(&cb_args, 0x0, sizeof(struct cb_args));
160    fconfig.compaction_cb_mask = FDB_CS_BEGIN |
161                                 FDB_CS_FLUSH_WAL |
162                                 FDB_CS_END;
163    fdb_open(&dbfile, "./compact_test2", &fconfig);
164
165    if (multi_kv) {
166        fdb_kvs_open(dbfile, &db, "db", &kvs_config);
167    } else {
168        fdb_kvs_open_default(dbfile, &db, &kvs_config);
169    }
170    cb_args.handle = db;
171    s = fdb_compact(dbfile, "./compact_test3");
172    TEST_CHK(s == FDB_RESULT_SUCCESS);
173    TEST_CHK(cb_args.n_moved_docs == 0);
174    TEST_CHK(cb_args.begin);
175    TEST_CHK(cb_args.end);
176    TEST_CHK(cb_args.wal_flush);
177    fdb_close(dbfile);
178
179    // open db without wal_flush
180    memset(&cb_args, 0x0, sizeof(struct cb_args));
181    fconfig.compaction_cb_mask = FDB_CS_BEGIN |
182                                 FDB_CS_MOVE_DOC |
183                                 FDB_CS_END;
184    fdb_open(&dbfile, "./compact_test3", &fconfig);
185    if (multi_kv) {
186        fdb_kvs_open(dbfile, &db, "db", &kvs_config);
187    } else {
188        fdb_kvs_open_default(dbfile, &db, &kvs_config);
189    }
190    cb_args.handle = db;
191    s = fdb_compact(dbfile, "./compact_test4");
192    TEST_CHK(s == FDB_RESULT_SUCCESS);
193    TEST_CHK(cb_args.n_moved_docs == n);
194    TEST_CHK(cb_args.begin);
195    TEST_CHK(cb_args.end);
196    TEST_CHK(!cb_args.wal_flush);
197    fdb_close(dbfile);
198
199    // open db without begin/end
200    memset(&cb_args, 0x0, sizeof(struct cb_args));
201    fconfig.compaction_cb_mask = FDB_CS_MOVE_DOC |
202                                 FDB_CS_FLUSH_WAL;
203    fdb_open(&dbfile, "./compact_test4", &fconfig);
204    if (multi_kv) {
205        fdb_kvs_open(dbfile, &db, "db", &kvs_config);
206    } else {
207        fdb_kvs_open_default(dbfile, &db, &kvs_config);
208    }
209    cb_args.handle = db;
210    s = fdb_compact(dbfile, "./compact_test5");
211    TEST_CHK(s == FDB_RESULT_SUCCESS);
212    TEST_CHK(cb_args.n_moved_docs == n);
213    TEST_CHK(!cb_args.begin);
214    TEST_CHK(!cb_args.end);
215    TEST_CHK(cb_args.wal_flush);
216    fdb_close(dbfile);
217
218    // open db with batch move
219    memset(&cb_args, 0x0, sizeof(struct cb_args));
220    fconfig.compaction_cb_mask = FDB_CS_BATCH_MOVE;
221    fdb_open(&dbfile, "./compact_test5", &fconfig);
222    if (multi_kv) {
223        fdb_kvs_open(dbfile, &db, "db", &kvs_config);
224    } else {
225        fdb_kvs_open_default(dbfile, &db, &kvs_config);
226    }
227    cb_args.handle = db;
228    s = fdb_compact(dbfile, "./compact_test6");
229    TEST_CHK(s == FDB_RESULT_SUCCESS);
230    TEST_CHK(cb_args.n_moved_docs == 0);
231    TEST_CHK(cb_args.n_batch_move && cb_args.n_batch_move <= n);
232    TEST_CHK(!cb_args.begin);
233    TEST_CHK(!cb_args.end);
234    TEST_CHK(!cb_args.wal_flush);
235    fdb_close(dbfile);
236
237    fdb_shutdown();
238
239    memleak_end();
240    if (multi_kv) {
241        TEST_RESULT("compaction callback function multi kv mode test");
242    } else {
243        TEST_RESULT("compaction callback function single kv mode test");
244    }
245}
246
247void compact_wo_reopen_test()
248{
249    TEST_INIT();
250
251    memleak_start();
252
253    int i, r;
254    int n = 3;
255    fdb_file_handle *dbfile, *dbfile_new;
256    fdb_kvs_handle *db;
257    fdb_kvs_handle *db_new;
258    fdb_doc **doc = alca(fdb_doc*, n);
259    fdb_doc *rdoc;
260    fdb_status status;
261
262    char keybuf[256], metabuf[256], bodybuf[256];
263
264    // remove previous compact_test files
265    r = system(SHELL_DEL" compact_test* > errorlog.txt");
266    (void)r;
267
268    fdb_config fconfig = fdb_get_default_config();
269    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
270    fconfig.buffercache_size = 16777216;
271    fconfig.wal_threshold = 1024;
272    fconfig.flags = FDB_OPEN_FLAG_CREATE;
273    fconfig.compaction_threshold = 0;
274
275    // open db
276    fdb_open(&dbfile, "./compact_test1", &fconfig);
277    fdb_kvs_open_default(dbfile, &db, &kvs_config);
278    status = fdb_set_log_callback(db, logCallbackFunc,
279                                  (void *) "compact_wo_reopen_test");
280    TEST_CHK(status == FDB_RESULT_SUCCESS);
281    fdb_open(&dbfile_new, "./compact_test1", &fconfig);
282    fdb_kvs_open_default(dbfile_new, &db_new, &kvs_config);
283    status = fdb_set_log_callback(db_new, logCallbackFunc,
284                                  (void *) "compact_wo_reopen_test");
285    TEST_CHK(status == FDB_RESULT_SUCCESS);
286
287    // insert documents
288    for (i=0;i<n;++i){
289        sprintf(keybuf, "key%d", i);
290        sprintf(metabuf, "meta%d", i);
291        sprintf(bodybuf, "body%d", i);
292        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
293            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
294        fdb_set(db, doc[i]);
295    }
296
297    // remove doc
298    fdb_doc_create(&rdoc, doc[1]->key, doc[1]->keylen, doc[1]->meta, doc[1]->metalen, NULL, 0);
299    rdoc->deleted = true;
300    fdb_set(db, rdoc);
301    fdb_doc_free(rdoc);
302
303    // commit
304    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
305
306    // perform compaction using one handle
307    fdb_compact(dbfile, (char *) "./compact_test2");
308
309    // retrieve documents using the other handle without close/re-open
310    for (i=0;i<n;++i){
311        // search by key
312        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
313        status = fdb_get(db_new, rdoc);
314
315        if (i != 1) {
316            TEST_CHK(status == FDB_RESULT_SUCCESS);
317            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
318            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
319        }else{
320            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
321        }
322
323        // free result document
324        fdb_doc_free(rdoc);
325    }
326    // check the other handle's filename
327    fdb_file_info info;
328    fdb_get_file_info(dbfile_new, &info);
329    TEST_CHK(!strcmp("./compact_test2", info.filename));
330
331    // free all documents
332    for (i=0;i<n;++i){
333        fdb_doc_free(doc[i]);
334    }
335
336    // close db file
337    fdb_kvs_close(db);
338    fdb_kvs_close(db_new);
339    fdb_close(dbfile);
340    fdb_close(dbfile_new);
341
342    // free all resources
343    fdb_shutdown();
344
345    memleak_end();
346
347    TEST_RESULT("compaction without reopen test");
348}
349
350void compact_with_reopen_test()
351{
352    TEST_INIT();
353
354    memleak_start();
355
356    int i, r;
357    int n = 100;
358    fdb_file_handle *dbfile;
359    fdb_kvs_handle *db;
360    fdb_doc **doc = alca(fdb_doc*, n);
361    fdb_doc *rdoc;
362    fdb_status status;
363
364    char keybuf[256], metabuf[256], bodybuf[256], temp[256];
365
366    // remove previous compact_test files
367    r = system(SHELL_DEL" compact_test* > errorlog.txt");
368    (void)r;
369
370    fdb_config fconfig = fdb_get_default_config();
371    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
372    fconfig.buffercache_size = 16777216;
373    fconfig.wal_threshold = 1024;
374    fconfig.flags = FDB_OPEN_FLAG_CREATE;
375    fconfig.compaction_threshold = 0;
376
377    // open db
378    fdb_open(&dbfile, "./compact_test1", &fconfig);
379    fdb_kvs_open_default(dbfile, &db, &kvs_config);
380    status = fdb_set_log_callback(db, logCallbackFunc,
381                                  (void *) "compact_with_reopen_test");
382    TEST_CHK(status == FDB_RESULT_SUCCESS);
383
384    // insert documents
385    for (i=0;i<n;++i){
386        sprintf(keybuf, "key%d", i);
387        sprintf(metabuf, "meta%d", i);
388        sprintf(bodybuf, "body%d", i);
389        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
390            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
391        fdb_set(db, doc[i]);
392    }
393
394    // remove doc
395    fdb_doc_create(&rdoc, doc[1]->key, doc[1]->keylen, doc[1]->meta, doc[1]->metalen, NULL, 0);
396    rdoc->deleted = true;
397    fdb_set(db, rdoc);
398    fdb_doc_free(rdoc);
399
400    // commit
401    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
402
403    // perform compaction using one handle
404    fdb_compact(dbfile, (char *) "./compact_test2");
405
406    // close db file
407    fdb_kvs_close(db);
408    fdb_close(dbfile);
409
410    r = system(SHELL_MOVE " compact_test2 compact_test1 > errorlog.txt");
411    (void)r;
412    fdb_open(&dbfile, "./compact_test1", &fconfig);
413    fdb_kvs_open_default(dbfile, &db, &kvs_config);
414    status = fdb_set_log_callback(db, logCallbackFunc,
415                                  (void *) "compact_with_reopen_test");
416    TEST_CHK(status == FDB_RESULT_SUCCESS);
417
418    // retrieve documents using the other handle without close/re-open
419    for (i=0;i<n;++i){
420        // search by key
421        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
422        status = fdb_get(db, rdoc);
423
424        if (i != 1) {
425            TEST_CHK(status == FDB_RESULT_SUCCESS);
426            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
427            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
428        }else{
429            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
430        }
431
432        // free result document
433        fdb_doc_free(rdoc);
434    }
435    // check the other handle's filename
436    fdb_file_info info;
437    fdb_get_file_info(dbfile, &info);
438    TEST_CHK(!strcmp("./compact_test1", info.filename));
439
440    // update documents
441    for (i=0;i<n;++i){
442        sprintf(metabuf, "newmeta%d", i);
443        sprintf(bodybuf, "newbody%d_%s", i, temp);
444        fdb_doc_update(&doc[i], (void *)metabuf, strlen(metabuf),
445                       (void *)bodybuf, strlen(bodybuf));
446        fdb_set(db, doc[i]);
447    }
448
449    // Open the database with another handle.
450    fdb_file_handle *second_dbfile;
451    fdb_kvs_handle *second_dbh;
452    fdb_open(&second_dbfile, "./compact_test1", &fconfig);
453    fdb_kvs_open_default(second_dbfile, &second_dbh, &kvs_config);
454    status = fdb_set_log_callback(second_dbh, logCallbackFunc,
455                                  (void *) "compact_with_reopen_test");
456    TEST_CHK(status == FDB_RESULT_SUCCESS);
457    // In-place compactions with a handle still open on the first old file
458    status = fdb_compact(dbfile, NULL);
459    TEST_CHK(status == FDB_RESULT_SUCCESS);
460
461    status = fdb_compact(dbfile, NULL);
462    TEST_CHK(status == FDB_RESULT_SUCCESS);
463
464    // MB-12977: retest compaction again..
465    status = fdb_compact(dbfile, NULL);
466    TEST_CHK(status == FDB_RESULT_SUCCESS);
467
468    fdb_kvs_close(db);
469    fdb_close(dbfile);
470
471    for (i=0;i<n;++i){
472        // search by key
473        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
474        status = fdb_get(second_dbh, rdoc);
475        TEST_CHK(status == FDB_RESULT_SUCCESS);
476        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
477        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
478        // free result document
479        fdb_doc_free(rdoc);
480    }
481
482    // Open database with an original name.
483    status = fdb_open(&dbfile, "./compact_test1", &fconfig);
484    TEST_CHK(status == FDB_RESULT_SUCCESS);
485    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
486    TEST_CHK(status == FDB_RESULT_SUCCESS);
487    status = fdb_set_log_callback(db, logCallbackFunc,
488                                  (void *) "compact_with_reopen_test");
489    TEST_CHK(status == FDB_RESULT_SUCCESS);
490    fdb_get_file_info(dbfile, &info);
491    // The actual file name should be a compacted one.
492    TEST_CHK(!strcmp("./compact_test1.3", info.filename));
493
494    fdb_kvs_close(second_dbh);
495    fdb_close(second_dbfile);
496
497    for (i=0;i<n;++i){
498        // search by key
499        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
500        status = fdb_get(db, rdoc);
501        TEST_CHK(status == FDB_RESULT_SUCCESS);
502        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
503        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
504        // free result document
505        fdb_doc_free(rdoc);
506    }
507
508    fdb_compact(dbfile, NULL);
509    fdb_kvs_close(db);
510    fdb_close(dbfile);
511
512    r = system(SHELL_MOVE " compact_test1 compact_test.fdb > errorlog.txt");
513    (void)r;
514    fdb_open(&dbfile, "./compact_test.fdb", &fconfig);
515    fdb_kvs_open_default(dbfile, &db, &kvs_config);
516    status = fdb_set_log_callback(db, logCallbackFunc,
517                                  (void *) "compact_with_reopen_test");
518    TEST_CHK(status == FDB_RESULT_SUCCESS);
519    // In-place compaction
520    fdb_compact(dbfile, NULL);
521    fdb_kvs_close(db);
522    fdb_close(dbfile);
523    // Open database with an original name.
524    status = fdb_open(&dbfile, "./compact_test.fdb", &fconfig);
525    TEST_CHK(status == FDB_RESULT_SUCCESS);
526    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
527    TEST_CHK(status == FDB_RESULT_SUCCESS);
528    status = fdb_set_log_callback(db, logCallbackFunc,
529                                  (void *) "compact_with_reopen_test");
530    TEST_CHK(status == FDB_RESULT_SUCCESS);
531    fdb_get_file_info(dbfile, &info);
532    TEST_CHK(!strcmp("./compact_test.fdb", info.filename));
533    TEST_CHK(info.doc_count == 100);
534
535    // free all documents
536    for (i=0;i<n;++i){
537        fdb_doc_free(doc[i]);
538    }
539
540    fdb_kvs_close(db);
541    fdb_close(dbfile);
542
543    // free all resources
544    fdb_shutdown();
545
546    memleak_end();
547
548    TEST_RESULT("compaction with reopen test");
549}
550
551void compact_reopen_named_kvs()
552{
553    TEST_INIT();
554
555    memleak_start();
556
557    int i, r;
558    int n = 100;
559    int nkvdocs;
560    fdb_file_handle *dbfile;
561    fdb_kvs_handle *db;
562    fdb_doc **doc = alca(fdb_doc*, n);
563    fdb_status status;
564    fdb_kvs_info kvs_info;
565
566    char keybuf[256], metabuf[256], bodybuf[256];
567
568    // remove previous compact_test files
569    r = system(SHELL_DEL" compact_test* > errorlog.txt");
570    (void)r;
571
572    fdb_config fconfig = fdb_get_default_config();
573    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
574    fconfig.wal_threshold = 1024;
575    fconfig.flags = FDB_OPEN_FLAG_CREATE;
576    fconfig.compaction_threshold = 0;
577
578    // open db
579    fdb_open(&dbfile, "./compact_test1", &fconfig);
580    fdb_kvs_open(dbfile, &db, "db",  &kvs_config);
581
582    status = fdb_set_log_callback(db, logCallbackFunc,
583                                  (void *) "compact_reopen_named_kvs");
584    TEST_CHK(status == FDB_RESULT_SUCCESS);
585
586    for (i=0;i<n;++i){
587        sprintf(keybuf, "key%d", i);
588        sprintf(metabuf, "meta%d", i);
589        sprintf(bodybuf, "body%d", i);
590        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
591            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
592        fdb_set(db, doc[i]);
593    }
594
595    // commit
596    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
597
598    // compact
599    fdb_compact(dbfile, NULL);
600
601    // save ndocs
602    fdb_get_kvs_info(db, &kvs_info);
603    nkvdocs = kvs_info.doc_count;
604
605    // close db file
606    fdb_kvs_close(db);
607    fdb_close(dbfile);
608
609    // reopen
610    fdb_open(&dbfile, "./compact_test1", &fconfig);
611    fdb_kvs_open(dbfile, &db, "db",  &kvs_config);
612
613    // verify kvs stats
614    fdb_get_kvs_info(db, &kvs_info);
615    TEST_CHK((uint64_t)nkvdocs == kvs_info.doc_count);
616
617    fdb_kvs_close(db);
618    fdb_close(dbfile);
619
620    // free all documents
621    for (i=0;i<n;++i){
622        fdb_doc_free(doc[i]);
623    }
624    fdb_shutdown();
625
626    memleak_end();
627
628    TEST_RESULT("compact reopen named kvs");
629}
630
631void compact_reopen_with_iterator()
632{
633    TEST_INIT();
634
635    memleak_start();
636
637    int i, r;
638    int n = 9;
639    int count = 0;
640    int nkvdocs;
641    fdb_file_handle *dbfile, *compact_file;
642    fdb_kvs_handle *db;
643    fdb_doc **doc = alca(fdb_doc*, n);
644    fdb_doc *rdoc = NULL;
645    fdb_status status;
646    fdb_kvs_info kvs_info;
647    fdb_iterator *iterator;
648
649    char keybuf[256], metabuf[256], bodybuf[256];
650
651    // remove previous compact_test files
652    r = system(SHELL_DEL" compact_test* > errorlog.txt");
653    (void)r;
654
655    fdb_config fconfig = fdb_get_default_config();
656    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
657    fconfig.wal_threshold = 1024;
658    fconfig.flags = FDB_OPEN_FLAG_CREATE;
659    fconfig.compaction_threshold = 0;
660
661    // open db
662    fdb_open(&dbfile, "./compact_test1", &fconfig);
663    fdb_kvs_open(dbfile, &db, "db",  &kvs_config);
664
665    status = fdb_set_log_callback(db, logCallbackFunc,
666                                  (void *) "compact_reopen_with_iterator");
667    TEST_CHK(status == FDB_RESULT_SUCCESS);
668
669    for (i=0;i<n;++i){
670        sprintf(keybuf, "key%d", i);
671        sprintf(metabuf, "meta%d", i);
672        sprintf(bodybuf, "body%d", i);
673        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
674            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
675        fdb_set(db, doc[i]);
676    }
677
678    // commit
679    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
680
681    // MB-13859: compact the file using a separate handle..
682    fdb_open(&compact_file, "./compact_test1", &fconfig);
683    // compact
684    status = fdb_compact(compact_file, "./compact_test2");
685    TEST_CHK(status == FDB_RESULT_SUCCESS);
686    // close file after compaction to make the new_file's ref count 0
687    fdb_close(compact_file);
688
689    i = 0;
690    status = fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, FDB_ITR_NONE);
691    TEST_CHK(status == FDB_RESULT_SUCCESS);
692    do {
693        status = fdb_iterator_get(iterator, &rdoc);
694        TEST_CHK(status == FDB_RESULT_SUCCESS);
695
696        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
697        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
698        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
699
700        fdb_doc_free(rdoc);
701        rdoc = NULL;
702        i++;
703        count++;
704    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
705
706    TEST_CHK(count==n);
707
708
709    status = fdb_iterator_close(iterator);
710    TEST_CHK(status == FDB_RESULT_SUCCESS);
711
712    // save ndocs
713    fdb_get_kvs_info(db, &kvs_info);
714    nkvdocs = kvs_info.doc_count;
715
716    // close db file
717    fdb_kvs_close(db);
718    fdb_close(dbfile);
719
720    // reopen
721    fdb_open(&dbfile, "./compact_test2", &fconfig);
722    fdb_kvs_open(dbfile, &db, "db",  &kvs_config);
723
724    // verify kvs stats
725    fdb_get_kvs_info(db, &kvs_info);
726    TEST_CHK((uint64_t)nkvdocs == kvs_info.doc_count);
727
728    fdb_kvs_close(db);
729    fdb_close(dbfile);
730
731    // free all documents
732    for (i=0;i<n;++i){
733        fdb_doc_free(doc[i]);
734    }
735    fdb_shutdown();
736
737    memleak_end();
738
739    TEST_RESULT("compact reopen with iterator");
740}
741
742void estimate_space_upto_test(bool multi_kv)
743{
744    TEST_INIT();
745
746    memleak_start();
747
748    int i, j, r;
749    int n = 300;
750    int num_kvs = 4; // keep this the same as number of fdb_commit() calls
751    fdb_file_handle *dbfile;
752    fdb_kvs_handle **db = alca(fdb_kvs_handle *, num_kvs);
753    fdb_doc **doc = alca(fdb_doc*, n);
754    fdb_status status;
755    fdb_snapshot_info_t *markers;
756    uint64_t num_markers;
757    size_t space_used;
758
759    char keybuf[256], metabuf[256], bodybuf[256];
760    char kv_name[8];
761
762    fdb_config fconfig = fdb_get_default_config();
763    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
764    fconfig.buffercache_size = 0;
765    fconfig.wal_threshold = 50;
766    fconfig.flags = FDB_OPEN_FLAG_CREATE;
767    fconfig.compaction_threshold = 0;
768    fconfig.multi_kv_instances = multi_kv;
769
770    // remove previous compact_test files
771    r = system(SHELL_DEL" compact_test* > errorlog.txt");
772    (void)r;
773
774    // open db
775    fdb_open(&dbfile, "./compact_test1", &fconfig);
776    if (multi_kv) {
777        for (r = 0; r < num_kvs; ++r) {
778            sprintf(kv_name, "kv%d", r);
779            fdb_kvs_open(dbfile, &db[r], kv_name, &kvs_config);
780        }
781    } else {
782        num_kvs = 1;
783        fdb_kvs_open_default(dbfile, &db[0], &kvs_config);
784    }
785
786   // ------- Setup test ----------------------------------
787   // insert first quarter of documents
788    for (i=0; i<n/4; i++){
789        sprintf(keybuf, "key%d", i);
790        sprintf(metabuf, "meta%d", i);
791        sprintf(bodybuf, "body%d", i);
792        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
793            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
794        for (r = 0; r < num_kvs; ++r) {
795            fdb_set(db[r], doc[i]);
796        }
797    }
798
799    // commit with a manual WAL flush (these docs go into HB-trie)
800    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
801
802    // insert second quarter of documents
803    for (; i < n/2; i++){
804        sprintf(keybuf, "key%d", i);
805        sprintf(metabuf, "meta%d", i);
806        sprintf(bodybuf, "body%d", i);
807        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
808            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
809        for (r = 0; r < num_kvs; ++r) {
810            fdb_set(db[r], doc[i]);
811        }
812    }
813    // Update first quarter of documents again..
814    for (j = 0; j < n/4; j++){
815        for (r = 0; r < num_kvs; ++r) {
816            fdb_set(db[r], doc[j]);
817        }
818    }
819    // Update first quarter of documents again overwriting previous update..
820    for (j = 0; j < n/4; j++){
821        for (r = 0; r < num_kvs; ++r) {
822            fdb_set(db[r], doc[j]);
823        }
824    }
825
826    // commit again without a WAL flush (some of these docs remain in WAL)
827    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
828
829    // insert third quarter of documents
830    for (; i < (n/2 + n/4); i++){
831        sprintf(keybuf, "key%d", i);
832        sprintf(metabuf, "meta%d", i);
833        sprintf(bodybuf, "body%d", i);
834        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
835            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
836        for (r = 0; r < num_kvs; ++r) {
837            fdb_set(db[r], doc[i]);
838        }
839    }
840    // manually flush WAL & commit (these docs go into HB-trie)
841    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
842
843    // insert fourth quarter of documents
844    for (; i < n; i++){
845        sprintf(keybuf, "key%d", i);
846        sprintf(metabuf, "meta%d", i);
847        sprintf(bodybuf, "body%d", i);
848        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
849            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
850        for (r = 0; r < num_kvs; ++r) {
851            fdb_set(db[r], doc[i]);
852        }
853    }
854    // commit without a WAL flush (some of these docs remain in the WAL)
855    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
856
857    for (r = 0; r < num_kvs; ++r) {
858        status = fdb_set_log_callback(db[r], logCallbackFunc,
859                                      (void *) "estimate_space_upto_test");
860        TEST_CHK(status == FDB_RESULT_SUCCESS);
861    }
862
863    status = fdb_get_all_snap_markers(dbfile, &markers, &num_markers);
864    TEST_CHK(status == FDB_RESULT_SUCCESS);
865
866    if (!multi_kv) {
867        size_t space_used2;
868        TEST_CHK(num_markers == 4);
869        space_used = fdb_estimate_space_used_from(dbfile, markers[1].marker);
870        space_used2 = fdb_estimate_space_used_from(dbfile, markers[2].marker);
871        TEST_CHK(space_used2 > space_used); // greater than space used by just 1
872    } else {
873        size_t space_used2;
874        TEST_CHK(num_markers == 8);
875        space_used = fdb_estimate_space_used_from(dbfile, markers[1].marker);
876        space_used2 = fdb_estimate_space_used_from(dbfile, markers[2].marker);
877        TEST_CHK(space_used2 > space_used); // greater than space used by just 1
878    }
879
880    status = fdb_free_snap_markers(markers, num_markers);
881    TEST_CHK(status == FDB_RESULT_SUCCESS);
882
883    // close db file
884    fdb_close(dbfile);
885
886    // free all documents
887    for (i=0;i<n;++i){
888        fdb_doc_free(doc[i]);
889    }
890
891    // free all resources
892    fdb_shutdown();
893
894    memleak_end();
895
896    sprintf(bodybuf, "estimate space upto marker in file test %s", multi_kv ?
897                                                           "multiple kv mode:"
898                                                         : "single kv mode:");
899    TEST_RESULT(bodybuf);
900}
901
902void compact_upto_test(bool multi_kv)
903{
904    TEST_INIT();
905
906    memleak_start();
907
908    int i, r;
909    int n = 20;
910    int num_kvs = 4; // keep this the same as number of fdb_commit() calls
911    fdb_file_handle *dbfile;
912    fdb_kvs_handle **db = alca(fdb_kvs_handle *, num_kvs);
913    fdb_doc **doc = alca(fdb_doc*, n);
914    fdb_status status;
915    fdb_snapshot_info_t *markers;
916    fdb_kvs_handle *snapshot;
917    uint64_t num_markers;
918
919    char keybuf[256], metabuf[256], bodybuf[256];
920    char kv_name[8];
921    char compact_filename[32];
922
923    fdb_config fconfig = fdb_get_default_config();
924    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
925    fconfig.buffercache_size = 0;
926    fconfig.wal_threshold = 1024;
927    fconfig.flags = FDB_OPEN_FLAG_CREATE;
928    fconfig.compaction_threshold = 0;
929    fconfig.multi_kv_instances = multi_kv;
930    // since this test requires static number of markers,
931    // disable block reusing
932    fconfig.block_reusing_threshold = 0;
933
934    // remove previous compact_test files
935    r = system(SHELL_DEL" compact_test* > errorlog.txt");
936    (void)r;
937
938    // open db
939    fdb_open(&dbfile, "./compact_test1", &fconfig);
940    if (multi_kv) {
941        for (r = 0; r < num_kvs; ++r) {
942            sprintf(kv_name, "kv%d", r);
943            fdb_kvs_open(dbfile, &db[r], kv_name, &kvs_config);
944        }
945    } else {
946        num_kvs = 1;
947        fdb_kvs_open_default(dbfile, &db[0], &kvs_config);
948    }
949
950   // ------- Setup test ----------------------------------
951   // insert documents of 0-4
952    for (i=0; i<n/4; i++){
953        sprintf(keybuf, "key%d", i);
954        sprintf(metabuf, "meta%d", i);
955        sprintf(bodybuf, "body%d", i);
956        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
957            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
958        for (r = 0; r < num_kvs; ++r) {
959            fdb_set(db[r], doc[i]);
960        }
961    }
962
963    // commit with a manual WAL flush (these docs go into HB-trie)
964    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
965
966    // insert documents from 5 - 9
967    for (; i < n/2; i++){
968        sprintf(keybuf, "key%d", i);
969        sprintf(metabuf, "meta%d", i);
970        sprintf(bodybuf, "body%d", i);
971        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
972            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
973        for (r = 0; r < num_kvs; ++r) {
974            fdb_set(db[r], doc[i]);
975        }
976    }
977
978    // commit again without a WAL flush
979    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
980
981    // insert documents from 10-14 into HB-trie
982    for (; i < (n/2 + n/4); i++){
983        sprintf(keybuf, "key%d", i);
984        sprintf(metabuf, "meta%d", i);
985        sprintf(bodybuf, "body%d", i);
986        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
987            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
988        for (r = 0; r < num_kvs; ++r) {
989            fdb_set(db[r], doc[i]);
990        }
991    }
992    // manually flush WAL & commit
993    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
994
995    // insert documents from 15 - 19 on file into the WAL
996    for (; i < n; i++){
997        sprintf(keybuf, "key%d", i);
998        sprintf(metabuf, "meta%d", i);
999        sprintf(bodybuf, "body%d", i);
1000        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1001            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1002        for (r = 0; r < num_kvs; ++r) {
1003            fdb_set(db[r], doc[i]);
1004        }
1005    }
1006    // commit without a WAL flush
1007    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1008
1009    for (r = 0; r < num_kvs; ++r) {
1010        status = fdb_set_log_callback(db[r], logCallbackFunc,
1011                                      (void *) "compact_upto_test");
1012        TEST_CHK(status == FDB_RESULT_SUCCESS);
1013    }
1014
1015    status = fdb_get_all_snap_markers(dbfile, &markers, &num_markers);
1016    TEST_CHK(status == FDB_RESULT_SUCCESS);
1017
1018    if (!multi_kv) {
1019        TEST_CHK(num_markers == 4);
1020        for (r = 0; (uint64_t)r < num_markers; ++r) {
1021            TEST_CHK(markers[r].num_kvs_markers == 1);
1022            TEST_CHK(markers[r].kvs_markers[0].seqnum ==
1023                     (fdb_seqnum_t)(n - r*5));
1024        }
1025        r = 1; // Test compacting upto sequence number 15
1026        sprintf(compact_filename, "compact_test_compact%d", r);
1027        status = fdb_compact_upto(dbfile, compact_filename,
1028                                  markers[r].marker);
1029        TEST_CHK(status == FDB_RESULT_SUCCESS);
1030        // create a snapshot
1031        status = fdb_snapshot_open(db[0], &snapshot,
1032                                   markers[r].kvs_markers[0].seqnum);
1033        TEST_CHK(status == FDB_RESULT_SUCCESS);
1034        // close snapshot
1035        fdb_kvs_close(snapshot);
1036    } else {
1037        TEST_CHK(num_markers == 8);
1038        for (r = 0; r < num_kvs; ++r) {
1039            TEST_CHK(markers[r].num_kvs_markers == num_kvs);
1040            for (i = 0; i < num_kvs; ++i) {
1041                TEST_CHK(markers[r].kvs_markers[i].seqnum
1042                         == (fdb_seqnum_t)(n - r*5));
1043                sprintf(kv_name, "kv%d", i);
1044                TEST_CMP(markers[r].kvs_markers[i].kv_store_name, kv_name, 3);
1045            }
1046        }
1047        i = r = 1;
1048        sprintf(compact_filename, "compact_test_compact%d", i);
1049        status = fdb_compact_upto(dbfile, compact_filename,
1050                markers[i].marker);
1051        TEST_CHK(status == FDB_RESULT_SUCCESS);
1052        // create a snapshot
1053        status = fdb_snapshot_open(db[r], &snapshot,
1054                markers[i].kvs_markers[r].seqnum);
1055        TEST_CHK(status == FDB_RESULT_SUCCESS);
1056        // close snapshot
1057        fdb_kvs_close(snapshot);
1058    }
1059
1060    status = fdb_free_snap_markers(markers, num_markers);
1061    TEST_CHK(status == FDB_RESULT_SUCCESS);
1062
1063    // close db file
1064    fdb_close(dbfile);
1065
1066    // free all documents
1067    for (i=0;i<n;++i){
1068        fdb_doc_free(doc[i]);
1069    }
1070
1071    // free all resources
1072    fdb_shutdown();
1073
1074    memleak_end();
1075
1076    sprintf(bodybuf, "compact upto marker in file test %s", multi_kv ?
1077                                                           "multiple kv mode:"
1078                                                         : "single kv mode:");
1079    TEST_RESULT(bodybuf);
1080}
1081
1082void auto_recover_compact_ok_test()
1083{
1084    TEST_INIT();
1085
1086    memleak_start();
1087
1088    int i, r;
1089    int n = 3;
1090    fdb_file_handle *dbfile, *dbfile_new;
1091    fdb_kvs_handle *db;
1092    fdb_kvs_handle *db_new;
1093    fdb_doc **doc = alca(fdb_doc *, n);
1094    fdb_doc *rdoc;
1095    fdb_status status;
1096
1097    char keybuf[256], metabuf[256], bodybuf[256];
1098
1099    // remove previous compact_test files
1100    r = system(SHELL_DEL " compact_test* > errorlog.txt");
1101    (void)r;
1102
1103    fdb_config fconfig = fdb_get_default_config();
1104    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1105    fconfig.buffercache_size = 16777216;
1106    fconfig.wal_threshold = 1024;
1107    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1108    fconfig.compaction_threshold = 0;
1109
1110    // open db
1111    fdb_open(&dbfile, "./compact_test1", &fconfig);
1112    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1113    status = fdb_set_log_callback(db, logCallbackFunc,
1114                                  (void *) "auto_recover_compact_ok_test");
1115    TEST_CHK(status == FDB_RESULT_SUCCESS);
1116    fdb_open(&dbfile_new, "./compact_test1", &fconfig);
1117    fdb_kvs_open_default(dbfile_new, &db_new, &kvs_config);
1118    status = fdb_set_log_callback(db_new, logCallbackFunc,
1119                                  (void *) "auto_recover_compact_ok_test");
1120    TEST_CHK(status == FDB_RESULT_SUCCESS);
1121
1122    // insert first two documents
1123    for (i=0;i<2;++i){
1124        sprintf(keybuf, "key%d", i);
1125        sprintf(metabuf, "meta%d", i);
1126        sprintf(bodybuf, "body%d", i);
1127        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1128            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1129        fdb_set(db, doc[i]);
1130    }
1131
1132    // remove second doc
1133    fdb_doc_create(&rdoc, doc[1]->key, doc[1]->keylen, doc[1]->meta, doc[1]->metalen, NULL, 0);
1134    rdoc->deleted = true;
1135    fdb_set(db, rdoc);
1136    fdb_doc_free(rdoc);
1137
1138    // commit
1139    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1140
1141    // perform compaction using one handle
1142    fdb_compact(dbfile, (char *) "./compact_test2");
1143
1144    // save the old file after compaction is done ..
1145    r = system(SHELL_COPY " compact_test1 compact_test11 > errorlog.txt");
1146    (void)r;
1147
1148    // now insert third doc: it should go to the newly compacted file.
1149    sprintf(keybuf, "key%d", i);
1150    sprintf(metabuf, "meta%d", i);
1151    sprintf(bodybuf, "body%d", i);
1152    fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1153        (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1154    fdb_set(db, doc[i]);
1155
1156    // commit
1157    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1158
1159    // close both the db files ...
1160    fdb_kvs_close(db);
1161    fdb_kvs_close(db_new);
1162    fdb_close(dbfile);
1163    fdb_close(dbfile_new);
1164
1165    // restore the old file after close is done ..
1166    r = system(SHELL_MOVE " compact_test11 compact_test1 > errorlog.txt");
1167    (void)r;
1168
1169    // now open the old saved compacted file, it should automatically recover
1170    // and use the new file since compaction was done successfully
1171    fdb_open(&dbfile_new, "./compact_test1", &fconfig);
1172    fdb_kvs_open_default(dbfile_new, &db_new, &kvs_config);
1173    status = fdb_set_log_callback(db_new, logCallbackFunc,
1174                                  (void *) "auto_recover_compact_ok_test");
1175    TEST_CHK(status == FDB_RESULT_SUCCESS);
1176
1177    // retrieve documents using the old handle and expect all 3 docs
1178    for (i=0;i<n;++i){
1179        // search by key
1180        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
1181        status = fdb_get(db_new, rdoc);
1182
1183        if (i != 1) {
1184            TEST_CHK(status == FDB_RESULT_SUCCESS);
1185            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
1186            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
1187        }else{
1188            TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
1189        }
1190
1191        // free result document
1192        fdb_doc_free(rdoc);
1193    }
1194    // check this handle's filename it should point to newly compacted file
1195    fdb_file_info info;
1196    fdb_get_file_info(dbfile_new, &info);
1197    TEST_CHK(!strcmp("./compact_test2", info.filename));
1198
1199    // close the file
1200    fdb_kvs_close(db_new);
1201    fdb_close(dbfile_new);
1202
1203    // free all documents
1204    for (i=0;i<n;++i){
1205        fdb_doc_free(doc[i]);
1206    }
1207
1208    // free all resources
1209    fdb_shutdown();
1210
1211    memleak_end();
1212
1213    TEST_RESULT("auto recovery after compaction test");
1214}
1215
1216#if !defined(WIN32) && !defined(_WIN32)
1217static bool does_file_exist(const char *filename) {
1218    struct stat st;
1219    int result = stat(filename, &st);
1220    return result == 0;
1221}
1222#else
1223static bool does_file_exist(const char *filename) {
1224    return GetFileAttributes(filename) != INVALID_FILE_ATTRIBUTES;
1225}
1226#endif
1227
1228void unlink_after_compaction_test()
1229{
1230    TEST_INIT();
1231    memleak_start();
1232
1233    int i, r;
1234    int n = 10;
1235    fdb_file_handle *dbfile;
1236    fdb_kvs_handle *db, *snap;
1237    fdb_doc **doc = alca(fdb_doc *, n);
1238    fdb_doc *rdoc;
1239    fdb_status s;
1240
1241    char keybuf[256], metabuf[256], bodybuf[256];
1242
1243    // remove previous compact_test files
1244    r = system(SHELL_DEL " compact_test* > errorlog.txt");
1245    (void)r;
1246
1247    fdb_config fconfig = fdb_get_default_config();
1248    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1249    fconfig.buffercache_size = 16777216;
1250    fconfig.wal_threshold = 1024;
1251    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1252    fconfig.compaction_threshold = 0;
1253
1254    // open db
1255    fdb_open(&dbfile, "./compact_test1", &fconfig);
1256    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1257    s = fdb_set_log_callback(db, logCallbackFunc,
1258                             (void *) "unlink_after_compaction_test");
1259    TEST_CHK(s == FDB_RESULT_SUCCESS);
1260
1261    // set docs
1262    for (i=0;i<n;++i){
1263        sprintf(keybuf, "key%d", i);
1264        sprintf(metabuf, "meta%d", i);
1265        sprintf(bodybuf, "body%d", i);
1266        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1267            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1268        fdb_set(db, doc[i]);
1269    }
1270
1271    // commit
1272    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1273
1274    // create a snapshot
1275    s = fdb_snapshot_open(db, &snap, n);
1276    TEST_CHK(s == FDB_RESULT_SUCCESS);
1277
1278    // compaction
1279    s = fdb_compact(dbfile, "./compact_test2");
1280    TEST_CHK(s == FDB_RESULT_SUCCESS);
1281
1282    // retrieve check on the new file
1283    for (i=0;i<n;++i){
1284        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
1285        s = fdb_get(db, rdoc);
1286        TEST_CHK(s == FDB_RESULT_SUCCESS);
1287        fdb_doc_free(rdoc);
1288    }
1289
1290    // the old file should not be seen by user level application
1291#if !defined(WIN32) && !defined(_WIN32)
1292#ifndef _MSC_VER
1293    TEST_CHK(!does_file_exist("./compact_test1"));
1294#endif
1295#endif
1296
1297    // retrieve check on the old file (snapshot)
1298    for (i=0;i<n;++i){
1299        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
1300        s = fdb_get(snap, rdoc);
1301        TEST_CHK(s == FDB_RESULT_SUCCESS);
1302        fdb_doc_free(rdoc);
1303    }
1304
1305    fdb_close(dbfile);
1306
1307    fdb_shutdown();
1308    for (i=0; i<n; ++i){
1309        fdb_doc_free(doc[i]);
1310    }
1311
1312    memleak_end();
1313    TEST_RESULT("unlink after compaction test");
1314}
1315
1316void db_compact_overwrite()
1317{
1318    TEST_INIT();
1319    memleak_start();
1320
1321    int i, r;
1322    int n = 30;
1323    fdb_file_handle *dbfile, *dbfile2;
1324    fdb_kvs_handle *db, *db2;
1325    fdb_doc **doc = alca(fdb_doc *, n);
1326    fdb_doc **doc2 = alca(fdb_doc *, 2*n);
1327    fdb_doc *rdoc;
1328    fdb_status status;
1329    fdb_config fconfig;
1330    fdb_kvs_info kvs_info;
1331    fdb_kvs_config kvs_config;
1332
1333    char keybuf[256], metabuf[256], bodybuf[256];
1334
1335    // remove previous compact_test files
1336    r = system(SHELL_DEL " compact_test* > errorlog.txt");
1337    (void)r;
1338
1339    fconfig = fdb_get_default_config();
1340    kvs_config = fdb_get_default_kvs_config();
1341    fconfig.buffercache_size = 16777216;
1342    fconfig.wal_threshold = 1024;
1343    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1344    fconfig.compaction_threshold = 0;
1345
1346    // write to db1
1347    fdb_open(&dbfile, "./compact_test1", &fconfig);
1348    fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
1349    status = fdb_set_log_callback(db, logCallbackFunc,
1350                                  (void *) "db_destroy_test");
1351    TEST_CHK(status == FDB_RESULT_SUCCESS);
1352    for (i=0;i<n;++i){
1353        sprintf(keybuf, "key%d", i);
1354        sprintf(metabuf, "meta%d", i);
1355        sprintf(bodybuf, "body%d", i);
1356        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1357            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1358        fdb_set(db, doc[i]);
1359    }
1360    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1361
1362    // Open the empty db with future compact name
1363    fdb_open(&dbfile2, "./compact_test1.1", &fconfig);
1364    fdb_kvs_open(dbfile2, &db2, NULL, &kvs_config);
1365    status = fdb_set_log_callback(db2, logCallbackFunc,
1366                                  (void *) "db_destroy_test");
1367    TEST_CHK(status == FDB_RESULT_SUCCESS);
1368    // write to db2
1369    for (i=0;i < 2*n;++i){
1370        sprintf(keybuf, "k2ey%d", i);
1371        sprintf(metabuf, "m2eta%d", i);
1372        sprintf(bodybuf, "b2ody%d", i);
1373        fdb_doc_create(&doc2[i], (void*)keybuf, strlen(keybuf),
1374            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1375        fdb_set(db2, doc2[i]);
1376    }
1377    fdb_commit(dbfile2, FDB_COMMIT_NORMAL);
1378
1379
1380    // verify db2 seqnum and close
1381    fdb_get_kvs_info(db2, &kvs_info);
1382    TEST_CHK(kvs_info.last_seqnum = 2*n);
1383    fdb_kvs_close(db2);
1384    fdb_close(dbfile2);
1385
1386    // compact db1
1387    status = fdb_compact(dbfile, NULL);
1388    TEST_CHK(status == FDB_RESULT_SUCCESS);
1389
1390    // close db1
1391    fdb_kvs_close(db);
1392    fdb_close(dbfile);
1393
1394    // reopen db1
1395    fdb_open(&dbfile, "./compact_test1", &fconfig);
1396    fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
1397    status = fdb_set_log_callback(db, logCallbackFunc,
1398                                  (void *) "db_destroy_test");
1399    TEST_CHK(status == FDB_RESULT_SUCCESS);
1400    // read db1
1401    for (i=0;i<n;++i){
1402        // search by key
1403        fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
1404        status = fdb_get(db, rdoc);
1405        TEST_CHK(status == FDB_RESULT_SUCCESS);
1406        TEST_CHK(!memcmp(rdoc->meta, doc[i]->meta, rdoc->metalen));
1407        TEST_CHK(!memcmp(rdoc->body, doc[i]->body, rdoc->bodylen));
1408        // free result document
1409        fdb_doc_free(rdoc);
1410    }
1411
1412    // reopen db2
1413    fdb_open(&dbfile2, "./compact_test1.1", &fconfig);
1414    fdb_kvs_open(dbfile2, &db2, NULL, &kvs_config);
1415    status = fdb_set_log_callback(db2, logCallbackFunc,
1416                                  (void *) "db_destroy_test");
1417    TEST_CHK(status == FDB_RESULT_SUCCESS);
1418
1419    fdb_get_kvs_info(db2, &kvs_info);
1420    TEST_CHK(kvs_info.last_seqnum = 2*n);
1421
1422    // read db2
1423    for (i=0;i<2*n;++i){
1424        // search by key
1425        fdb_doc_create(&rdoc, doc2[i]->key, doc2[i]->keylen, NULL, 0, NULL, 0);
1426        status = fdb_get(db2, rdoc);
1427        TEST_CHK(status == FDB_RESULT_SUCCESS);
1428        TEST_CHK(!memcmp(rdoc->meta, doc2[i]->meta, rdoc->metalen));
1429        TEST_CHK(!memcmp(rdoc->body, doc2[i]->body, rdoc->bodylen));
1430        // free result document
1431        fdb_doc_free(rdoc);
1432    }
1433
1434    // free all documents
1435    for (i=0;i<n;++i){
1436        fdb_doc_free(doc[i]);
1437        fdb_doc_free(doc2[i]);
1438    }
1439    for (i=n;i<2*n;++i){
1440        fdb_doc_free(doc2[i]);
1441    }
1442
1443    fdb_kvs_close(db);
1444    fdb_close(dbfile);
1445    fdb_kvs_close(db2);
1446    fdb_close(dbfile2);
1447
1448
1449    fdb_shutdown();
1450    memleak_end();
1451    TEST_RESULT("compact overwrite");
1452}
1453
1454void *db_compact_during_doc_delete(void *args)
1455{
1456
1457    TEST_INIT();
1458    memleak_start();
1459
1460    int i, r;
1461    int n = 100;
1462    fdb_file_handle *dbfile;
1463    fdb_kvs_handle *db;
1464    fdb_status status;
1465    fdb_config fconfig;
1466    fdb_kvs_config kvs_config;
1467    fdb_kvs_info kvs_info;
1468    thread_t tid;
1469    void *thread_ret;
1470    fdb_doc **doc = alca(fdb_doc*, n);
1471    char keybuf[256], metabuf[256], bodybuf[256];
1472
1473    // init dbfile
1474    kvs_config = fdb_get_default_kvs_config();
1475    fconfig = fdb_get_default_config();
1476    fconfig.buffercache_size = 0;
1477    fconfig.wal_threshold = 1024;
1478    fconfig.compaction_threshold = 0;
1479
1480    if (args == NULL)
1481    { // parent
1482
1483        r = system(SHELL_DEL" compact_test* > errorlog.txt");
1484        (void)r;
1485
1486        status = fdb_open(&dbfile, "./compact_test1", &fconfig);
1487        TEST_CHK(status == FDB_RESULT_SUCCESS);
1488        status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
1489        TEST_CHK(status == FDB_RESULT_SUCCESS);
1490
1491        // insert documents
1492        for (i=0;i<n;++i){
1493            sprintf(keybuf, "key%d", i);
1494            sprintf(metabuf, "meta%d", i);
1495            sprintf(bodybuf, "body%d", i);
1496            fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1497                (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1498            fdb_set(db, doc[i]);
1499        }
1500
1501        fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1502        // verify no docs remaining
1503        fdb_get_kvs_info(db, &kvs_info);
1504        TEST_CHK(kvs_info.doc_count == (uint64_t)n);
1505
1506        // start deleting docs
1507        for (i=0;i<n;++i){
1508            fdb_del(db, doc[i]);
1509            if (i == n/2){
1510                // compact half-way
1511                thread_create(&tid, db_compact_during_doc_delete, (void *)dbfile);
1512            }
1513        }
1514
1515        // join compactor
1516        thread_join(tid, &thread_ret);
1517
1518        fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1519
1520        // verify no docs remaining
1521        fdb_get_kvs_info(db, &kvs_info);
1522        TEST_CHK(kvs_info.doc_count == 0);
1523
1524        // reopen
1525        fdb_kvs_close(db);
1526        fdb_close(dbfile);
1527        status = fdb_open(&dbfile, "./compact_test1", &fconfig);
1528        TEST_CHK(status == FDB_RESULT_SUCCESS);
1529        status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
1530        TEST_CHK(status == FDB_RESULT_SUCCESS);
1531
1532        fdb_get_kvs_info(db, &kvs_info);
1533        TEST_CHK(kvs_info.doc_count == 0);
1534
1535        // cleanup
1536        for (i=0;i<n;++i){
1537            fdb_doc_free(doc[i]);
1538        }
1539        fdb_kvs_close(db);
1540        fdb_close(dbfile);
1541        fdb_shutdown();
1542
1543        memleak_end();
1544        TEST_RESULT("multi thread client shutdown");
1545        return NULL;
1546    }
1547
1548    // compaction thread enters here //
1549    status = fdb_open(&dbfile, "./compact_test1", &fconfig);
1550    TEST_CHK(status == FDB_RESULT_SUCCESS);
1551    status = fdb_compact(dbfile, NULL);
1552    TEST_CHK(status == FDB_RESULT_SUCCESS);
1553    fdb_close(dbfile);
1554
1555    // shutdown
1556    thread_exit(0);
1557    return NULL;
1558}
1559
1560void compaction_daemon_test(size_t time_sec)
1561{
1562    TEST_INIT();
1563
1564    memleak_start();
1565
1566    int i, r;
1567    int n = 10000;
1568    int compaction_threshold = 30;
1569    int escape = 0;
1570    fdb_file_handle *dbfile, *dbfile_less, *dbfile_non, *dbfile_manual, *dbfile_new;
1571    fdb_kvs_handle *db, *db_less, *db_non, *db_manual;
1572    fdb_kvs_handle *snapshot;
1573    fdb_doc **doc = alca(fdb_doc*, n);
1574    fdb_doc *rdoc;
1575    fdb_file_info info;
1576    fdb_status status;
1577    struct timeval ts_begin, ts_cur, ts_gap;
1578
1579    char keybuf[256], metabuf[256], bodybuf[256];
1580
1581    // remove previous compact_test files
1582    r = system(SHELL_DEL" compact_test* > errorlog.txt");
1583    (void)r;
1584
1585    fdb_config fconfig = fdb_get_default_config();
1586    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1587    fconfig.buffercache_size = 0;
1588    fconfig.wal_threshold = 1024;
1589    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1590    fconfig.compaction_mode = FDB_COMPACTION_AUTO;
1591    fconfig.compaction_threshold = compaction_threshold;
1592    fconfig.compactor_sleep_duration = 1; // for quick test
1593
1594    fconfig.num_compactor_threads = 0;
1595    status = fdb_open(&dbfile, "compact_test", &fconfig);
1596    TEST_CHK(status == FDB_RESULT_INVALID_CONFIG);
1597
1598    fconfig.num_compactor_threads = DEFAULT_NUM_COMPACTOR_THREADS;
1599    // open db
1600    fdb_open(&dbfile, "compact_test", &fconfig);
1601    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1602    status = fdb_set_log_callback(db, logCallbackFunc,
1603                                  (void *) "compaction_daemon_test");
1604    TEST_CHK(status == FDB_RESULT_SUCCESS);
1605    // insert documents
1606    printf("Initialize..\n");
1607    for (i=0;i<n;++i){
1608        sprintf(keybuf, "key%04d", i);
1609        sprintf(metabuf, "meta%04d", i);
1610        sprintf(bodybuf, "body%04d", i);
1611        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf)+1,
1612            (void*)metabuf, strlen(metabuf)+1, (void*)bodybuf, strlen(bodybuf)+1);
1613        fdb_set(db, doc[i]);
1614    }
1615    // commit
1616    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1617    // close db file
1618    fdb_close(dbfile);
1619
1620    // ---- basic retrieve test ------------------------
1621    // reopen db file
1622    status = fdb_open(&dbfile, "compact_test", &fconfig);
1623    TEST_CHK(status == FDB_RESULT_SUCCESS);
1624    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
1625    TEST_CHK(status == FDB_RESULT_SUCCESS);
1626    status = fdb_set_log_callback(db, logCallbackFunc,
1627                                  (void *) "compaction_daemon_test");
1628    TEST_CHK(status == FDB_RESULT_SUCCESS);
1629    // check db filename
1630    fdb_get_file_info(dbfile, &info);
1631    TEST_CHK(!strcmp(info.filename, "compact_test"));
1632
1633    // retrieve documents
1634    for (i=0;i<n;++i){
1635        sprintf(keybuf, "key%04d", i);
1636        fdb_doc_create(&rdoc, (void*)keybuf, strlen(keybuf)+1,
1637            NULL, 0, NULL, 0);
1638        status = fdb_get(db, rdoc);
1639        //printf("%s %s\n", rdoc->key, rdoc->body);
1640        TEST_CHK(status == FDB_RESULT_SUCCESS);
1641        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
1642        fdb_doc_free(rdoc);
1643    }
1644
1645    // create a snapshot
1646    status = fdb_snapshot_open(db, &snapshot, n);
1647    TEST_CHK(status == FDB_RESULT_SUCCESS);
1648    // close snapshot
1649    fdb_kvs_close(snapshot);
1650
1651    // close db file
1652    fdb_close(dbfile);
1653
1654    // ---- handling when metafile is removed ------------
1655    // remove meta file
1656    r = system(SHELL_DEL" compact_test.meta > errorlog.txt");
1657    (void)r;
1658    // reopen db file
1659    status = fdb_open(&dbfile, "compact_test", &fconfig);
1660    TEST_CHK(status == FDB_RESULT_SUCCESS);
1661    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
1662    TEST_CHK(status == FDB_RESULT_SUCCESS);
1663    status = fdb_set_log_callback(db, logCallbackFunc,
1664                                  (void *) "compaction_daemon_test");
1665    TEST_CHK(status == FDB_RESULT_SUCCESS);
1666    // retrieve documents
1667    for (i=0;i<n;++i){
1668        sprintf(keybuf, "key%04d", i);
1669        fdb_doc_create(&rdoc, (void*)keybuf, strlen(keybuf)+1,
1670            NULL, 0, NULL, 0);
1671        status = fdb_get(db, rdoc);
1672        //printf("%s %s\n", rdoc->key, rdoc->body);
1673        TEST_CHK(status == FDB_RESULT_SUCCESS);
1674        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
1675        fdb_doc_free(rdoc);
1676    }
1677    // close db file
1678    fdb_close(dbfile);
1679
1680    // ---- handling when metafile points to non-exist file ------------
1681    // remove meta file
1682    r = system(SHELL_MOVE" compact_test.0 compact_test.23 > errorlog.txt");
1683    (void)r;
1684    // reopen db file
1685    status = fdb_open(&dbfile, "compact_test", &fconfig);
1686    TEST_CHK(status == FDB_RESULT_SUCCESS);
1687    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
1688    TEST_CHK(status == FDB_RESULT_SUCCESS);
1689    status = fdb_set_log_callback(db, logCallbackFunc,
1690                                  (void *) "compaction_daemon_test");
1691    TEST_CHK(status == FDB_RESULT_SUCCESS);
1692    // retrieve documents
1693    for (i=0;i<n;++i){
1694        sprintf(keybuf, "key%04d", i);
1695        fdb_doc_create(&rdoc, (void*)keybuf, strlen(keybuf)+1,
1696            NULL, 0, NULL, 0);
1697        status = fdb_get(db, rdoc);
1698        //printf("%s %s\n", rdoc->key, rdoc->body);
1699        TEST_CHK(status == FDB_RESULT_SUCCESS);
1700        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
1701        fdb_doc_free(rdoc);
1702    }
1703    // close db file
1704    fdb_close(dbfile);
1705
1706    // ---- compaction daemon test -------------------
1707    // db: DB instance to be compacted
1708    // db_less: DB instance to be compacted but with much lower update throughput
1709    // db_non: DB instance not to be compacted (auto compaction with threshold = 0)
1710    // db_manual: DB instance not to be compacted (manual compaction)
1711
1712    // open & create db_less, db_non and db_manual
1713    status = fdb_open(&dbfile_less, "compact_test_less", &fconfig);
1714    TEST_CHK(status == FDB_RESULT_SUCCESS);
1715    status = fdb_kvs_open_default(dbfile_less, &db_less, &kvs_config);
1716    TEST_CHK(status == FDB_RESULT_SUCCESS);
1717
1718    fconfig.compaction_threshold = 0;
1719    status = fdb_open(&dbfile_non, "compact_test_non", &fconfig);
1720    TEST_CHK(status == FDB_RESULT_SUCCESS);
1721    status = fdb_kvs_open_default(dbfile_non, &db_non, &kvs_config);
1722    TEST_CHK(status == FDB_RESULT_SUCCESS);
1723
1724    fconfig.compaction_mode = FDB_COMPACTION_MANUAL;
1725    status = fdb_open(&dbfile_manual, "compact_test_manual", &fconfig);
1726    TEST_CHK(status == FDB_RESULT_SUCCESS);
1727    status = fdb_kvs_open_default(dbfile_manual, &db_manual, &kvs_config);
1728    TEST_CHK(status == FDB_RESULT_SUCCESS);
1729
1730    // reopen db file
1731    fconfig.compaction_threshold = 30;
1732    fconfig.compaction_mode = FDB_COMPACTION_AUTO;
1733    status = fdb_open(&dbfile, "compact_test", &fconfig);
1734    TEST_CHK(status == FDB_RESULT_SUCCESS);
1735    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
1736    TEST_CHK(status == FDB_RESULT_SUCCESS);
1737    status = fdb_set_log_callback(db, logCallbackFunc,
1738                                  (void *) "compaction_daemon_test");
1739    TEST_CHK(status == FDB_RESULT_SUCCESS);
1740    // continuously update documents
1741    printf("wait for %d seconds..\n", (int)time_sec);
1742    gettimeofday(&ts_begin, NULL);
1743    while (!escape) {
1744        for (i=0;i<n;++i){
1745            // update db
1746            fdb_set(db, doc[i]);
1747            fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1748
1749            // update db_less (1/100 throughput)
1750            if (i%100 == 0){
1751                fdb_set(db_less, doc[i]);
1752                fdb_commit(dbfile_less, FDB_COMMIT_NORMAL);
1753            }
1754
1755            // update db_non
1756            fdb_set(db_non, doc[i]);
1757            fdb_commit(dbfile_non, FDB_COMMIT_NORMAL);
1758
1759            // update db_manual
1760            fdb_set(db_manual, doc[i]);
1761            fdb_commit(dbfile_manual, FDB_COMMIT_NORMAL);
1762
1763            gettimeofday(&ts_cur, NULL);
1764            ts_gap = _utime_gap(ts_begin, ts_cur);
1765            if ((size_t)ts_gap.tv_sec >= time_sec) {
1766                escape = 1;
1767                break;
1768            }
1769        }
1770    }
1771
1772    // Change the compaction interval to 60 secs.
1773    status = fdb_set_daemon_compaction_interval(dbfile, 60);
1774    TEST_CHK(status == FDB_RESULT_SUCCESS);
1775    // Change the compaction interval back to 1 sec.
1776    status = fdb_set_daemon_compaction_interval(dbfile, 1);
1777    TEST_CHK(status == FDB_RESULT_SUCCESS);
1778
1779    // perform manual compaction of auto-compact file
1780    status = fdb_compact(dbfile_non, NULL);
1781    TEST_CHK(status == FDB_RESULT_SUCCESS);
1782
1783    // perform manual compaction of manual-compact file
1784    status = fdb_compact(dbfile_manual, "compact_test_manual_compacted");
1785    TEST_CHK(status == FDB_RESULT_SUCCESS);
1786
1787    // open compact_test_manual_compacted using new db handle
1788    fconfig.compaction_mode = FDB_COMPACTION_MANUAL;
1789    status = fdb_open(&dbfile_new, "compact_test_manual_compacted", &fconfig);
1790    TEST_CHK(status == FDB_RESULT_SUCCESS);
1791
1792    // try to switch compaction mode
1793    status = fdb_switch_compaction_mode(dbfile_manual, FDB_COMPACTION_AUTO, 30);
1794    TEST_CHK(status == FDB_RESULT_FILE_IS_BUSY);
1795
1796    // close db_new
1797    status = fdb_close(dbfile_new);
1798    TEST_CHK(status == FDB_RESULT_SUCCESS);
1799
1800    // switch compaction mode of 'db_manual' from MANUAL to AUTO
1801    status = fdb_switch_compaction_mode(dbfile_manual, FDB_COMPACTION_AUTO, 10);
1802    TEST_CHK(status == FDB_RESULT_SUCCESS);
1803
1804    // change compaction value
1805    status = fdb_switch_compaction_mode(dbfile_manual, FDB_COMPACTION_AUTO, 30);
1806    TEST_CHK(status == FDB_RESULT_SUCCESS);
1807
1808    // close and open with auto-compact option
1809    status = fdb_close(dbfile_manual);
1810    TEST_CHK(status == FDB_RESULT_SUCCESS);
1811    fconfig.compaction_mode = FDB_COMPACTION_AUTO;
1812    status = fdb_open(&dbfile_manual, "compact_test_manual_compacted", &fconfig);
1813    TEST_CHK(status == FDB_RESULT_SUCCESS);
1814
1815    // switch compaction mode of 'db_non' from AUTO to MANUAL
1816    status = fdb_switch_compaction_mode(dbfile_non, FDB_COMPACTION_MANUAL, 0);
1817    TEST_CHK(status == FDB_RESULT_SUCCESS);
1818
1819    // close and open with manual-compact option
1820    status = fdb_close(dbfile_non);
1821    TEST_CHK(status == FDB_RESULT_SUCCESS);
1822    fconfig.compaction_mode = FDB_COMPACTION_MANUAL;
1823    status = fdb_open(&dbfile_non, "compact_test_non", &fconfig);
1824    TEST_CHK(status == FDB_RESULT_SUCCESS);
1825
1826    // Now perform one manual compaction on compact_test_non
1827    fdb_compact(dbfile_non, "compact_test_non.manual");
1828
1829    // close all db files except compact_test_non
1830    fdb_close(dbfile);
1831    fdb_close(dbfile_less);
1832    fdb_close(dbfile_manual);
1833
1834    // open manual compact file (compact_test_non) using auto compact mode
1835    fconfig.compaction_mode = FDB_COMPACTION_AUTO;
1836    status = fdb_open(&dbfile, "compact_test_non.manual", &fconfig);
1837    TEST_CHK(status == FDB_RESULT_INVALID_COMPACTION_MODE);
1838
1839    // Attempt to destroy manual compact file using auto compact mode
1840    status = fdb_destroy("compact_test_non.manual", &fconfig);
1841    TEST_CHK(status == FDB_RESULT_INVALID_COMPACTION_MODE);
1842
1843    // open auto copmact file (compact_test_manual_compacted) using manual compact mode
1844    fconfig.compaction_mode = FDB_COMPACTION_MANUAL;
1845    status = fdb_open(&dbfile, "compact_test_manual_compacted", &fconfig);
1846    TEST_CHK(status == FDB_RESULT_INVALID_COMPACTION_MODE);
1847
1848    // Attempt to destroy auto copmact file using manual compact mode
1849    status = fdb_destroy("compact_test_manual_compacted", &fconfig);
1850    TEST_CHK(status == FDB_RESULT_INVALID_COMPACTION_MODE);
1851
1852    // DESTROY auto copmact file with correct mode
1853    fconfig.compaction_mode = FDB_COMPACTION_AUTO;
1854    status = fdb_destroy("compact_test_manual_compacted", &fconfig);
1855    TEST_CHK(status == FDB_RESULT_SUCCESS);
1856
1857    // DESTROY manual compacted file with past version open!
1858    fconfig.compaction_mode = FDB_COMPACTION_MANUAL;
1859    status = fdb_destroy("compact_test_non.manual", &fconfig);
1860    TEST_CHK(status == FDB_RESULT_FILE_IS_BUSY);
1861    fdb_close(dbfile_non);
1862
1863    // Simulate a database crash by doing a premature shutdown
1864    // Note that db_non was never closed properly
1865    status = fdb_shutdown();
1866    TEST_CHK(status == FDB_RESULT_SUCCESS);
1867
1868    status = fdb_destroy("compact_test_non.manual", &fconfig);
1869    TEST_CHK(status == FDB_RESULT_SUCCESS);
1870
1871    // Attempt to read-only auto compacted and destroyed file
1872    fconfig.flags = FDB_OPEN_FLAG_RDONLY;
1873    status = fdb_open(&dbfile, "./compact_test_manual_compacted", &fconfig);
1874    TEST_CHK(status == FDB_RESULT_NO_SUCH_FILE);
1875
1876    status = fdb_open(&dbfile, "./compact_test_manual_compacted.meta", &fconfig);
1877    TEST_CHK(status == FDB_RESULT_NO_SUCH_FILE);
1878
1879    // Attempt to read-only past version of manually compacted destroyed file
1880    status = fdb_open(&dbfile, "compact_test_non", &fconfig);
1881    TEST_CHK(status == FDB_RESULT_NO_SUCH_FILE);
1882
1883    // Attempt to read-only current version of manually compacted destroyed file
1884    status = fdb_open(&dbfile, "compact_test_non.manual", &fconfig);
1885    TEST_CHK(status == FDB_RESULT_NO_SUCH_FILE);
1886
1887    // free all documents
1888    for (i=0;i<n;++i){
1889        fdb_doc_free(doc[i]);
1890    }
1891
1892    // free all resources
1893    fdb_shutdown();
1894
1895    memleak_end();
1896
1897    TEST_RESULT("compaction daemon test");
1898}
1899
1900// MB-13117
1901void auto_compaction_with_concurrent_insert_test(size_t t_limit)
1902{
1903    TEST_INIT();
1904
1905    memleak_start();
1906
1907    int i, r;
1908    fdb_file_handle *file;
1909    fdb_kvs_handle *kvs;
1910    fdb_status status;
1911    fdb_config config;
1912    fdb_kvs_config kvs_config;
1913    struct timeval ts_begin, ts_cur, ts_gap;
1914
1915    r = system(SHELL_DEL" compact_test* > errorlog.txt");
1916    (void)r;
1917
1918    // Open Database File
1919    config = fdb_get_default_config();
1920    config.compaction_mode=FDB_COMPACTION_AUTO;
1921    config.compactor_sleep_duration = 1;
1922    status = fdb_open(&file, "compact_test", &config);
1923    TEST_CHK(status == FDB_RESULT_SUCCESS);
1924
1925    // Open KV Store
1926    kvs_config = fdb_get_default_kvs_config();
1927    status = fdb_kvs_open_default(file, &kvs, &kvs_config);
1928    TEST_CHK(status == FDB_RESULT_SUCCESS);
1929
1930    gettimeofday(&ts_begin, NULL);
1931    printf("wait for %d seconds..\n", (int)t_limit);
1932
1933    // Several kv pairs
1934    for(i=0;i<100000;i++) {
1935        char str[15];
1936        sprintf(str, "%d", i);
1937        status = fdb_set_kv(kvs, str, strlen(str), (void*)"value", 5);
1938        TEST_CHK(status == FDB_RESULT_SUCCESS);
1939
1940        // Commit
1941        status = fdb_commit(file, FDB_COMMIT_NORMAL);
1942        TEST_CHK(status == FDB_RESULT_SUCCESS);
1943
1944        gettimeofday(&ts_cur, NULL);
1945        ts_gap = _utime_gap(ts_begin, ts_cur);
1946        if ((size_t)ts_gap.tv_sec >= t_limit) {
1947            break;
1948        }
1949    }
1950
1951    status = fdb_close(file);
1952    TEST_CHK(status == FDB_RESULT_SUCCESS);
1953    status = fdb_shutdown();
1954    TEST_CHK(status == FDB_RESULT_SUCCESS);
1955
1956    memleak_end();
1957
1958    TEST_RESULT("auto compaction with concurrent insert test");
1959}
1960
1961// lexicographically compares two variable-length binary streams
1962#define MIN(a,b) (((a)<(b))?(a):(b))
1963static int _compact_test_keycmp(void *key1, size_t keylen1,
1964                                void *key2, size_t keylen2)
1965{
1966    if (keylen1 == keylen2) {
1967        return memcmp(key1, key2, keylen1);
1968    }else {
1969        size_t len = MIN(keylen1, keylen2);
1970        int cmp = memcmp(key1, key2, len);
1971        if (cmp != 0) return cmp;
1972        else {
1973            return (int)((int)keylen1 - (int)keylen2);
1974        }
1975    }
1976}
1977
1978void auto_compaction_with_custom_cmp_function()
1979{
1980    TEST_INIT();
1981
1982    memleak_start();
1983
1984    int i, r, n=10000;
1985    char keybuf[256], bodybuf[256];
1986    uint64_t max_filesize = 0;
1987    fdb_file_handle *file;
1988    fdb_kvs_handle *db1, *db2, *db3;
1989    fdb_status status;
1990    fdb_config config;
1991    fdb_kvs_config kvs_config;
1992    fdb_file_info file_info;
1993    char *kvs_names[] = {NULL};
1994    fdb_custom_cmp_variable functions[] = {_compact_test_keycmp};
1995
1996    r = system(SHELL_DEL" compact_test* > errorlog.txt");
1997    (void)r;
1998
1999    // Open Database File
2000    config = fdb_get_default_config();
2001    config.wal_threshold = 4096; // reset WAL threshold for correct file size estimation
2002    config.compaction_mode=FDB_COMPACTION_AUTO;
2003    config.compactor_sleep_duration = 1;
2004    config.compaction_threshold = 10;
2005    status = fdb_open_custom_cmp(&file, "compact_test", &config,
2006                                 1, kvs_names, functions);
2007    TEST_CHK(status == FDB_RESULT_SUCCESS);
2008
2009    // Open 2 KV Stores
2010    kvs_config = fdb_get_default_kvs_config();
2011    status = fdb_kvs_open_default(file, &db1, &kvs_config);
2012    TEST_CHK(status == FDB_RESULT_SUCCESS);
2013    status = fdb_kvs_open(file, &db2, "db", &kvs_config);
2014    TEST_CHK(status == FDB_RESULT_SUCCESS);
2015
2016    // initial load
2017    for(i=0;i<n;i++) {
2018        sprintf(keybuf, "key%06d", i);
2019        sprintf(bodybuf, "body%06d", i);
2020        status = fdb_set_kv(db1, keybuf, strlen(keybuf),
2021                                 bodybuf, strlen(bodybuf));
2022        TEST_CHK(status == FDB_RESULT_SUCCESS);
2023        status = fdb_set_kv(db2, keybuf, strlen(keybuf),
2024                                 bodybuf, strlen(bodybuf));
2025        TEST_CHK(status == FDB_RESULT_SUCCESS);
2026    }
2027
2028    // create one more KVS using custom cmp
2029    // it doesn't exist on the initial cmp_func list
2030    kvs_config.custom_cmp = _compact_test_keycmp;
2031    status = fdb_kvs_open(file, &db3, "db_custom", &kvs_config);
2032    TEST_CHK(status == FDB_RESULT_SUCCESS);
2033    i = 0;
2034    sprintf(keybuf, "key%06d", i);
2035    sprintf(bodybuf, "body%06d", i);
2036    status = fdb_set_kv(db3, keybuf, strlen(keybuf),
2037                             bodybuf, strlen(bodybuf));
2038    TEST_CHK(status == FDB_RESULT_SUCCESS);
2039
2040    // Commit
2041    status = fdb_commit(file, FDB_COMMIT_NORMAL);
2042    TEST_CHK(status == FDB_RESULT_SUCCESS);
2043
2044    // update to trigger compaction
2045    for(i=0;i<n;i++) {
2046        sprintf(keybuf, "key%06d", i);
2047        sprintf(bodybuf, "body%06d", i);
2048        status = fdb_set_kv(db1, keybuf, strlen(keybuf),
2049                                 bodybuf, strlen(bodybuf));
2050        TEST_CHK(status == FDB_RESULT_SUCCESS);
2051
2052        status = fdb_get_file_info(file, &file_info);
2053        TEST_CHK(status == FDB_RESULT_SUCCESS);
2054        if (file_info.file_size > max_filesize) {
2055            max_filesize = file_info.file_size;
2056        }
2057    }
2058    // Commit
2059    status = fdb_commit(file, FDB_COMMIT_NORMAL);
2060    TEST_CHK(status == FDB_RESULT_SUCCESS);
2061
2062    status = fdb_get_file_info(file, &file_info);
2063    TEST_CHK(status == FDB_RESULT_SUCCESS);
2064    if (file_info.file_size > max_filesize) {
2065        max_filesize = file_info.file_size;
2066    }
2067
2068    printf("wait for daemon compaction completion... (max file size: %" _F64 ")\n", max_filesize);
2069    while (true) {
2070        sleep(1);
2071
2072        status = fdb_get_file_info(file, &file_info);
2073        TEST_CHK(status == FDB_RESULT_SUCCESS);
2074        if (file_info.file_size < max_filesize) {
2075            break;
2076        }
2077    }
2078    // should be compacted
2079    TEST_CHK(file_info.file_size < max_filesize);
2080
2081    status = fdb_close(file);
2082    TEST_CHK(status == FDB_RESULT_SUCCESS);
2083    status = fdb_shutdown();
2084    TEST_CHK(status == FDB_RESULT_SUCCESS);
2085
2086    memleak_end();
2087
2088    TEST_RESULT("auto compaction with custom comparison function");
2089}
2090
2091struct cb_txn_args {
2092    fdb_file_handle *file;
2093    fdb_kvs_handle *handle;
2094    int ndocs;
2095    int nupdates;
2096    int done;
2097};
2098
2099static int cb_txn(fdb_file_handle *fhandle,
2100                  fdb_compaction_status status, const char *kv_name,
2101                  fdb_doc *doc, uint64_t old_offset, uint64_t new_offset,
2102                  void *ctx)
2103{
2104    struct cb_txn_args *args = (struct cb_txn_args *)ctx;
2105    (void) fhandle;
2106    (void) doc;
2107    (void) old_offset;
2108    (void) new_offset;
2109
2110    if (status == FDB_CS_END && !args->done) {
2111        int i;
2112        int n = 10;
2113        char keybuf[256], bodybuf[256];
2114        fdb_status s;
2115
2116        // begin transaction
2117        fdb_begin_transaction(args->file, FDB_ISOLATION_READ_COMMITTED);
2118        // insert new docs, but do not end transaction
2119        for (i=0;i<n/2;++i){
2120            sprintf(keybuf, "txn%04d", i);
2121            sprintf(bodybuf, "txn_body%04d", i);
2122            s = fdb_set_kv(args->handle, keybuf, strlen(keybuf)+1,
2123                                         bodybuf, strlen(bodybuf)+1);
2124            (void)s;
2125        }
2126        args->done = 1;
2127    }
2128
2129    return 0;
2130}
2131
2132void compaction_with_concurrent_transaction_test()
2133{
2134    TEST_INIT();
2135    memleak_start();
2136
2137    int i, r;
2138    int n = 10;
2139    size_t valuelen;
2140    char keybuf[256], bodybuf[256];
2141    fdb_file_handle *dbfile, *txn_file;
2142    fdb_kvs_handle *db, *txn;
2143    fdb_status s;
2144    fdb_config fconfig = fdb_get_default_config();
2145    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2146    void *value;
2147    struct cb_txn_args cb_args;
2148
2149    memset(&cb_args, 0x0, sizeof(struct cb_txn_args));
2150    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2151    fconfig.compaction_cb = cb_txn;
2152    fconfig.compaction_cb_ctx = &cb_args;
2153    fconfig.compaction_cb_mask = FDB_CS_BEGIN |
2154                                 FDB_CS_END;
2155
2156    // remove previous compact_test files
2157    r = system(SHELL_DEL" compact_test* > errorlog.txt");
2158    (void)r;
2159
2160    // open db
2161    fdb_open(&dbfile, "./compact_test1", &fconfig);
2162    fdb_kvs_open(dbfile, &db, "db", &kvs_config);
2163
2164    fdb_open(&txn_file, "./compact_test1", &fconfig);
2165    fdb_kvs_open(txn_file, &txn, "db", &kvs_config);
2166    cb_args.file = txn_file;
2167    cb_args.handle = txn;
2168
2169    // write docs & commit
2170    for (i=0;i<n;++i){
2171        sprintf(keybuf, "key%04d", i);
2172        sprintf(bodybuf, "body%04d", i);
2173        s = fdb_set_kv(db, keybuf, strlen(keybuf)+1,
2174                           bodybuf, strlen(bodybuf)+1);
2175        TEST_CHK(s == FDB_RESULT_SUCCESS);
2176    }
2177    s = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2178    TEST_CHK(s == FDB_RESULT_SUCCESS);
2179    cb_args.ndocs = n;
2180    cb_args.nupdates = 2;
2181
2182    s = fdb_compact(dbfile, "./compact_test2");
2183    TEST_CHK(s == FDB_RESULT_SUCCESS);
2184
2185    // insert new docs through transaction
2186    for (i=n/2;i<n;++i){
2187        sprintf(keybuf, "txn%04d", i);
2188        sprintf(bodybuf, "txn_body%04d", i);
2189        s = fdb_set_kv(txn, keybuf, strlen(keybuf)+1,
2190                            bodybuf, strlen(bodybuf)+1);
2191        TEST_CHK(s == FDB_RESULT_SUCCESS);
2192    }
2193    // all txn docs should be retrieved
2194    for (i=0;i<n;++i){
2195        sprintf(keybuf, "txn%04d", i);
2196        sprintf(bodybuf, "txn_body%04d", i);
2197        s = fdb_get_kv(txn, keybuf, strlen(keybuf)+1,
2198                            &value, &valuelen);
2199        TEST_CHK(s == FDB_RESULT_SUCCESS);
2200        TEST_CMP(value, bodybuf, valuelen);
2201        fdb_free_block(value);
2202    }
2203    s = fdb_end_transaction(txn_file, FDB_COMMIT_MANUAL_WAL_FLUSH);
2204    TEST_CHK(s == FDB_RESULT_SUCCESS);
2205    fdb_close(txn_file);
2206
2207    s = fdb_compact(dbfile, "./compact_test3");
2208    TEST_CHK(s == FDB_RESULT_SUCCESS);
2209
2210    fdb_close(dbfile);
2211    fdb_shutdown();
2212    memleak_end();
2213    TEST_RESULT("compaction with concurrent transaction test");
2214}
2215
2216struct cb_upt_args {
2217    fdb_file_handle *file;
2218    fdb_kvs_handle *handle;
2219    fdb_kvs_handle *handle2;
2220    int ndocs;
2221    int nupdates;
2222    int done;
2223    int nmoves;
2224};
2225
2226static fdb_compact_decision cb_upt(fdb_file_handle *fhandle,
2227                                   fdb_compaction_status status,
2228                                   const char *kv_name,
2229                                   fdb_doc *doc, uint64_t old_offset,
2230                                   uint64_t new_offset,
2231                                   void *ctx)
2232{
2233    TEST_INIT();
2234    struct cb_upt_args *args = (struct cb_upt_args *)ctx;
2235    (void) fhandle;
2236    (void) doc;
2237    (void) old_offset;
2238    (void) new_offset;
2239
2240    if (status == FDB_CS_MOVE_DOC) {
2241        int i, j;
2242        int n = 10;
2243        char keybuf[256], bodybuf[256];
2244        char keystr[] = "new%04d";
2245        char valuestr[] = "new_body%04d";
2246        fdb_status s;
2247        args->nmoves++;
2248        if (args->nmoves > n/2 && !args->done) {
2249            // verify if the key is stripped off its prefix
2250            fdb_doc tdoc;
2251            memset(&tdoc, 0, sizeof(fdb_doc));
2252            tdoc.key = doc->key;
2253            tdoc.keylen = doc->keylen;
2254            s = fdb_get(args->handle, &tdoc);
2255            TEST_CHK(s == FDB_RESULT_SUCCESS);
2256            free(tdoc.meta);
2257            free(tdoc.body);
2258            // insert new docs
2259            for (i=0;i<n/2;++i){
2260                sprintf(keybuf, keystr, i);
2261                sprintf(bodybuf, valuestr, i);
2262                s = fdb_set_kv(args->handle, keybuf, strlen(keybuf)+1,
2263                                             bodybuf, strlen(bodybuf)+1);
2264                TEST_CHK(s == FDB_RESULT_SUCCESS);
2265                s = fdb_commit(args->file, FDB_COMMIT_NORMAL);
2266                TEST_CHK(s == FDB_RESULT_SUCCESS);
2267
2268                for (j=0; j<=i; ++j){
2269                    void *v_out;
2270                    size_t vlen_out;
2271                    sprintf(keybuf, keystr, i);
2272                    sprintf(bodybuf, valuestr, i);
2273                    s = fdb_get_kv(args->handle2, keybuf, strlen(keybuf)+1,
2274                        &v_out, &vlen_out);
2275                    TEST_CHK(s == FDB_RESULT_SUCCESS);
2276                    fdb_free_block(v_out);
2277                }
2278            }
2279            args->done = 1;
2280        }
2281        if (args->nmoves > n && args->done == 1) {
2282            // the first phase is done. insert new docs.
2283            for (i=0;i<2;++i){
2284                sprintf(keybuf, "xxx%d", i);
2285                sprintf(bodybuf, "xxxvalue%d", i);
2286                s = fdb_set_kv(args->handle, keybuf, strlen(keybuf)+1,
2287                                             bodybuf, strlen(bodybuf)+1);
2288                TEST_CHK(s == FDB_RESULT_SUCCESS);
2289                s = fdb_commit(args->file, FDB_COMMIT_NORMAL);
2290                TEST_CHK(s == FDB_RESULT_SUCCESS);
2291            }
2292            args->done = 2;
2293        }
2294        if (args->nmoves == (n*3/2 + 2) && args->done == 2) {
2295            // during the second-second phase,
2296            // insert new docs, and do not commit.
2297            for (i=0;i<2;++i){
2298                sprintf(keybuf, "zzz%d", i);
2299                sprintf(bodybuf, "zzzvalue%d", i);
2300                s = fdb_set_kv(args->handle, keybuf, strlen(keybuf)+1,
2301                                             bodybuf, strlen(bodybuf)+1);
2302                TEST_CHK(s == FDB_RESULT_SUCCESS);
2303            }
2304            args->done = 3;
2305        }
2306        TEST_CMP(kv_name, "db", 2);
2307    }
2308
2309    return 0;
2310}
2311
2312void compaction_with_concurrent_update_test()
2313{
2314    TEST_INIT();
2315    memleak_start();
2316
2317    int i, r;
2318    int n = 10;
2319    char keybuf[256], bodybuf[256];
2320    fdb_file_handle *dbfile, *dbfile2, *dbfile3;
2321    fdb_kvs_handle *db, *db2, *db3;
2322    fdb_status s;
2323    fdb_config fconfig = fdb_get_default_config();
2324    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2325    fdb_kvs_info info;
2326    fdb_seqnum_t seqnum;
2327    struct cb_upt_args cb_args;
2328    void *value;
2329    size_t valuelen;
2330
2331    memset(&cb_args, 0x0, sizeof(struct cb_upt_args));
2332    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2333    fconfig.compaction_cb = cb_upt;
2334    fconfig.compaction_cb_ctx = &cb_args;
2335    fconfig.compaction_cb_mask = FDB_CS_BEGIN |
2336                                 FDB_CS_END | FDB_CS_MOVE_DOC;
2337
2338    // remove previous compact_test files
2339    r = system(SHELL_DEL" compact_test* > errorlog.txt");
2340    (void)r;
2341
2342    // open db
2343    fdb_open(&dbfile, "./compact_test1", &fconfig);
2344    fdb_kvs_open(dbfile, &db, "db", &kvs_config);
2345
2346    fdb_open(&dbfile2, "./compact_test1", &fconfig);
2347    fdb_kvs_open(dbfile2, &db2, "db", &kvs_config);
2348    fdb_open(&dbfile3, "./compact_test1", &fconfig);
2349    fdb_kvs_open(dbfile3, &db3, "db", &kvs_config);
2350
2351    cb_args.file = dbfile2;
2352    cb_args.handle = db2;
2353    cb_args.handle2 = db3;
2354
2355    // write docs & commit
2356    for (i=0;i<n;++i){
2357        sprintf(keybuf, "key%04d", i);
2358        sprintf(bodybuf, "body%04d", i);
2359        s = fdb_set_kv(db, keybuf, strlen(keybuf)+1,
2360                           bodybuf, strlen(bodybuf)+1);
2361        TEST_CHK(s == FDB_RESULT_SUCCESS);
2362    }
2363    s = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2364    TEST_CHK(s == FDB_RESULT_SUCCESS);
2365    cb_args.ndocs = n;
2366    cb_args.nupdates = 2;
2367    cb_args.nmoves = 0;
2368
2369    s = fdb_compact(dbfile, "./compact_test2");
2370    TEST_CHK(s == FDB_RESULT_SUCCESS);
2371
2372    // insert new docs after compaction
2373    for (i=n/2;i<n;++i){
2374        sprintf(keybuf, "new%04d", i);
2375        sprintf(bodybuf, "new_body%04d", i);
2376        s = fdb_set_kv(db2, keybuf, strlen(keybuf)+1,
2377                            bodybuf, strlen(bodybuf)+1);
2378        TEST_CHK(s == FDB_RESULT_SUCCESS);
2379    }
2380    // all interleaved docs should be retrieved
2381    // 1. inserted during the first phase of the compaction
2382    for (i=0;i<n;++i){
2383        sprintf(keybuf, "new%04d", i);
2384        sprintf(bodybuf, "new_body%04d", i);
2385        s = fdb_get_kv(db2, keybuf, strlen(keybuf)+1,
2386                            &value, &valuelen);
2387        TEST_CHK(s == FDB_RESULT_SUCCESS);
2388        TEST_CMP(value, bodybuf, valuelen);
2389        fdb_free_block(value);
2390    }
2391    // 2. inserted during the second phase of the compaction
2392    for (i=0; i<2; ++i) {
2393        sprintf(keybuf, "xxx%d", i);
2394        sprintf(bodybuf, "xxxvalue%d", i);
2395        s = fdb_get_kv(db2, keybuf, strlen(keybuf)+1,
2396                            &value, &valuelen);
2397        TEST_CHK(s == FDB_RESULT_SUCCESS);
2398        TEST_CMP(value, bodybuf, valuelen);
2399        fdb_free_block(value);
2400
2401        sprintf(keybuf, "zzz%d", i);
2402        sprintf(bodybuf, "zzzvalue%d", i);
2403        s = fdb_get_kv(db2, keybuf, strlen(keybuf)+1,
2404                            &value, &valuelen);
2405        TEST_CHK(s == FDB_RESULT_SUCCESS);
2406        TEST_CMP(value, bodybuf, valuelen);
2407        fdb_free_block(value);
2408    }
2409
2410    fdb_close(dbfile2);
2411    fdb_close(dbfile3);
2412
2413    s = fdb_get_kvs_info(db, &info);
2414    TEST_CHK(s == FDB_RESULT_SUCCESS);
2415    seqnum = info.last_seqnum;
2416
2417    cb_args.nmoves = 0;
2418    s = fdb_compact(dbfile, "./compact_test3");
2419    TEST_CHK(s == FDB_RESULT_SUCCESS);
2420
2421    s = fdb_get_kvs_info(db, &info);
2422    TEST_CHK(s == FDB_RESULT_SUCCESS);
2423    // No docs are inserted during the compaction.
2424    // Seqnum should be same.
2425    TEST_CHK(info.last_seqnum == seqnum);
2426
2427    // insert the last document
2428    sprintf(keybuf, "last");
2429    sprintf(bodybuf, "last_value");
2430    s = fdb_set_kv(db, keybuf, strlen(keybuf)+1, bodybuf, strlen(bodybuf)+1);
2431    TEST_CHK(s == FDB_RESULT_SUCCESS);
2432    s = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2433    TEST_CHK(s == FDB_RESULT_SUCCESS);
2434
2435    fdb_close(dbfile);
2436    fdb_shutdown();
2437    memleak_end();
2438    TEST_RESULT("compaction with concurrent update test");
2439}
2440
2441static int compaction_del_cb(fdb_file_handle *fhandle,
2442                             fdb_compaction_status status,
2443                             const char *kv_name,
2444                             fdb_doc *doc, uint64_t old_offset,
2445                             uint64_t new_offset,
2446                             void *ctx)
2447{
2448    TEST_INIT();
2449    fdb_status s;
2450    fdb_file_handle *dbfile;
2451    fdb_kvs_handle *db = *(fdb_kvs_handle **)ctx;
2452    int i;
2453    int n = 10; // if changing this change caller function too
2454    char keybuf[256];
2455    (void) doc;
2456    (void) new_offset;
2457    (void) old_offset;
2458
2459    if (status == FDB_CS_BEGIN) {
2460        TEST_CHK(false);
2461    } else if (status == FDB_CS_END) {
2462        TEST_CHK(!kv_name);
2463        if (db) { // At end of first phase, mutate more docs...
2464            s = fdb_open(&dbfile, "./compact_test1", &fhandle->root->config);
2465            TEST_CHK(s == FDB_RESULT_SUCCESS);
2466            fdb_kvs_open_default(dbfile, &db, &db->kvs_config);
2467            for (i = 0; i < n; ++i){
2468                sprintf(keybuf, "key%d", i);
2469                s = fdb_del_kv(db, keybuf, strlen(keybuf));
2470                TEST_CHK(s == FDB_RESULT_SUCCESS);
2471            }
2472            // Now insert and delete a bunch of keys (all in WAL)
2473            for (i = 0; i < n; ++i){
2474                sprintf(keybuf, "KEY%d", i);
2475                s = fdb_set_kv(db, keybuf, strlen(keybuf), NULL, 0);
2476                TEST_CHK(s == FDB_RESULT_SUCCESS);
2477                s = fdb_del_kv(db, keybuf, strlen(keybuf));
2478                TEST_CHK(s == FDB_RESULT_SUCCESS);
2479            }
2480            sprintf(keybuf, "key%d", i);
2481            s = fdb_set_kv(db, keybuf, strlen(keybuf), NULL, 0);
2482            TEST_CHK(s == FDB_RESULT_SUCCESS);
2483            s = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2484            TEST_CHK(s == FDB_RESULT_SUCCESS);
2485            fdb_close(dbfile);
2486            *(fdb_kvs_handle **)ctx = NULL; // Don't run in 2nd, 3rd phases
2487        }
2488    } else if (status == FDB_CS_FLUSH_WAL) {
2489        TEST_CHK(false);
2490    } else if (status == FDB_CS_MOVE_DOC) {
2491        TEST_CHK(false);
2492    } else { // FDB_CS_BATCH_MOVE
2493        TEST_CHK(false);
2494    }
2495    return 0;
2496}
2497
2498void compact_deleted_doc_test()
2499{
2500    TEST_INIT();
2501    memleak_start();
2502    int i, r;
2503    int n = 10; // if changing this change the callback too
2504    fdb_file_handle *dbfile;
2505    fdb_kvs_handle *db, *cb_db;
2506    fdb_doc *rdoc;
2507    fdb_status s;
2508    char keybuf[256];
2509    void *value;
2510    size_t valuelen;
2511
2512    // remove previous compact_test files
2513    r = system(SHELL_DEL " compact_test* > errorlog.txt");
2514    (void)r;
2515
2516    fdb_config fconfig = fdb_get_default_config();
2517    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2518    fconfig.multi_kv_instances = false;
2519    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2520
2521    fconfig.compaction_cb = compaction_del_cb;
2522    fconfig.compaction_cb_ctx = &cb_db;
2523    fconfig.compaction_cb_mask = FDB_CS_END;
2524
2525    // open db
2526    fdb_open(&dbfile, "./compact_test1", &fconfig);
2527    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2528    cb_db = db;
2529
2530    for (i=0;i<n;++i){
2531        sprintf(keybuf, "key%d", i);
2532        s = fdb_set_kv(db, keybuf, strlen(keybuf),
2533                            (void*)"value", 5);
2534        TEST_CHK(s == FDB_RESULT_SUCCESS);
2535    }
2536    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2537
2538    // At end of phase 1, all documents get deleted
2539    s = fdb_compact(dbfile, "compact_test2");
2540    TEST_CHK(s == FDB_RESULT_SUCCESS);
2541    TEST_CHK(wal_get_num_flushable(dbfile->root->file) == 0);
2542
2543    for (i=0;i<n;++i){
2544        sprintf(keybuf, "key%d", i);
2545        fdb_doc_create(&rdoc, keybuf, strlen(keybuf), NULL, 0, NULL, 0);
2546        s = fdb_get_metaonly(db, rdoc);
2547        TEST_CHK(s == FDB_RESULT_KEY_NOT_FOUND);
2548        fdb_doc_free(rdoc);
2549    }
2550
2551    // documents inserted in the middle of phase 1 should be present
2552    sprintf(keybuf, "key%d", i);
2553    s = fdb_get_kv(db, keybuf, strlen(keybuf), &value, &valuelen);
2554    TEST_CHK(s == FDB_RESULT_SUCCESS);
2555    fdb_free_block(value);
2556
2557    fdb_kvs_close(db);
2558    fdb_close(dbfile);
2559    fdb_shutdown();
2560    memleak_end();
2561    TEST_RESULT("compact deleted doc test");
2562}
2563
2564void compact_upto_twice_test()
2565{
2566    TEST_INIT();
2567    memleak_start();
2568    int i, r;
2569    fdb_file_handle *dbfile;
2570    fdb_kvs_handle *db;
2571    fdb_snapshot_info_t *markers;
2572    fdb_status status;
2573    uint64_t num_markers;
2574    char keybuf[256];
2575
2576    // remove previous compact_test files
2577    r = system(SHELL_DEL " compact_test* > errorlog.txt");
2578    (void)r;
2579
2580    fdb_config fconfig = fdb_get_default_config();
2581    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2582    fconfig.wal_threshold = 1024;
2583    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2584
2585    // open db
2586    fdb_open(&dbfile, "./compact_test1", &fconfig);
2587    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2588
2589    for (i=0;i<10;++i){
2590        sprintf(keybuf, "key%d", i);
2591        status = fdb_set_kv(db, keybuf, strlen(keybuf),
2592                            (void*)"value", 5);
2593        TEST_CHK(status == FDB_RESULT_SUCCESS);
2594        fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2595    }
2596
2597    // compact upto twice with incrementing seqnums
2598    status = fdb_get_all_snap_markers(dbfile, &markers,
2599                                      &num_markers);
2600    TEST_CHK(status == FDB_RESULT_SUCCESS);
2601    status = fdb_compact_upto(dbfile, NULL, markers[num_markers-1].marker);
2602    TEST_CHK(status == FDB_RESULT_SUCCESS);
2603    status = fdb_free_snap_markers(markers, num_markers);
2604    TEST_CHK(status == FDB_RESULT_SUCCESS);
2605
2606    // compact upto
2607    status = fdb_get_all_snap_markers(dbfile, &markers,
2608                                      &num_markers);
2609    TEST_CHK(status == FDB_RESULT_SUCCESS);
2610    status = fdb_compact_upto(dbfile, NULL, markers[num_markers-2].marker);
2611    TEST_CHK(status == FDB_RESULT_SUCCESS);
2612    status = fdb_free_snap_markers(markers, num_markers);
2613    TEST_CHK(status == FDB_RESULT_SUCCESS);
2614
2615    fdb_close(dbfile);
2616    fdb_shutdown();
2617    memleak_end();
2618    TEST_RESULT("compact upto twice");
2619}
2620
2621void wal_delete_compact_upto_test()
2622{
2623    TEST_INIT();
2624    memleak_start();
2625    int i, r;
2626    fdb_file_handle *dbfile;
2627    fdb_kvs_handle *db;
2628    fdb_snapshot_info_t *markers;
2629    fdb_status status;
2630    uint64_t num_markers;
2631    fdb_kvs_info kvs_info;
2632    char keybuf[256];
2633    void *value;
2634    size_t valuelen;
2635
2636    // remove previous compact_test files
2637    r = system(SHELL_DEL " compact_test* > errorlog.txt");
2638    (void)r;
2639
2640    fdb_config fconfig = fdb_get_default_config();
2641    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2642    fconfig.wal_threshold = 19;
2643    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2644
2645    // open db
2646    fdb_open(&dbfile, "./compact_test5", &fconfig);
2647    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2648
2649    // insert a few keys...
2650    for (i=0;i<10;++i){
2651        sprintf(keybuf, "key%d", i);
2652        status = fdb_set_kv(db, keybuf, strlen(keybuf),
2653                            (void*)"value", 5);
2654        TEST_CHK(status == FDB_RESULT_SUCCESS);
2655    }
2656
2657    status = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2658    TEST_CHK(status == FDB_RESULT_SUCCESS);
2659
2660    // insert keys such that last insert causes wal flush...
2661    for (; i<20;++i){
2662        sprintf(keybuf, "key%d", i);
2663        status = fdb_set_kv(db, keybuf, strlen(keybuf),
2664                            (void*)"value", 5);
2665        TEST_CHK(status == FDB_RESULT_SUCCESS);
2666    }
2667    // Now delete half of the newly inserted keys
2668    for (i = 15; i < 20; ++i){
2669        sprintf(keybuf, "key%d", i);
2670        status = fdb_del_kv(db, keybuf, strlen(keybuf));
2671        TEST_CHK(status == FDB_RESULT_SUCCESS);
2672    }
2673    // Now deleted items are in the unflushed wal..
2674    status = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2675    TEST_CHK(status == FDB_RESULT_SUCCESS);
2676
2677    // Create another header for compact_upto()
2678    sprintf(keybuf, "key%d", i);
2679    status = fdb_set_kv(db, keybuf, strlen(keybuf),
2680            (void*)"value", 5);
2681    TEST_CHK(status == FDB_RESULT_SUCCESS);
2682    status = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2683    TEST_CHK(status == FDB_RESULT_SUCCESS);
2684
2685    // Get all snap markers..
2686    status = fdb_get_all_snap_markers(dbfile, &markers,
2687                                      &num_markers);
2688    TEST_CHK(status == FDB_RESULT_SUCCESS);
2689
2690    // compact upto the delete wal's marker...
2691    status = fdb_compact_upto(dbfile, NULL, markers[1].marker);
2692    TEST_CHK(status == FDB_RESULT_SUCCESS);
2693    status = fdb_free_snap_markers(markers, num_markers);
2694    TEST_CHK(status == FDB_RESULT_SUCCESS);
2695
2696    status = fdb_get_kvs_info(db, &kvs_info);
2697    TEST_CHK(status == FDB_RESULT_SUCCESS);
2698
2699    // Deleted items should not be found..
2700    for (i = 15; i < 20; ++i){
2701        sprintf(keybuf, "key%d", i);
2702        status = fdb_get_kv(db, keybuf, strlen(keybuf), &value, &valuelen);
2703        TEST_CHK(status == FDB_RESULT_KEY_NOT_FOUND);
2704    }
2705
2706    fdb_kvs_close(db);
2707    fdb_close(dbfile);
2708    fdb_shutdown();
2709    memleak_end();
2710    TEST_RESULT("compact upto with wal deletes test");
2711}
2712
2713void compact_upto_post_snapshot_test()
2714{
2715    TEST_INIT();
2716    memleak_start();
2717    int i, r;
2718    fdb_file_handle *dbfile;
2719    fdb_kvs_handle *db, *snap_db;
2720    fdb_snapshot_info_t *markers;
2721    fdb_status status;
2722    fdb_iterator *iterator;
2723    fdb_doc *rdoc = NULL;
2724    fdb_kvs_info kvs_info;
2725    uint64_t num_markers;
2726    char keybuf[256];
2727
2728    // remove previous compact_test files
2729    r = system(SHELL_DEL " compact_test* > errorlog.txt");
2730    (void)r;
2731
2732    fdb_config fconfig = fdb_get_default_config();
2733    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2734    fconfig.wal_threshold = 1024;
2735    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2736    // prevent block reusing to keep snapshots
2737    fconfig.block_reusing_threshold = 0;
2738
2739    // open db
2740    fdb_open(&dbfile, "./compact_test1", &fconfig);
2741    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2742
2743    for (i=0;i<10;++i){
2744        sprintf(keybuf, "key%d", i);
2745        status = fdb_set_kv(db, keybuf, strlen(keybuf),
2746                            (void*)"value", 5);
2747        TEST_CHK(status == FDB_RESULT_SUCCESS);
2748        fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2749    }
2750
2751    // check  db info
2752    status = fdb_get_kvs_info(db, &kvs_info);
2753    TEST_CHK(status == FDB_RESULT_SUCCESS);
2754    TEST_CHK(kvs_info.last_seqnum == 10);
2755
2756    // open snapshot at seqnum 5
2757    status = fdb_snapshot_open(db, &snap_db, 5);
2758    TEST_CHK(status == FDB_RESULT_SUCCESS);
2759
2760    // check snap info
2761    status = fdb_get_kvs_info(snap_db, &kvs_info);
2762    TEST_CHK(status == FDB_RESULT_SUCCESS);
2763    TEST_CHK(kvs_info.last_seqnum == 5);
2764
2765
2766    // compact upto
2767    status = fdb_get_all_snap_markers(dbfile, &markers,
2768                                      &num_markers);
2769    TEST_CHK(status == FDB_RESULT_SUCCESS);
2770    status = fdb_compact_upto(dbfile, NULL, markers[5].marker);
2771    TEST_CHK(status == FDB_RESULT_SUCCESS);
2772    status = fdb_free_snap_markers(markers, num_markers);
2773    TEST_CHK(status == FDB_RESULT_SUCCESS);
2774
2775    // check db and snap info
2776    status = fdb_get_kvs_info(db, &kvs_info);
2777    TEST_CHK(status == FDB_RESULT_SUCCESS);
2778    TEST_CHK(kvs_info.last_seqnum == 10);
2779    status = fdb_get_kvs_info(snap_db, &kvs_info);
2780    TEST_CHK(status == FDB_RESULT_SUCCESS);
2781    TEST_CHK(kvs_info.last_seqnum == 5);
2782
2783
2784    // iterate over snapshot
2785    status = fdb_iterator_init(snap_db,
2786                               &iterator, NULL, 0, NULL, 0, FDB_ITR_NONE);
2787    TEST_CHK(status == FDB_RESULT_SUCCESS);
2788    i = 0;
2789    do {
2790        status = fdb_iterator_get(iterator, &rdoc);
2791        TEST_CHK(status == FDB_RESULT_SUCCESS);
2792        fdb_doc_free(rdoc);
2793        rdoc = NULL;
2794        i++;
2795
2796        // check db and snap info
2797        status = fdb_get_kvs_info(db, &kvs_info);
2798        TEST_CHK(status == FDB_RESULT_SUCCESS);
2799        TEST_CHK(kvs_info.last_seqnum == 10);
2800        status = fdb_get_kvs_info(snap_db, &kvs_info);
2801        TEST_CHK(status == FDB_RESULT_SUCCESS);
2802        TEST_CHK(kvs_info.last_seqnum == 5);
2803    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2804
2805    TEST_CHK(i == 5);
2806
2807    fdb_iterator_close(iterator);
2808    fdb_kvs_close(snap_db);
2809    fdb_close(dbfile);
2810    fdb_shutdown();
2811    memleak_end();
2812    TEST_RESULT("compact upto post snapshot test");
2813}
2814
2815void compact_upto_overwrite_test(int opt)
2816{
2817    TEST_INIT();
2818
2819    int n = 10, value_len=32;
2820    int i, r, idx, c;
2821    char cmd[256];
2822    char key[256], *value;
2823    char keystr[] = "key%06d";
2824    char valuestr[] = "value%08d";
2825    char valuestr2[] = "updated_value%08d";
2826    fdb_file_handle *db_file;
2827    fdb_kvs_handle *db, *snap;
2828    fdb_config config;
2829    fdb_kvs_config kvs_config;
2830    fdb_status s;
2831    fdb_kvs_info kvs_info;
2832    fdb_iterator *fit;
2833    fdb_doc *doc;
2834    fdb_snapshot_info_t *markers;
2835    fdb_seqnum_t seqnum;
2836    fdb_commit_opt_t commit_opt;
2837    uint64_t n_markers;
2838
2839    sprintf(cmd, SHELL_DEL " compact_test* > errorlog.txt");
2840    r = system(cmd);
2841    (void)r;
2842
2843    memleak_start();
2844
2845    value = (char*)malloc(value_len);
2846
2847    config = fdb_get_default_config();
2848    config.durability_opt = FDB_DRB_ASYNC;
2849    config.seqtree_opt = FDB_SEQTREE_USE;
2850    config.wal_flush_before_commit = true;
2851    config.multi_kv_instances = true;
2852    config.buffercache_size = 0;
2853
2854    commit_opt = (opt)?FDB_COMMIT_NORMAL:FDB_COMMIT_MANUAL_WAL_FLUSH;
2855
2856    kvs_config = fdb_get_default_kvs_config();
2857
2858    s = fdb_open(&db_file, "./compact_test", &config);
2859    TEST_CHK(s == FDB_RESULT_SUCCESS);
2860    s = fdb_kvs_open(db_file, &db, "db", &kvs_config);
2861    TEST_CHK(s == FDB_RESULT_SUCCESS);
2862
2863    // === write ===
2864    for (i=0;i<n;++i){
2865        idx = i % (n/2);
2866        sprintf(key, keystr, idx);
2867        memset(value, 'x', value_len);
2868        memcpy(value + value_len - 6, "<end>", 6);
2869        if (i < (n/2)) {
2870            sprintf(value, valuestr, idx);
2871        } else {
2872            sprintf(value, valuestr2, idx);
2873        }
2874        s = fdb_set_kv(db, key, strlen(key)+1, value, value_len);
2875        TEST_CHK(s == FDB_RESULT_SUCCESS);
2876
2877        if (opt == 2) {
2878            // HB+trie, WAL, HB+trie, WAL...
2879            commit_opt = (i%2)?FDB_COMMIT_NORMAL:FDB_COMMIT_MANUAL_WAL_FLUSH;
2880        } else if (opt == 3) {
2881            // WAL, HB+trie, WAL, HB+trie...
2882            commit_opt = (i%2)?FDB_COMMIT_MANUAL_WAL_FLUSH:FDB_COMMIT_NORMAL;
2883        }
2884
2885        s = fdb_commit(db_file, commit_opt);
2886        TEST_CHK(s == FDB_RESULT_SUCCESS);
2887        s = fdb_get_kvs_info(db, &kvs_info);
2888        TEST_CHK(s == FDB_RESULT_SUCCESS);
2889    }
2890
2891    s = fdb_get_all_snap_markers(db_file, &markers, &n_markers);
2892    TEST_CHK(s == FDB_RESULT_SUCCESS);
2893
2894    int upto = n_markers/2;
2895    s = fdb_compact_upto(db_file, "./compact_test2", markers[upto].marker);
2896    TEST_CHK(s == FDB_RESULT_SUCCESS);
2897
2898    // iterating using snapshots with various seqnums
2899    for (i=n_markers-1; i>=0; --i) {
2900        seqnum = markers[i].kvs_markers->seqnum;
2901
2902        s = fdb_snapshot_open(db, &snap, seqnum);
2903        if (i<=upto) {
2904            TEST_CHK(s == FDB_RESULT_SUCCESS);
2905        } else {
2906            // seqnum < (n/2) must fail
2907            TEST_CHK(s != FDB_RESULT_SUCCESS);
2908            continue;
2909        }
2910
2911        s = fdb_iterator_init(snap, &fit, NULL, 0, NULL, 0, 0);
2912        TEST_CHK(s == FDB_RESULT_SUCCESS);
2913        c = 0;
2914        do {
2915            doc = NULL;
2916            s = fdb_iterator_get(fit, &doc);
2917            if (s != FDB_RESULT_SUCCESS) break;
2918            idx = c;
2919            sprintf(key, keystr, idx);
2920            memset(value, 'x', value_len);
2921            memcpy(value + value_len - 6, "<end>", 6);
2922            if ((fdb_seqnum_t)c >= seqnum-(n/2)) {
2923                sprintf(value, valuestr, idx);
2924            } else {
2925                sprintf(value, valuestr2, idx);
2926            }
2927            TEST_CMP(doc->key, key, doc->keylen);
2928            TEST_CMP(doc->body, value, doc->bodylen);
2929            c++;
2930            fdb_doc_free(doc);
2931        } while(fdb_iterator_next(fit) == FDB_RESULT_SUCCESS);
2932        s = fdb_iterator_close(fit);
2933        TEST_CHK(s == FDB_RESULT_SUCCESS);
2934        TEST_CHK(c == (n/2));
2935
2936        s = fdb_kvs_close(snap);
2937        TEST_CHK(s == FDB_RESULT_SUCCESS);
2938    }
2939
2940    // iterating using the original handle
2941    s = fdb_iterator_init(db, &fit, NULL, 0, NULL, 0, 0);
2942    TEST_CHK(s == FDB_RESULT_SUCCESS);
2943    c = 0;
2944    do {
2945        doc = NULL;
2946        s = fdb_iterator_get(fit, &doc);
2947        if (s != FDB_RESULT_SUCCESS) break;
2948        idx = c;
2949        sprintf(key, keystr, idx);
2950        memset(value, 'x', value_len);
2951        memcpy(value + value_len - 6, "<end>", 6);
2952        sprintf(value, valuestr2, idx);
2953        TEST_CMP(doc->key, key, doc->keylen);
2954        TEST_CMP(doc->body, value, doc->bodylen);
2955        c++;
2956        fdb_doc_free(doc);
2957    } while(fdb_iterator_next(fit) == FDB_RESULT_SUCCESS);
2958    s = fdb_iterator_close(fit);
2959    TEST_CHK(s == FDB_RESULT_SUCCESS);
2960    TEST_CHK(c == (n/2));
2961
2962    s = fdb_free_snap_markers(markers, n_markers);
2963    TEST_CHK(s == FDB_RESULT_SUCCESS);
2964    free(value);
2965    s = fdb_close(db_file);
2966    TEST_CHK(s == FDB_RESULT_SUCCESS);
2967    s = fdb_shutdown();
2968    TEST_CHK(s == FDB_RESULT_SUCCESS);
2969
2970    memleak_end();
2971
2972    sprintf(cmd, "compact upto overwrite test");
2973    if (opt == 0) {
2974        strcat(cmd, " (HB+trie)");
2975    } else if (opt == 1) {
2976        strcat(cmd, " (WAL)");
2977    } else if (opt == 2) {
2978        strcat(cmd, " (mixed, HB+trie/WAL)");
2979    } else if (opt == 3) {
2980        strcat(cmd, " (mixed, WAL/HB+trie)");
2981    }
2982    TEST_RESULT(cmd);
2983}
2984static int compaction_cb_get(fdb_file_handle *fhandle,
2985                         fdb_compaction_status status, const char *kv_name,
2986                         fdb_doc *doc, uint64_t old_offset,
2987                         uint64_t new_offset, void *ctx)
2988{
2989    TEST_INIT();
2990    fdb_status s;
2991    struct cb_args *args = (struct cb_args *)ctx;
2992    fdb_kvs_handle *snap_db;
2993    fdb_snapshot_info_t *markers;
2994    uint64_t num_markers;
2995    fdb_seqnum_t seqno;
2996
2997    (void) doc;
2998    (void) old_offset;
2999    (void) new_offset;
3000
3001    if (status == FDB_CS_MOVE_DOC) {
3002        TEST_CHK(kv_name);
3003    } else {
3004        TEST_CHK(!kv_name);
3005    }
3006
3007    // use snap markers
3008    s = fdb_get_all_snap_markers(fhandle, &markers, &num_markers);
3009    TEST_CHK(s == FDB_RESULT_SUCCESS);
3010    seqno = markers[0].kvs_markers[0].seqnum;
3011
3012    // snapshot open
3013    s = fdb_snapshot_open(args->handle, &snap_db, seqno);
3014    TEST_CHK(s==FDB_RESULT_SUCCESS);
3015    s = fdb_kvs_close(snap_db);
3016    TEST_CHK(s==FDB_RESULT_SUCCESS);
3017    return 0;
3018}
3019
3020void compact_with_snapshot_open_test()
3021{
3022  TEST_INIT();
3023  memleak_start();
3024
3025  int i, r;
3026  int n = 100000;
3027  char keybuf[256], bodybuf[256];
3028  fdb_file_handle *dbfile;
3029  fdb_kvs_handle *db, *db2, *snap_db;
3030  fdb_status s;
3031  fdb_config fconfig = fdb_get_default_config();
3032  fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
3033  struct cb_args cb_args;
3034
3035  memset(&cb_args, 0x0, sizeof(struct cb_args));
3036  fconfig.wal_threshold = 1024;
3037  fconfig.flags = FDB_OPEN_FLAG_CREATE;
3038  fconfig.compaction_cb = compaction_cb_get;
3039  fconfig.compaction_cb_ctx = &cb_args;
3040  fconfig.compaction_cb_mask = FDB_CS_BEGIN |
3041                               FDB_CS_MOVE_DOC |
3042                               FDB_CS_FLUSH_WAL |
3043                               FDB_CS_END;
3044  // remove previous compact_test files
3045  r = system(SHELL_DEL" compact_test* > errorlog.txt");
3046  (void)r;
3047
3048  // open two handles for kvs
3049  fdb_open(&dbfile, "./compact_test1", &fconfig);
3050  fdb_kvs_open(dbfile, &db, "db", &kvs_config);
3051  fdb_kvs_open(dbfile, &db2, "db", &kvs_config);
3052  for (i=0;i<n;++i){
3053      sprintf(keybuf, "key%04d", i);
3054      sprintf(bodybuf, "body%04d", i);
3055      s = fdb_set_kv(db, keybuf, strlen(keybuf), bodybuf, strlen(bodybuf));
3056      TEST_CHK(s == FDB_RESULT_SUCCESS);
3057  }
3058
3059
3060  // commit
3061  s = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
3062  TEST_CHK(s == FDB_RESULT_SUCCESS);
3063
3064  // point compact callback handle to db2
3065  cb_args.handle = db2;
3066
3067  // compact
3068  s = fdb_compact(dbfile, NULL);
3069  TEST_CHK(s == FDB_RESULT_SUCCESS);
3070
3071  // open compaction end
3072  s = fdb_snapshot_open(db2, &snap_db, n);
3073  TEST_CHK(s == FDB_RESULT_SUCCESS);
3074
3075  fdb_kvs_close(snap_db);
3076  s = fdb_close(dbfile);
3077  TEST_CHK(s == FDB_RESULT_SUCCESS);
3078  s = fdb_shutdown();
3079  TEST_CHK(s == FDB_RESULT_SUCCESS);
3080  TEST_RESULT("compact with snapshot_open test");
3081}
3082
3083static int compaction_cb_markers(fdb_file_handle *fhandle,
3084                         fdb_compaction_status status, const char *kv_name,
3085                         fdb_doc *doc, uint64_t old_offset,
3086                         uint64_t new_offset, void *ctx)
3087{
3088    TEST_INIT();
3089    uint64_t i, j;
3090    fdb_status s;
3091    fdb_kvs_handle *db, *snap_db;
3092    fdb_snapshot_info_t *markers;
3093    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
3094    fdb_seqnum_t seqno;
3095    uint64_t num_markers;
3096    struct cb_args *args = (struct cb_args *)ctx;
3097    uint64_t n = (uint64_t) args->n_moved_docs;
3098
3099    (void) status;
3100    (void) doc;
3101    (void) old_offset;
3102    (void) new_offset;
3103
3104    // get snap markers
3105    s = fdb_get_all_snap_markers(fhandle, &markers, &num_markers);
3106    TEST_CHK(s == FDB_RESULT_SUCCESS);
3107    TEST_CHK(markers[0].num_kvs_markers == 5);
3108    seqno = markers[0].kvs_markers[0].seqnum;
3109    TEST_CHK(seqno == n);
3110
3111    // snapshot open for each kvs for each marker
3112    for(j=0;j<num_markers;++j){
3113        for (i=0;i<(uint64_t)markers[j].num_kvs_markers;++i){
3114            // open kv for this marker
3115            fdb_kvs_open(fhandle, &db,
3116                         markers[j].kvs_markers[i].kv_store_name, &kvs_config);
3117            // snapshot the kv
3118            s = fdb_snapshot_open(db, &snap_db, seqno);
3119            TEST_CHK(s==FDB_RESULT_SUCCESS);
3120
3121            s = fdb_kvs_close(snap_db);
3122            TEST_CHK(s==FDB_RESULT_SUCCESS);
3123            s = fdb_kvs_close(db);
3124            TEST_CHK(s==FDB_RESULT_SUCCESS);
3125        }
3126    }
3127    return 0;
3128}
3129
3130void compact_with_snapshot_open_multi_kvs_test()
3131{
3132    TEST_INIT();
3133    memleak_start();
3134
3135    int i, j, r;
3136    int n = 1000;
3137    char keybuf[256], bodybuf[256];
3138    fdb_file_handle *dbfile;
3139    fdb_kvs_handle *db1, *db2, *db3, *db4, *db5;
3140    fdb_status s;
3141    fdb_config fconfig = fdb_get_default_config();
3142    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
3143    struct cb_args cb_args;
3144
3145    memset(&cb_args, 0x0, sizeof(struct cb_args));
3146
3147    fconfig.wal_threshold = 1024;
3148    fconfig.flags = FDB_OPEN_FLAG_CREATE;
3149    fconfig.compaction_cb = compaction_cb_markers;
3150    fconfig.compaction_cb_ctx = &cb_args;
3151    fconfig.compaction_cb_mask = FDB_CS_BEGIN |
3152                                 FDB_CS_MOVE_DOC |
3153                                 FDB_CS_FLUSH_WAL |
3154                                 FDB_CS_END;
3155
3156    // remove previous compact_test files
3157    r = system(SHELL_DEL" compact_test* > errorlog.txt");
3158    (void)r;
3159
3160    // open two handles for kvs
3161    fdb_open(&dbfile, "./compact_test1", &fconfig);
3162    fdb_kvs_open(dbfile, &db1, "db1", &kvs_config);
3163    fdb_kvs_open(dbfile, &db2, "db2", &kvs_config);
3164    fdb_kvs_open(dbfile, &db3, "db3", &kvs_config);
3165    fdb_kvs_open(dbfile, &db4, "db4", &kvs_config);
3166    fdb_kvs_open(dbfile, &db5, "db5", &kvs_config);
3167    for (j=0;j<5;++j){
3168        for (i=0;i<n;++i){
3169            sprintf(keybuf, "key%04d", i);
3170            sprintf(bodybuf, "body%04d", i);
3171            fdb_set_kv(db1, keybuf, strlen(keybuf), bodybuf, strlen(bodybuf));
3172            fdb_set_kv(db2, keybuf, strlen(keybuf), bodybuf, strlen(bodybuf));
3173            fdb_set_kv(db3, keybuf, strlen(keybuf), bodybuf, strlen(bodybuf));
3174            fdb_set_kv(db4, keybuf, strlen(keybuf), bodybuf, strlen(bodybuf));
3175            fdb_set_kv(db5, keybuf, strlen(keybuf), bodybuf, strlen(bodybuf));
3176        }
3177
3178        // commit
3179        s = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
3180        TEST_CHK(s == FDB_RESULT_SUCCESS);
3181
3182        // compact
3183        cb_args.n_moved_docs = n*(j+1);
3184        s = fdb_compact(dbfile, NULL);
3185        TEST_CHK(s == FDB_RESULT_SUCCESS);
3186    }
3187
3188    s = fdb_close(dbfile);
3189    TEST_CHK(s == FDB_RESULT_SUCCESS);
3190    s = fdb_shutdown();
3191    TEST_CHK(s == FDB_RESULT_SUCCESS);
3192    TEST_RESULT("compact with snapshot_open multi kvs test");
3193}
3194
3195
3196static atomic_uint8_t cancel_test_signal_begin(0);
3197static atomic_uint8_t cancel_test_signal_end(0);
3198
3199static int cb_cancel_test(fdb_file_handle *fhandle,
3200                          fdb_compaction_status status, const char *kv_name,
3201                          fdb_doc *doc, uint64_t old_offset, uint64_t new_offset,
3202                          void *ctx)
3203{
3204    (void) fhandle;
3205    (void) kv_name;
3206    (void) doc;
3207    (void) old_offset;
3208    (void) new_offset;
3209    (void) ctx;
3210
3211    if (status == FDB_CS_BEGIN) {
3212        cancel_test_signal_begin = 1;
3213    } else if (status == FDB_CS_END) {
3214        cancel_test_signal_end = 1;
3215    }
3216
3217    return 0;
3218}
3219
3220void *db_compact_during_compaction_cancellation(void *args)
3221{
3222
3223    TEST_INIT();
3224
3225    fdb_file_handle *dbfile;
3226    fdb_status status;
3227    fdb_config config;
3228
3229    // Open Database File
3230    config = fdb_get_default_config();
3231    config.compaction_cb = cb_cancel_test;
3232    config.compaction_cb_ctx = NULL;
3233    config.compaction_cb_mask = FDB_CS_BEGIN | FDB_CS_END;
3234
3235    status = fdb_open(&dbfile, "compact_test", &config);
3236    TEST_CHK(status == FDB_RESULT_SUCCESS);
3237    // compaction thread enters here
3238    status = fdb_compact(dbfile, NULL);
3239    TEST_CHK(status == FDB_RESULT_SUCCESS ||
3240             status == FDB_RESULT_COMPACTION_CANCELLATION ||
3241             status == FDB_RESULT_FAIL_BY_ROLLBACK);
3242    fdb_close(dbfile);
3243
3244    // shutdown
3245    thread_exit(0);
3246    return NULL;
3247}
3248
3249typedef enum {
3250    COMPACTION_CANCEL_MODE = 0,
3251    COMPACTION_ROLLBACK_MODE = 1
3252} compaction_test_mode;
3253
3254void compaction_cancellation_test(compaction_test_mode mode)
3255{
3256    TEST_INIT();
3257
3258    memleak_start();
3259
3260    int i, r, n=100000;
3261    fdb_file_handle *file;
3262    fdb_kvs_handle *kvs;
3263    fdb_status status;
3264    fdb_config config;
3265    fdb_kvs_config kvs_config;
3266
3267    r = system(SHELL_DEL" compact_test* > errorlog.txt");
3268    (void)r;
3269
3270    // Open Database File
3271    config = fdb_get_default_config();
3272    status = fdb_open(&file, "compact_test", &config);
3273    TEST_CHK(status == FDB_RESULT_SUCCESS);
3274
3275    // Open KV Store
3276    kvs_config = fdb_get_default_kvs_config();
3277    status = fdb_kvs_open_default(file, &kvs, &kvs_config);
3278    TEST_CHK(status == FDB_RESULT_SUCCESS);
3279
3280    // Load kv pairs
3281    for(i=0;i<n;i++) {
3282        char str[15];
3283        sprintf(str, "%d", i);
3284        status = fdb_set_kv(kvs, str, strlen(str), (void*)"value", 5);
3285        TEST_CHK(status == FDB_RESULT_SUCCESS);
3286        // Commit every 100 SETs
3287        if (i % 100 == 0) {
3288            status = fdb_commit(file, FDB_COMMIT_NORMAL);
3289            TEST_CHK(status == FDB_RESULT_SUCCESS);
3290        }
3291    }
3292
3293    thread_t tid;
3294    void *thread_ret;
3295    bool rollback_failed = false;
3296
3297    thread_create(&tid, db_compact_during_compaction_cancellation, NULL);
3298
3299    // wait until compaction begins
3300    while (!cancel_test_signal_begin) {
3301        usleep(1000); // Sleep for 1ms
3302    }
3303
3304    if (mode == COMPACTION_ROLLBACK_MODE) {
3305        // append more mutations
3306        for(i=0;i<n/10;i++) {
3307            char str[15];
3308            sprintf(str, "%d", i);
3309            status = fdb_set_kv(kvs, str, strlen(str), (void*)"value", 5);
3310            TEST_CHK(status == FDB_RESULT_SUCCESS);
3311            // Commit every 100 SETs
3312            if (i % 100 == 0) {
3313                status = fdb_commit(file, FDB_COMMIT_NORMAL);
3314                TEST_CHK(status == FDB_RESULT_SUCCESS);
3315            }
3316        }
3317    }
3318
3319    if (mode == COMPACTION_CANCEL_MODE) {
3320        // Cancel the compaction task
3321        status = fdb_cancel_compaction(file);
3322        TEST_CHK(status == FDB_RESULT_SUCCESS);
3323    } else if (mode == COMPACTION_ROLLBACK_MODE) {
3324        // Rollback
3325
3326        // wait until the 1st phase is done
3327        while (!cancel_test_signal_end) {
3328            usleep(1000); // Sleep for 1ms
3329        }
3330
3331        status = fdb_rollback(&kvs, n/2 + 1);
3332        if (status != FDB_RESULT_SUCCESS) {
3333            // if compactor thread is done before reaching this line,
3334            // rollback may fail.
3335            rollback_failed = true;
3336        }
3337    }
3338    // join compactor
3339    thread_join(tid, &thread_ret);
3340
3341    // if rollback failed, we don't need to compact the file again.
3342    if (!rollback_failed) {
3343        // Compact the database
3344        status = fdb_compact(file, NULL);
3345        TEST_CHK(status == FDB_RESULT_SUCCESS);
3346    }
3347
3348    char key[15];
3349    void *value;
3350    size_t val_size;
3351
3352    for(i=0;i<n;i++) {
3353        sprintf(key, "%d", i);
3354        status = fdb_get_kv(kvs, key, strlen(key), &value, &val_size);
3355        if (mode == COMPACTION_CANCEL_MODE) {
3356            // compaction cancel mode: all docs should be retrieved
3357            TEST_CHK(status == FDB_RESULT_SUCCESS);
3358            fdb_free_block(value);
3359        } else if (mode == COMPACTION_ROLLBACK_MODE) {
3360            // rollback mode: only the first half docs should be retrieved
3361            if (i<=n/2 || rollback_failed) {
3362                // if rollback failed, all doc should be retrieved
3363                TEST_CHK(status == FDB_RESULT_SUCCESS);
3364                fdb_free_block(value);
3365            } else {
3366                TEST_CHK(status != FDB_RESULT_SUCCESS);
3367            }
3368        }
3369    }
3370
3371    status = fdb_close(file);
3372    TEST_CHK(status == FDB_RESULT_SUCCESS);
3373    status = fdb_shutdown();
3374    TEST_CHK(status == FDB_RESULT_SUCCESS);
3375
3376    memleak_end();
3377
3378    if (mode == COMPACTION_CANCEL_MODE) {
3379        TEST_RESULT("compaction cancellation test");
3380    } else if (mode == COMPACTION_ROLLBACK_MODE) {
3381        TEST_RESULT("rollback during the 2nd phase of compaction test");
3382    }
3383}
3384
3385int main(){
3386    int i;
3387
3388    compact_deleted_doc_test();
3389    compact_upto_test(false); // single kv instance in file
3390    compact_upto_test(true); // multiple kv instance in file
3391    wal_delete_compact_upto_test();
3392    for (i=0;i<4;++i) {
3393        compact_upto_overwrite_test(i);
3394    }
3395    compact_with_snapshot_open_multi_kvs_test();
3396    compact_with_snapshot_open_test();
3397    compact_upto_post_snapshot_test();
3398    compact_upto_twice_test();
3399    compaction_callback_test(true); // multi kv instance mode
3400    compaction_callback_test(false); // single kv instance mode
3401    compact_wo_reopen_test();
3402    compact_with_reopen_test();
3403#if !defined(THREAD_SANITIZER)
3404    compact_reopen_with_iterator();
3405#endif
3406    compact_reopen_named_kvs();
3407    estimate_space_upto_test(false); // single kv instance in file
3408    estimate_space_upto_test(true); // multiple kv instance in file
3409    // Since we call unlink() to the old file after compaction,
3410    // cloning old file after the compaction doesn't work anymore.
3411    // So we temporarily disable this test.
3412    //auto_recover_compact_ok_test();
3413    unlink_after_compaction_test();
3414    db_compact_overwrite();
3415    db_compact_during_doc_delete(NULL);
3416    compaction_with_concurrent_transaction_test();
3417    compaction_with_concurrent_update_test();
3418    auto_compaction_with_custom_cmp_function();
3419    compaction_daemon_test(20);
3420    auto_compaction_with_concurrent_insert_test(20);
3421    compaction_cancellation_test(COMPACTION_CANCEL_MODE);
3422    // Disable it temporarily until it is resolved.
3423    //compaction_cancellation_test(COMPACTION_ROLLBACK_MODE);
3424
3425    return 0;
3426}
3427
3428