1 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3  *     Copyright 2010 Couchbase, Inc
4  *
5  *   Licensed under the Apache License, Version 2.0 (the "License");
6  *   you may not use this file except in compliance with the License.
7  *   You may obtain a copy of the License at
8  *
9  *       http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *   Unless required by applicable law or agreed to in writing, software
12  *   distributed under the License is distributed on an "AS IS" BASIS,
13  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *   See the License for the specific language governing permissions and
15  *   limitations under the License.
16  */
17 
18 #include <stdio.h>
19 #include <stdlib.h>
20 #include <string.h>
21 #include <stdint.h>
22 #include <time.h>
23 #if !defined(WIN32) && !defined(_WIN32)
24 #include <unistd.h>
25 #include <errno.h>
26 #endif
27 
28 #include "libforestdb/forestdb.h"
29 #include "test.h"
30 #include "filemgr_anomalous_ops.h"
31 #include "filemgr.h"
32 #include "internal_types.h"
33 
logCallbackFunc(int err_code, const char *err_msg, void *pCtxData)34 void logCallbackFunc(int err_code,
35                      const char *err_msg,
36                      void *pCtxData) {
37     fprintf(stderr, "%s - error code: %d, error message: %s\n",
38             (char *) pCtxData, err_code, err_msg);
39 }
40 
41 // callback context for test specific data
42 typedef struct fail_ctx_t {
43     int num_fails;
44     int num_ops;
45     int start_failing_after;
46 } fail_ctx_t;
47 
pwrite_failure_cb(void *ctx, struct filemgr_ops *normal_ops, int fd, void *buf, size_t count, cs_off_t offset)48 ssize_t pwrite_failure_cb(void *ctx, struct filemgr_ops *normal_ops,
49                           int fd, void *buf, size_t count, cs_off_t offset)
50 {
51     fail_ctx_t *wctx = (fail_ctx_t *)ctx;
52     wctx->num_ops++;
53     if (wctx->num_ops > wctx->start_failing_after) {
54         wctx->num_fails++;
55         errno = -2;
56         return (ssize_t)FDB_RESULT_WRITE_FAIL;
57     }
58     return normal_ops->pwrite(fd, buf, count, offset);
59 }
60 
write_failure_test()61 void write_failure_test()
62 {
63     TEST_INIT();
64 
65     memleak_start();
66 
67     int i, j, idx, r;
68     int n=300, m=20; // n: # prefixes, m: # postfixes
69     int keylen_limit;
70     fdb_file_handle *dbfile;
71     fdb_kvs_handle *db;
72     fdb_doc **doc = alca(fdb_doc*, n*m);
73     fdb_doc *rdoc;
74     fdb_status status;
75     int anomaly_hit = 0;
76 
77     char *keybuf;
78     char metabuf[256], bodybuf[256], temp[256];
79     // Get the default callbacks which result in normal operation for other ops
80     struct anomalous_callbacks *write_fail_cb = get_default_anon_cbs();
81     fail_ctx_t fail_ctx;
82     memset(&fail_ctx, 0, sizeof(fail_ctx_t));
83     // Modify the pwrite callback to redirect to test-specific function
84     write_fail_cb->pwrite_cb = &pwrite_failure_cb;
85 
86     // remove previous anomaly_test files
87     r = system(SHELL_DEL" anomaly_test* > errorlog.txt");
88     (void)r;
89 
90     // Reset anomalous behavior stats..
91     filemgr_ops_anomalous_init(write_fail_cb, &fail_ctx);
92 
93     // The number indicates the count after which all writes begin to fail
94     // This number is unique to this test suite and can cause a segmentation
95     // fault if the underlying fixed issue resurfaces
96     fail_ctx.start_failing_after = 10112;
97 
98     fdb_config fconfig = fdb_get_default_config();
99     fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
100     fconfig.buffercache_size = 0;
101     fconfig.flags = FDB_OPEN_FLAG_CREATE;
102     fconfig.purging_interval = 0;
103     fconfig.compaction_threshold = 0;
104 
105     keylen_limit = fconfig.blocksize - 256;
106     keybuf = alca(char, keylen_limit);
107 
108     // open db
109     fdb_open(&dbfile, "anomaly_test1", &fconfig);
110     fdb_kvs_open_default(dbfile, &db, &kvs_config);
111 
112     // key structure:
113     // <------------ <keylen_limit> bytes ------------->
114     // <-- 8 bytes -->             <-- 8 bytes  -->< 1 >
115     // [prefix number]____ ... ____[postfix number][ \0]
116     // e.g.)
117     // 00000001____ ... ____00000013[\0]
118 
119     // create docs
120     for (i=0;i<keylen_limit-1;++i){
121         keybuf[i] = '_';
122     }
123     keybuf[keylen_limit-1] = 0;
124 
125     for (i=0;i<n;++i){
126         // set prefix
127         sprintf(temp, "%08d", i);
128         memcpy(keybuf, temp, 8);
129         for (j=0;j<m;++j){
130             idx = i*m + j;
131             // set postfix
132             sprintf(temp, "%08d", j);
133             memcpy(keybuf + (keylen_limit-1) - 8, temp, 8);
134             sprintf(metabuf, "meta%d", idx);
135             sprintf(bodybuf, "body%d", idx);
136             fdb_doc_create(&doc[idx], (void*)keybuf, strlen(keybuf)+1,
137                                     (void*)metabuf, strlen(metabuf)+1,
138                                     (void*)bodybuf, strlen(bodybuf)+1);
139         }
140     }
141 
142     // insert docs
143     for (i=0;i<n*m;++i) {
144         fdb_set(db, doc[i]);
145     }
146     fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
147 
148     // retrieval check
149     for (i=0;i<n*m;++i){
150         fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
151         status = fdb_get(db, rdoc);
152         if (status != FDB_RESULT_SUCCESS) {
153             fdb_doc_free(rdoc);
154             anomaly_hit = 1;
155             break;
156         }
157         TEST_CHK(status == FDB_RESULT_SUCCESS);
158         TEST_CHK(!memcmp(rdoc->key, doc[i]->key, rdoc->keylen));
159         TEST_CHK(!memcmp(rdoc->meta, doc[i]->meta, rdoc->metalen));
160         TEST_CHK(!memcmp(rdoc->body, doc[i]->body, rdoc->bodylen));
161         fdb_doc_free(rdoc);
162     }
163     TEST_CHK(anomaly_hit);
164 
165     fdb_close(dbfile);
166 
167     // free all documents
168     for (i=0;i<n*m;++i){
169         fdb_doc_free(doc[i]);
170     }
171 
172     // free all resources
173     fdb_shutdown();
174 
175     memleak_end();
176 
177     sprintf(temp, "write failure test: %d failures out of %d writes",
178             fail_ctx.num_fails, fail_ctx.num_ops);
179 
180     TEST_RESULT(temp);
181 }
182 
pread_failure_cb(void *ctx, struct filemgr_ops *normal_ops, int fd, void *buf, size_t count, cs_off_t offset)183 ssize_t pread_failure_cb(void *ctx, struct filemgr_ops *normal_ops,
184                          int fd, void *buf, size_t count, cs_off_t offset)
185 {
186     fail_ctx_t *wctx = (fail_ctx_t *)ctx;
187     wctx->num_ops++;
188     if (wctx->num_ops > wctx->start_failing_after) {
189         wctx->num_fails++;
190         errno = -2;
191         return (ssize_t)FDB_RESULT_READ_FAIL;
192     }
193     return normal_ops->pread(fd, buf, count, offset);
194 }
195 
read_failure_test()196 void read_failure_test()
197 {
198     TEST_INIT();
199 
200     memleak_start();
201 
202     int i, r;
203     int n=300;
204     int keylen_limit;
205     fdb_file_handle *dbfile;
206     fdb_kvs_handle *db;
207     fdb_doc **doc = alca(fdb_doc*, n);
208     fdb_doc *rdoc;
209     fdb_status status;
210 
211     char *keybuf;
212     char metabuf[256], bodybuf[256], temp[256];
213     // Get the default callbacks which result in normal operation for other ops
214     struct anomalous_callbacks *read_fail_cb = get_default_anon_cbs();
215     fail_ctx_t fail_ctx;
216     memset(&fail_ctx, 0, sizeof(fail_ctx_t));
217 
218     // Modify the pread callback to redirect to test-specific function
219     read_fail_cb->pread_cb = &pread_failure_cb;
220 
221     // remove previous anomaly_test files
222     r = system(SHELL_DEL" anomaly_test* > errorlog.txt");
223     (void)r;
224 
225     // Reset anomalous behavior stats..
226     filemgr_ops_anomalous_init(read_fail_cb, &fail_ctx);
227 
228     fail_ctx.start_failing_after = 1000; // some large value
229 
230     fdb_config fconfig = fdb_get_default_config();
231     fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
232     fconfig.buffercache_size = 0;
233     fconfig.flags = FDB_OPEN_FLAG_CREATE;
234     fconfig.purging_interval = 0;
235     fconfig.compaction_threshold = 0;
236 
237     keylen_limit = fconfig.blocksize - 256;
238     keybuf = alca(char, keylen_limit);
239 
240     // open db
241     status = fdb_open(&dbfile, "./anomaly_test1", &fconfig);
242     TEST_CHK(status == FDB_RESULT_SUCCESS);
243     status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
244     TEST_CHK(status == FDB_RESULT_SUCCESS);
245 
246     status = fdb_set_log_callback(db, logCallbackFunc,
247                                   (void *) "read_failure_test");
248     TEST_CHK(status == FDB_RESULT_SUCCESS);
249 
250     // insert documents
251     for (i=0;i<n;++i){
252         sprintf(keybuf, "key%d", i);
253         sprintf(metabuf, "meta%d", i);
254         sprintf(bodybuf, "body%d", i);
255         fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
256             (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
257         status = fdb_set(db, doc[i]);
258         TEST_CHK(status == FDB_RESULT_SUCCESS);
259     }
260 
261     // commit
262     status = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
263     TEST_CHK(status == FDB_RESULT_SUCCESS);
264     fdb_kvs_close(db);
265     fdb_close(dbfile);
266 
267     fail_ctx.start_failing_after = fail_ctx.num_ops; // immediately fail
268 
269     status = fdb_open(&dbfile, "./anomaly_test1", &fconfig);
270     TEST_CHK(status == FDB_RESULT_READ_FAIL || status == FDB_RESULT_SB_READ_FAIL);
271 
272     fail_ctx.start_failing_after = fail_ctx.num_ops+1000; //normal operation
273 
274     status = fdb_open(&dbfile, "./anomaly_test1", &fconfig);
275     TEST_CHK(status == FDB_RESULT_SUCCESS);
276     status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
277     TEST_CHK(status == FDB_RESULT_SUCCESS);
278 
279     status = fdb_set_log_callback(db, logCallbackFunc,
280                                   (void *) "read_failure_test");
281     TEST_CHK(status == FDB_RESULT_SUCCESS);
282 
283     fail_ctx.start_failing_after = fail_ctx.num_ops; // immediately fail
284     i = 0;
285     fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
286     status = fdb_get(db, rdoc);
287 
288     TEST_CHK(status == FDB_RESULT_READ_FAIL);
289     // free result document
290     fdb_doc_free(rdoc);
291 
292     fail_ctx.start_failing_after = fail_ctx.num_ops+1000; //normal operation
293     // retrieve documents
294     for (i=0;i<n;++i){
295         // search by key
296         fdb_doc_create(&rdoc, doc[i]->key, doc[i]->keylen, NULL, 0, NULL, 0);
297         status = fdb_get(db, rdoc);
298 
299         TEST_CHK(status == FDB_RESULT_SUCCESS);
300         TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
301         TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
302 
303         // free result document
304         fdb_doc_free(rdoc);
305     }
306 
307     fdb_kvs_close(db);
308     fdb_close(dbfile);
309 
310     // free all documents
311     for (i=0;i<n;++i){
312         fdb_doc_free(doc[i]);
313     }
314 
315     // free all resources
316     fdb_shutdown();
317 
318     memleak_end();
319 
320     sprintf(temp, "read failure test: %d failures out of %d reads",
321             fail_ctx.num_fails, fail_ctx.num_ops);
322 
323     TEST_RESULT(temp);
324 }
325 
326 struct shared_data {
327     fdb_file_handle *dbfile;
328     fdb_kvs_handle *db;
329     fdb_iterator *iterator;
330     bool test_handle_busy;
331 };
332 
bad_thread(void *voidargs)333 void *bad_thread(void *voidargs) {
334     struct shared_data *data = (struct shared_data *)voidargs;
335     fdb_kvs_handle *db = data->db;
336     fdb_iterator *itr = data->iterator;
337     fdb_status s;
338     fdb_doc doc;
339     TEST_INIT();
340 
341     memset(&doc, 0, sizeof(fdb_doc));
342     doc.key = &doc; // some non-null value
343     doc.keylen = 2; // some non-zero value
344     doc.body = &doc; // some non-null value
345     doc.bodylen = 2; // some non-zero value
346 
347     if (!itr) {
348         // since the parent thread is hung in the fdb_set callback
349         // all the forestdb apis calls on the same handle must return failure
350         s = fdb_set(db, &doc);
351         TEST_CHK(s == FDB_RESULT_HANDLE_BUSY);
352         s = fdb_del(db, &doc);
353         TEST_CHK(s == FDB_RESULT_HANDLE_BUSY);
354         s = fdb_get(db, &doc);
355         TEST_CHK(s == FDB_RESULT_HANDLE_BUSY);
356         s = fdb_get_metaonly(db, &doc);
357         TEST_CHK(s == FDB_RESULT_HANDLE_BUSY);
358         s = fdb_get_byseq(db, &doc);
359         TEST_CHK(s == FDB_RESULT_HANDLE_BUSY);
360         s = fdb_get_metaonly_byseq(db, &doc);
361         TEST_CHK(s == FDB_RESULT_HANDLE_BUSY);
362         doc.offset = 5000; // some random non-zero value
363         s = fdb_get_byoffset(db, &doc);
364         TEST_CHK(s == FDB_RESULT_HANDLE_BUSY);
365     } else {
366         s = fdb_iterator_next(itr);
367         TEST_CHK(s == FDB_RESULT_HANDLE_BUSY);
368         s = fdb_iterator_prev(itr);
369         TEST_CHK(s == FDB_RESULT_HANDLE_BUSY);
370         s = fdb_iterator_seek_to_min(itr);
371         TEST_CHK(s == FDB_RESULT_HANDLE_BUSY);
372         s = fdb_iterator_seek_to_max(itr);
373         TEST_CHK(s == FDB_RESULT_HANDLE_BUSY);
374         s = fdb_iterator_seek(itr, doc.key, doc.keylen, 0);
375         TEST_CHK(s == FDB_RESULT_HANDLE_BUSY);
376     }
377 
378     return NULL;
379 }
380 
381 // Calling apis from a callback simulates concurrent access from multiple
382 // threads
pwrite_hang_cb(void *ctx, struct filemgr_ops *normal_ops, int fd, void *buf, size_t count, cs_off_t offset)383 ssize_t pwrite_hang_cb(void *ctx, struct filemgr_ops *normal_ops,
384                        int fd, void *buf, size_t count, cs_off_t offset)
385 {
386     struct shared_data *data = (struct shared_data *)ctx;
387     if (data->test_handle_busy) {
388         bad_thread(ctx);
389     }
390     return normal_ops->pwrite(fd, buf, count, offset);
391 }
392 
handle_busy_test()393 void handle_busy_test()
394 {
395     TEST_INIT();
396 
397     memleak_start();
398 
399     int n = 32;
400     int i = 0, r;
401 
402     char keybuf[16], metabuf[16], bodybuf[16];
403     fdb_doc **doc = alca(fdb_doc *, n);
404     struct shared_data data;
405     fdb_kvs_handle *db;
406     fdb_iterator *itr;
407     fdb_status status;
408 
409     // Get the default callbacks which result in normal operation for other ops
410     struct anomalous_callbacks *write_hang_cb = get_default_anon_cbs();
411 
412     // Modify the pwrite callback to redirect to test-specific function
413     write_hang_cb->pwrite_cb = &pwrite_hang_cb;
414 
415     // remove previous anomaly_test files
416     r = system(SHELL_DEL" anomaly_test* > errorlog.txt");
417     (void)r;
418 
419     memset(&data, 0, sizeof(struct shared_data));
420 
421     // Create anomalous behavior with shared handle for the callback ctx
422     filemgr_ops_anomalous_init(write_hang_cb, &data);
423 
424     fdb_config fconfig = fdb_get_default_config();
425     fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
426     fconfig.buffercache_size = 0;
427     fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
428     fconfig.flags = FDB_OPEN_FLAG_CREATE;
429     fconfig.purging_interval = 0;
430     fconfig.compaction_threshold = 0;
431 
432     // open db
433     status = fdb_open(&data.dbfile, "anomaly_test5", &fconfig);
434     TEST_CHK(status == FDB_RESULT_SUCCESS);
435     status = fdb_kvs_open_default(data.dbfile, &data.db, &kvs_config);
436     TEST_CHK(status == FDB_RESULT_SUCCESS);
437     db = data.db;
438 
439     // insert documents
440     for (i=0;i<n;++i){
441         sprintf(keybuf, "key%d", i);
442         sprintf(metabuf, "meta%d", i);
443         sprintf(bodybuf, "body%d", i);
444         fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf) + 1,
445             (void*)metabuf, strlen(metabuf) + 1, (void*)bodybuf,
446             strlen(bodybuf) + 1);
447         status = fdb_set(db, doc[i]);
448         TEST_CHK(status == FDB_RESULT_SUCCESS);
449     }
450 
451     // commit
452     status = fdb_commit(data.dbfile, FDB_COMMIT_NORMAL);
453     TEST_CHK(status == FDB_RESULT_SUCCESS);
454 
455     status = fdb_iterator_init(db, &itr, NULL, 0, NULL, 0,
456                                FDB_ITR_NONE);
457     TEST_CHK(status == FDB_RESULT_SUCCESS);
458 
459     // Set callback context to call bad_thread() and do a set invoking callback
460     data.test_handle_busy = 1;
461     status = fdb_set(db, doc[0]);
462     TEST_CHK(status == FDB_RESULT_SUCCESS);
463 
464     // Test iterator callbacks by attemping a set call on the iterator handle..
465     data.iterator = itr;
466     // TODO: remove if concurrent access on iterator handle can never happen
467     //status = fdb_set(itr->handle, doc[0]);
468     //TEST_CHK(status == FDB_RESULT_SUCCESS);
469 
470     fdb_iterator_close(itr);
471     data.test_handle_busy = 0;
472     fdb_close(data.dbfile);
473 
474     for (i = n - 1; i >=0; --i) {
475         fdb_doc_free(doc[i]);
476     }
477     // free all resources
478     fdb_shutdown();
479 
480     memleak_end();
481 
482     TEST_RESULT("Handle Busy Test");
483 }
484 
get_fs_type_cb(void *ctx, struct filemgr_ops *normal_ops, int srcfd)485 int get_fs_type_cb(void *ctx, struct filemgr_ops *normal_ops, int srcfd)
486 {
487     return FILEMGR_FS_EXT4_WITH_COW;
488 }
489 
490 struct cb_cmp_args {
491     fdb_kvs_handle *handle;
492     int ndocs;
493     int nmoves;
494 };
495 
cb_compact(fdb_file_handle *fhandle, fdb_compaction_status status, const char *kv_name, fdb_doc *doc, uint64_t old_offset, uint64_t new_offset, void *ctx)496 static fdb_compact_decision cb_compact(fdb_file_handle *fhandle,
497                             fdb_compaction_status status, const char *kv_name,
498                             fdb_doc *doc, uint64_t old_offset,
499                             uint64_t new_offset, void *ctx)
500 {
501     TEST_INIT();
502     struct cb_cmp_args *args = (struct cb_cmp_args *)ctx;
503     fdb_status fs;
504     fdb_compact_decision ret = FDB_CS_KEEP_DOC;
505     (void) fhandle;
506     (void) doc;
507     (void) old_offset;
508     (void) new_offset;
509 
510     if (status == FDB_CS_MOVE_DOC) {
511         TEST_CHK(kv_name);
512         args->nmoves++;
513         if (doc->deleted) {
514             ret = FDB_CS_DROP_DOC;
515         }
516         if (args->nmoves == args->ndocs - 1) {
517             char key[256], value[256];
518             // phase 3 of compaction - uncommitted docs in old_file
519             sprintf(key, "key%250d", args->ndocs);
520             sprintf(value, "body%250d", args->ndocs);
521             fs = fdb_set_kv(args->handle, key, 253, value, 254);
522             TEST_CHK(fs == FDB_RESULT_SUCCESS);
523             fs = fdb_commit(args->handle->fhandle, FDB_COMMIT_NORMAL);
524             TEST_CHK(fs == FDB_RESULT_SUCCESS);
525 
526             sprintf(key, "zzz%250d", args->ndocs);
527             fs = fdb_set_kv(args->handle, key, 253, value, 254);
528             TEST_CHK(fs == FDB_RESULT_SUCCESS);
529         }
530     }
531 
532     return ret;
533 }
534 
append_batch_delta(void)535 static void append_batch_delta(void)
536 {
537     TEST_INIT();
538     fdb_file_handle *dbfile;
539     fdb_kvs_handle *db;
540     fdb_status status;
541     int N = 5000;
542     int start = N/2;
543     int i;
544 
545     char key[256], value[256];
546     fdb_config fconfig = fdb_get_default_config();
547     fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
548     fconfig.buffercache_size = 0;
549     // open db
550     status = fdb_open(&dbfile, "anomaly_test1a", &fconfig);
551     TEST_CHK(status == FDB_RESULT_SUCCESS);
552     status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
553     TEST_CHK(status == FDB_RESULT_SUCCESS);
554 
555     // phase 2 of compaction...
556     // insert docs
557     for (i=start;i<N;++i){
558         sprintf(key, "key%250d", i);
559         sprintf(value, "body%250d", i);
560         status = fdb_set_kv(db, key, 253, value, 254);
561         TEST_CHK(status == FDB_RESULT_SUCCESS);
562         if (i == start + start/2) {
563             status = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
564             TEST_CHK(status == FDB_RESULT_SUCCESS);
565         }
566     }
567 
568     status = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
569     TEST_CHK(status == FDB_RESULT_SUCCESS);
570 
571     status = fdb_close(dbfile);
572     TEST_CHK(status == FDB_RESULT_SUCCESS);
573 }
574 
copy_file_range_cb(void *ctx, struct filemgr_ops *normal_ops, int fstype, int src, int dst, uint64_t src_off, uint64_t dst_off, uint64_t len)575 static int copy_file_range_cb(void *ctx, struct filemgr_ops *normal_ops,
576                               int fstype, int src, int dst,
577                               uint64_t src_off, uint64_t dst_off, uint64_t len)
578 {
579     uint8_t *buf = alca(uint8_t, len);
580     bool *append_delta = (bool *)ctx;
581 
582     TEST_INIT();
583     TEST_CHK(src_off % 4096 == 0);
584     TEST_CHK(dst_off % 4096 == 0);
585     TEST_CHK(len && len % 4096 == 0);
586     printf("File Range Copy src bid - %" _F64
587            " to dst bid = %" _F64 ", %" _F64" blocks\n",
588            src_off / 4096, dst_off / 4096, (len / 4096) + 1);
589     normal_ops->pread(src, buf, len, src_off);
590     normal_ops->pwrite(dst, buf, len, dst_off);
591     if (*append_delta) {
592         // While the compactor is stuck doing compaction append more documents
593         append_batch_delta();
594         *append_delta = false;
595     }
596     return FDB_RESULT_SUCCESS;
597 }
598 
copy_file_range_test()599 void copy_file_range_test()
600 {
601     TEST_INIT();
602 
603     memleak_start();
604 
605     int i, r;
606     int N = 5000; // total docs after append batch delta
607     int n = N/2;
608     bool append_delta = true;
609     fdb_file_handle *dbfile;
610     fdb_kvs_handle *db;
611     fdb_status status;
612 
613     char key[256], value[256];
614     void *value_out;
615     size_t valuelen;
616     struct cb_cmp_args cb_args;
617 
618     // Get the default callbacks which result in normal operation for other ops
619     struct anomalous_callbacks *cow_compact_cb = get_default_anon_cbs();
620     cow_compact_cb->get_fs_type_cb = &get_fs_type_cb;
621     cow_compact_cb->copy_file_range_cb = &copy_file_range_cb;
622 
623     memset(&cb_args, 0x0, sizeof(struct cb_cmp_args));
624 
625     // remove previous dummy files
626     r = system(SHELL_DEL" anomaly_test1a anomaly_test1b > errorlog.txt");
627     (void)r;
628 
629     fdb_config fconfig = fdb_get_default_config();
630     fconfig.compaction_cb = cb_compact;
631     fconfig.compaction_cb_ctx = &cb_args;
632     fconfig.compaction_cb_mask = FDB_CS_BEGIN |
633                                  FDB_CS_END | FDB_CS_MOVE_DOC;
634 
635     fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
636 
637     // Create anomalous behavior with shared handle for the callback ctx
638     filemgr_ops_anomalous_init(cow_compact_cb, &append_delta);
639 
640     // open db
641     status = fdb_open(&dbfile, "anomaly_test1a", &fconfig);
642     TEST_CHK(status == FDB_RESULT_SUCCESS);
643     status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
644     TEST_CHK(status == FDB_RESULT_SUCCESS);
645 
646     cb_args.handle = db;
647     cb_args.ndocs = N;
648 
649     // insert docs
650     for (i=0;i<n;++i){
651         sprintf(key, "key%250d", i);
652         sprintf(value, "body%250d", i);
653         status = fdb_set_kv(db, key, 253, value, 254);
654         TEST_CHK(status == FDB_RESULT_SUCCESS);
655     }
656 
657     status = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
658     TEST_CHK(status == FDB_RESULT_SUCCESS);
659 
660     // update docs making half the docs stale
661     for (i=n/2; i<n; ++i){
662         sprintf(key, "key%250d", i);
663         sprintf(value, "BODY%250d", i);
664         status = fdb_set_kv(db, key, 253, value, 254);
665         TEST_CHK(status == FDB_RESULT_SUCCESS);
666     }
667 
668     status = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
669     TEST_CHK(status == FDB_RESULT_SUCCESS);
670 
671     status = fdb_compact_with_cow(dbfile, "anomaly_test1b");
672     TEST_CHK(status == FDB_RESULT_SUCCESS);
673 
674     // retrieve check again
675     for (i=0; i<n; ++i){
676         sprintf(key, "key%250d", i);
677         if (i < n/2) {
678             sprintf(value, "body%250d", i);
679         } else {
680             sprintf(value, "BODY%250d", i);
681         }
682         status = fdb_get_kv(db, key, 253, &value_out, &valuelen);
683         TEST_CHK(status == FDB_RESULT_SUCCESS);
684         TEST_CMP(value_out, value, valuelen);
685         fdb_free_block(value_out);
686     }
687 
688     // retrieve docs after append batched delta
689     for (; i<N; ++i){
690         sprintf(key, "key%250d", i);
691         sprintf(value, "body%250d", i);
692         status = fdb_get_kv(db, key, 253, &value_out, &valuelen);
693         TEST_CHK(status == FDB_RESULT_SUCCESS);
694         TEST_CMP(value_out, value, valuelen);
695         fdb_free_block(value_out);
696     }
697 
698     // check on phase 3 inserted documents..
699     sprintf(key, "key%250d", i);
700     sprintf(value, "body%250d", i);
701     status = fdb_get_kv(db, key, 253, &value_out, &valuelen);
702     TEST_CHK(status == FDB_RESULT_SUCCESS);
703     TEST_CMP(value_out, value, valuelen);
704     fdb_free_block(value_out);
705 
706     sprintf(key, "zzz%250d", i);
707     status = fdb_get_kv(db, key, 253, &value_out, &valuelen);
708     TEST_CHK(status == FDB_RESULT_SUCCESS);
709     TEST_CHK(status == FDB_RESULT_SUCCESS);
710     TEST_CMP(value_out, value, valuelen);
711     fdb_free_block(value_out);
712 
713     // free all resources
714     status = fdb_close(dbfile);
715     TEST_CHK(status == FDB_RESULT_SUCCESS);
716     fdb_shutdown();
717 
718     memleak_end();
719 
720     TEST_RESULT("copy file range test");
721 }
722 
read_old_file()723 void read_old_file()
724 {
725     TEST_INIT();
726     int n=200, i, r;
727     fdb_file_handle *dbfile;
728     fdb_kvs_handle *db;
729     fdb_config config;
730     fdb_kvs_config kvs_config;
731     fdb_doc *doc;
732     fdb_status s; (void)s;
733     char keybuf[256], valuebuf[256];
734     void *normal_ops_ptr;
735 
736     memleak_start();
737 
738     // remove previous dummy files
739     r = system(SHELL_DEL" anomaly_test* > errorlog.txt");
740     (void)r;
741 
742     config = fdb_get_default_config();
743     config.buffercache_size = 0;
744     kvs_config = fdb_get_default_kvs_config();
745 
746     struct anomalous_callbacks *cbs = get_default_anon_cbs();
747     filemgr_ops_anomalous_init(cbs, NULL);
748 
749     normal_ops_ptr = get_normal_ops_ptr();
750 
751     // create a file
752     s = fdb_open(&dbfile, "anomaly_test1", &config);
753     TEST_CHK(s == FDB_RESULT_SUCCESS);
754 
755     s = fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
756     TEST_CHK(s == FDB_RESULT_SUCCESS);
757 
758     for (i=0; i<n; ++i) {
759         sprintf(keybuf, "k%06d", i);
760         sprintf(valuebuf, "v%06d", i);
761         fdb_doc_create(&doc, keybuf, 8, NULL, 0, valuebuf, 8);
762         s = fdb_set(db, doc);
763         TEST_CHK(s == FDB_RESULT_SUCCESS);
764         fdb_doc_free(doc);
765     }
766 
767     s = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
768     TEST_CHK(s == FDB_RESULT_SUCCESS);
769 
770     s = fdb_close(dbfile);
771     TEST_CHK(s == FDB_RESULT_SUCCESS);
772 
773     // hack the last 9 bytes (magic number + block marker) in the file
774     int fd = cbs->open_cb(NULL, (struct filemgr_ops*)normal_ops_ptr,
775                           "anomaly_test1", O_RDWR, 0644);
776     uint64_t offset = cbs->file_size_cb(NULL, (struct filemgr_ops*)normal_ops_ptr,
777                                         "anomaly_test1");
778     uint8_t magic[10] = {0xde, 0xad, 0xca, 0xfe, 0xbe, 0xef, 0xbe, 0xef, 0xee};
779     cbs->pwrite_cb(NULL, (struct filemgr_ops*)normal_ops_ptr, fd,
780                    (void*)magic, 9, offset-9);
781     cbs->close_cb(NULL, (struct filemgr_ops*)normal_ops_ptr, fd);
782 
783     // reopen
784     s = fdb_open(&dbfile, "anomaly_test1", &config);
785     // successfully read
786     TEST_CHK(s == FDB_RESULT_SUCCESS);
787 
788     s = fdb_close(dbfile);
789     TEST_CHK(s == FDB_RESULT_SUCCESS);
790 
791     fdb_shutdown();
792     memleak_end();
793 
794     TEST_RESULT("read an old file test");
795 }
796 
corrupted_header_correct_superblock_test()797 void corrupted_header_correct_superblock_test()
798 {
799     TEST_INIT();
800     int n=200, n_commits=4, i, j, r;
801     fdb_file_handle *dbfile;
802     fdb_kvs_handle *db;
803     fdb_config config;
804     fdb_kvs_config kvs_config;
805     fdb_doc *doc;
806     fdb_status s; (void)s;
807     char keybuf[256], valuebuf[256];
808     struct filemgr_ops *normal_ops;
809 
810     memleak_start();
811 
812     // remove previous dummy files
813     r = system(SHELL_DEL" anomaly_test* > errorlog.txt");
814     (void)r;
815 
816     config = fdb_get_default_config();
817     config.buffercache_size = 0;
818     kvs_config = fdb_get_default_kvs_config();
819 
820     struct anomalous_callbacks *cbs = get_default_anon_cbs();
821     filemgr_ops_anomalous_init(cbs, NULL);
822 
823     normal_ops = get_normal_ops_ptr();
824 
825     // create a file
826     s = fdb_open(&dbfile, "anomaly_test1", &config);
827     TEST_CHK(s == FDB_RESULT_SUCCESS);
828 
829     s = fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
830     TEST_CHK(s == FDB_RESULT_SUCCESS);
831 
832     for (j=0; j<n_commits; ++j) {
833         for (i=0; i<n; ++i) {
834             sprintf(keybuf, "k%06d", i);
835             sprintf(valuebuf, "v%d_%04d", j, i);
836             fdb_doc_create(&doc, keybuf, 8, NULL, 0, valuebuf, 8);
837             s = fdb_set(db, doc);
838             TEST_CHK(s == FDB_RESULT_SUCCESS);
839             fdb_doc_free(doc);
840         }
841         s = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
842         TEST_CHK(s == FDB_RESULT_SUCCESS);
843     }
844 
845     s = fdb_close(dbfile);
846     TEST_CHK(s == FDB_RESULT_SUCCESS);
847 
848     // copy the data in the file except for the last (header) block
849     int fd = cbs->open_cb(NULL, normal_ops, "anomaly_test1", O_RDWR, 0644);
850     int fd_dst = cbs->open_cb(NULL, normal_ops,
851                               "anomaly_test2", O_CREAT | O_RDWR, 0644);
852     uint64_t offset = cbs->file_size_cb(NULL, normal_ops, "anomaly_test1");
853     uint8_t *filedata = (uint8_t*)malloc(offset);
854     cbs->pread_cb(NULL, normal_ops, fd, (void*)filedata, offset, 0);
855     cbs->pwrite_cb(NULL, normal_ops, fd_dst, (void*)filedata, offset - 4096, 0);
856     cbs->close_cb(NULL, normal_ops, fd);
857     cbs->close_cb(NULL, normal_ops, fd_dst);
858     free(filedata);
859 
860     // open the corrupted file
861     s = fdb_open(&dbfile, "anomaly_test2", &config);
862     TEST_CHK(s == FDB_RESULT_SUCCESS);
863 
864     s = fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
865     TEST_CHK(s == FDB_RESULT_SUCCESS);
866 
867     // data at the 3rd commit should be read
868     j = 2;
869     for (i=0; i<n; ++i) {
870         sprintf(keybuf, "k%06d", i);
871         sprintf(valuebuf, "v%d_%04d", j, i);
872         fdb_doc_create(&doc, keybuf, 8, NULL, 0, NULL, 0);
873         s = fdb_get(db, doc);
874         TEST_CHK(s == FDB_RESULT_SUCCESS);
875         TEST_CMP(doc->body, valuebuf, doc->bodylen);
876         fdb_doc_free(doc);
877     }
878 
879     s = fdb_close(dbfile);
880     TEST_CHK(s == FDB_RESULT_SUCCESS);
881 
882     fdb_shutdown();
883     memleak_end();
884 
885     TEST_RESULT("corrupted DB header from correct superblock test");
886 }
887 
fsync_failure_cb(void *ctx, struct filemgr_ops *normal_ops, int fd)888 int fsync_failure_cb(void *ctx, struct filemgr_ops *normal_ops,
889                      int fd) {
890     fail_ctx_t *wctx = (fail_ctx_t *)ctx;
891     wctx->num_ops++;
892     if (wctx->num_ops > wctx->start_failing_after) {
893         wctx->num_fails++;
894         errno = -2;
895         return (ssize_t)FDB_RESULT_FSYNC_FAIL;
896     }
897 
898     return normal_ops->fsync(fd);
899 }
900 
compaction_failure_hangs_rollback_test()901 void compaction_failure_hangs_rollback_test()
902 {
903     TEST_INIT();
904 
905     int i, r;
906     int n=300; // n: # prefixes, m: # postfixes
907     fdb_file_handle *dbfile, *dbfile_comp;
908     fdb_kvs_handle *db;
909     fdb_doc **doc = alca(fdb_doc*, n);
910     fdb_status status;
911 
912     char keybuf[32], metabuf[32], bodybuf[128];
913     // Get the default callbacks which result in normal operation for other ops
914     struct anomalous_callbacks *commit_fail_cb = get_default_anon_cbs();
915     fail_ctx_t fail_ctx;
916     memset(&fail_ctx, 0, sizeof(fail_ctx_t));
917     // Modify the fsync callback to redirect to test-specific function
918     commit_fail_cb->fsync_cb = &fsync_failure_cb;
919 
920     // remove previous anomaly_test files
921     r = system(SHELL_DEL" anomaly_test* > errorlog.txt");
922     (void)r;
923 
924     // Reset anomalous behavior stats..
925     filemgr_ops_anomalous_init(commit_fail_cb, &fail_ctx);
926 
927     // The number indicates the count after which all commits begin to fail
928     // This number is unique to this test suite and can cause a hang
929     // if the underlying fixed issue resurfaces
930     fail_ctx.start_failing_after = 3;
931 
932     fdb_config fconfig = fdb_get_default_config();
933     fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
934     fconfig.flags = FDB_OPEN_FLAG_CREATE;
935     fconfig.purging_interval = 0;
936 
937     // open db
938     fdb_open(&dbfile, "anomaly_test2", &fconfig);
939     fdb_kvs_open_default(dbfile, &db, &kvs_config);
940     for (i=0;i<n;++i){
941         sprintf(keybuf, "key%d", i);
942         sprintf(metabuf, "meta%d", i);
943         sprintf(bodybuf, "body%d", i);
944         fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf)+1,
945                        (void*)metabuf, strlen(metabuf)+1,
946                        (void*)bodybuf, strlen(bodybuf)+1);
947         status = fdb_set(db, doc[i]);
948         TEST_CHK(status == FDB_RESULT_SUCCESS);
949         if (i == n / 2) {
950             status = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
951             TEST_CHK(status == FDB_RESULT_SUCCESS);
952         }
953     }
954 
955     status = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
956     TEST_CHK(status == FDB_RESULT_SUCCESS);
957 
958     status = fdb_open(&dbfile_comp, "anomaly_test2", &fconfig);
959     TEST_CHK(status == FDB_RESULT_SUCCESS);
960 
961     // Due to anomalous ops this compaction should fail after new file
962     // is created and a final commit is attempted..
963     status = fdb_compact(dbfile_comp, "anomaly_test3");
964     TEST_CHK(status == FDB_RESULT_FSYNC_FAIL);
965     fail_ctx.start_failing_after = 99999; // reset this so rollback can proceed
966 
967     status = fdb_close(dbfile_comp);
968     TEST_CHK(status == FDB_RESULT_SUCCESS);
969 
970     // The above compaction failure should not result in rollback hanging..
971     // MB-21953: Rollback hangs infinitely in decaying_usleep due to compaction
972     // failure above which does not reset the file status to NORMAL
973     status = fdb_rollback(&db, n/2 + 1);
974     TEST_CHK(status == FDB_RESULT_SUCCESS);
975 
976     fdb_close(dbfile);
977 
978     // free all documents
979     for (i=0;i<n;++i){
980         fdb_doc_free(doc[i]);
981     }
982 
983     // free all resources
984     fdb_shutdown();
985 
986     sprintf(bodybuf, "compaction failure hangs rollback test :%d "
987                      "failures out of %d commits",
988                      fail_ctx.num_fails, fail_ctx.num_ops);
989 
990     TEST_RESULT(bodybuf);
991 }
992 
compaction_failure_recompact_remove_test()993 void compaction_failure_recompact_remove_test()
994 {
995     TEST_INIT();
996 
997     int i, r;
998     int n=300; // n: # prefixes, m: # postfixes
999     fdb_file_handle *dbfile, *dbfile_comp;
1000     fdb_kvs_handle *db;
1001     fdb_doc **doc = alca(fdb_doc*, n);
1002     fdb_status status;
1003 
1004     char keybuf[32], metabuf[32], bodybuf[128];
1005     // Get the default callbacks which result in normal operation for other ops
1006     struct anomalous_callbacks *commit_fail_cb = get_default_anon_cbs();
1007     fail_ctx_t fail_ctx;
1008     memset(&fail_ctx, 0, sizeof(fail_ctx_t));
1009     // Modify the fsync callback to redirect to test-specific function
1010     commit_fail_cb->fsync_cb = &fsync_failure_cb;
1011 
1012     // remove previous anomaly_test files
1013     r = system(SHELL_DEL" anomaly_test* > errorlog.txt");
1014     (void)r;
1015 
1016     // Reset anomalous behavior stats..
1017     filemgr_ops_anomalous_init(commit_fail_cb, &fail_ctx);
1018 
1019     // The number indicates the count after which all commits begin to fail
1020     // This number is unique to this test suite and can cause a hang
1021     // if the underlying fixed issue resurfaces
1022     fail_ctx.start_failing_after = 3;
1023 
1024     fdb_config fconfig = fdb_get_default_config();
1025     fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1026     fconfig.flags = FDB_OPEN_FLAG_CREATE;
1027     fconfig.purging_interval = 0;
1028 
1029     // open db
1030     fdb_open(&dbfile, "anomaly_test_compact_rm_1", &fconfig);
1031     fdb_kvs_open_default(dbfile, &db, &kvs_config);
1032     for (i=0;i<n;++i){
1033         sprintf(keybuf, "key%d", i);
1034         sprintf(metabuf, "meta%d", i);
1035         sprintf(bodybuf, "body%d", i);
1036         fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf)+1,
1037                        (void*)metabuf, strlen(metabuf)+1,
1038                        (void*)bodybuf, strlen(bodybuf)+1);
1039         status = fdb_set(db, doc[i]);
1040         TEST_CHK(status == FDB_RESULT_SUCCESS);
1041         if (i == n / 2) {
1042             status = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1043             TEST_CHK(status == FDB_RESULT_SUCCESS);
1044         }
1045     }
1046 
1047     status = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1048     TEST_CHK(status == FDB_RESULT_SUCCESS);
1049 
1050     status = fdb_close(dbfile);
1051     TEST_CHK(status == FDB_RESULT_SUCCESS);
1052 
1053     status = fdb_open(&dbfile_comp, "anomaly_test_compact_rm_1", &fconfig);
1054     TEST_CHK(status == FDB_RESULT_SUCCESS);
1055 
1056     // Due to anomalous ops this compaction should fail after new file
1057     // is created and a final commit is attempted..
1058     status = fdb_compact(dbfile_comp, "anomaly_test_compact_rm_2");
1059     TEST_CHK(status == FDB_RESULT_FSYNC_FAIL);
1060     fail_ctx.start_failing_after = 99999; // reset this so rollback can proceed
1061     status = fdb_commit(dbfile_comp, FDB_COMMIT_NORMAL);
1062     TEST_CHK(status == FDB_RESULT_SUCCESS);
1063     status = fdb_close(dbfile_comp);
1064     TEST_CHK(status == FDB_RESULT_SUCCESS);
1065 
1066     // remove the failed compacted file
1067     r = system(SHELL_DEL" anomaly_test_compact_rm_2 > errorlog.txt");
1068     (void)r;
1069 
1070      // shutdown
1071     status = fdb_shutdown();
1072     TEST_CHK(status == FDB_RESULT_SUCCESS);
1073     fconfig.flags = 0;
1074     fconfig.purging_interval = 0;
1075     //reopen and recompact the original file
1076     status = fdb_open(&dbfile_comp, "anomaly_test_compact_rm_1", &fconfig);
1077     TEST_CHK(status == FDB_RESULT_SUCCESS);
1078     status = fdb_compact(dbfile_comp, "anomaly_test_compact_rm_2");
1079     TEST_CHK(status == FDB_RESULT_SUCCESS);
1080     status = fdb_close(dbfile_comp);
1081     TEST_CHK(status == FDB_RESULT_SUCCESS);
1082 
1083     // free all documents
1084     for (i=0;i<n;++i){
1085         fdb_doc_free(doc[i]);
1086     }
1087 
1088     // free all resources
1089     status = fdb_shutdown();
1090     TEST_CHK(status == FDB_RESULT_SUCCESS);
1091 
1092     sprintf(bodybuf, "failed compaction recompaction failure test :%d "
1093                      "failures out of %d commits",
1094                      fail_ctx.num_fails, fail_ctx.num_ops);
1095 
1096     TEST_RESULT(bodybuf);
1097 }
1098 
compaction_failure_recompact_test()1099 void compaction_failure_recompact_test()
1100 {
1101     TEST_INIT();
1102 
1103     int i, r;
1104     int n=300; // n: # prefixes, m: # postfixes
1105     fdb_file_handle *dbfile, *dbfile_comp;
1106     fdb_kvs_handle *db;
1107     fdb_doc **doc = alca(fdb_doc*, n);
1108     fdb_status status;
1109 
1110     char keybuf[32], metabuf[32], bodybuf[128];
1111     // Get the default callbacks which result in normal operation for other ops
1112     struct anomalous_callbacks *commit_fail_cb = get_default_anon_cbs();
1113     fail_ctx_t fail_ctx;
1114     memset(&fail_ctx, 0, sizeof(fail_ctx_t));
1115     // Modify the fsync callback to redirect to test-specific function
1116     commit_fail_cb->fsync_cb = &fsync_failure_cb;
1117 
1118     // remove previous anomaly_test files
1119     r = system(SHELL_DEL" anomaly_test* > errorlog.txt");
1120     (void)r;
1121 
1122     // Reset anomalous behavior stats..
1123     filemgr_ops_anomalous_init(commit_fail_cb, &fail_ctx);
1124 
1125     // The number indicates the count after which all commits begin to fail
1126     // This number is unique to this test suite and can cause a hang
1127     // if the underlying fixed issue resurfaces
1128     fail_ctx.start_failing_after = 3;
1129 
1130     fdb_config fconfig = fdb_get_default_config();
1131     fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1132     fconfig.flags = FDB_OPEN_FLAG_CREATE;
1133     fconfig.purging_interval = 0;
1134 
1135     // open db
1136     fdb_open(&dbfile, "anomaly_test_compact_1", &fconfig);
1137     fdb_kvs_open_default(dbfile, &db, &kvs_config);
1138     for (i=0;i<n;++i){
1139         sprintf(keybuf, "key%d", i);
1140         sprintf(metabuf, "meta%d", i);
1141         sprintf(bodybuf, "body%d", i);
1142         fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf)+1,
1143                        (void*)metabuf, strlen(metabuf)+1,
1144                        (void*)bodybuf, strlen(bodybuf)+1);
1145         status = fdb_set(db, doc[i]);
1146         TEST_CHK(status == FDB_RESULT_SUCCESS);
1147         if (i == n / 2) {
1148             status = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1149             TEST_CHK(status == FDB_RESULT_SUCCESS);
1150         }
1151     }
1152 
1153     status = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1154     TEST_CHK(status == FDB_RESULT_SUCCESS);
1155 
1156     status = fdb_close(dbfile);
1157     TEST_CHK(status == FDB_RESULT_SUCCESS);
1158 
1159     status = fdb_open(&dbfile_comp, "anomaly_test_compact_1", &fconfig);
1160     TEST_CHK(status == FDB_RESULT_SUCCESS);
1161 
1162     // Due to anomalous ops this compaction should fail after new file
1163     // is created and a final commit is attempted..
1164     status = fdb_compact(dbfile_comp, "anomaly_test_compact_2");
1165     TEST_CHK(status == FDB_RESULT_FSYNC_FAIL);
1166     fail_ctx.start_failing_after = 99999; // reset this so rollback can proceed
1167 
1168     status = fdb_close(dbfile_comp);
1169     TEST_CHK(status == FDB_RESULT_SUCCESS);
1170 
1171     // shutdown
1172     status = fdb_shutdown();
1173     TEST_CHK(status == FDB_RESULT_SUCCESS);
1174     fconfig.flags = 0;
1175     fconfig.purging_interval = 0;
1176     //reopen the orignal file, triggering a recovery of the comapcted file
1177     status = fdb_open(&dbfile_comp, "anomaly_test_compact_1", &fconfig);
1178     TEST_CHK(status == FDB_RESULT_SUCCESS);
1179     // try to compact the orignal file giving the same new name as before,
1180     // it should fail with invalid arguments
1181     // as the compacted file is already recovered to that new file name.
1182     status = fdb_compact(dbfile_comp, "anomaly_test_compact_2");
1183     TEST_CHK(status == FDB_RESULT_INVALID_ARGS);
1184     // try to compact again with a new name, this should succeed.
1185     status = fdb_compact(dbfile_comp, "anomaly_test_compact_3");
1186     TEST_CHK(status == FDB_RESULT_SUCCESS);
1187 
1188     // free all documents
1189     for (i=0;i<n;++i){
1190         fdb_doc_free(doc[i]);
1191     }
1192 
1193     status = fdb_close(dbfile_comp);
1194     TEST_CHK(status == FDB_RESULT_SUCCESS);
1195 
1196 
1197     // free all resources
1198     status = fdb_shutdown();
1199     TEST_CHK(status == FDB_RESULT_SUCCESS);
1200 
1201     sprintf(bodybuf, "failed compaction recompaction failure test :%d "
1202                      "failures out of %d commits",
1203                      fail_ctx.num_fails, fail_ctx.num_ops);
1204 
1205     TEST_RESULT(bodybuf);
1206 }
1207 
main()1208 int main(){
1209 
1210     /**
1211      * Commented out this test for now; it copies consecutive document blocks
1212      * to other file but they are written in different BID compared to the source
1213      * file, so that meta section at the end of each document block points to wrong
1214      * block and consequently documents cannot be read correctly.
1215      */
1216     //copy_file_range_test();
1217     write_failure_test();
1218     read_failure_test();
1219     handle_busy_test();
1220     read_old_file();
1221     corrupted_header_correct_superblock_test();
1222     compaction_failure_hangs_rollback_test();
1223     compaction_failure_recompact_test();
1224     compaction_failure_recompact_remove_test();
1225     return 0;
1226 }
1227