1/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/*
3 *     Copyright 2010 Couchbase, Inc
4 *
5 *   Licensed under the Apache License, Version 2.0 (the "License");
6 *   you may not use this file except in compliance with the License.
7 *   You may obtain a copy of the License at
8 *
9 *       http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *   Unless required by applicable law or agreed to in writing, software
12 *   distributed under the License is distributed on an "AS IS" BASIS,
13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *   See the License for the specific language governing permissions and
15 *   limitations under the License.
16 */
17
18#include <stdio.h>
19#include <stdlib.h>
20#include <string.h>
21#include <stdint.h>
22#include <time.h>
23#if !defined(WIN32) && !defined(_WIN32)
24#include <unistd.h>
25#endif
26
27#include "libforestdb/forestdb.h"
28#include "test.h"
29
30#include "internal_types.h"
31#include "functional_util.h"
32
33
34void iterator_test()
35{
36    TEST_INIT();
37
38    memleak_start();
39
40    int i, r;
41    int n = 10;
42    fdb_file_handle *dbfile;
43    fdb_kvs_handle *db, *kv1;;
44    fdb_doc **doc = alca(fdb_doc*, n);
45    fdb_doc *rdoc = NULL;
46    fdb_status status;
47    fdb_iterator *iterator;
48
49    char keybuf[256], metabuf[256], bodybuf[256], temp[256];
50
51    // remove  all previous iterator_test files
52    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
53    (void)r;
54
55    fdb_config fconfig = fdb_get_default_config();
56    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
57    fconfig.buffercache_size = 0;
58    fconfig.wal_threshold = 1024;
59    fconfig.flags = FDB_OPEN_FLAG_CREATE;
60    fconfig.compaction_threshold = 0;
61    fconfig.purging_interval = 1; // retain deletes before compaction
62
63    // open db
64    fdb_open(&dbfile, "./iterator_test1", &fconfig);
65    fdb_kvs_open_default(dbfile, &db, &kvs_config);
66    status = fdb_set_log_callback(db, logCallbackFunc,
67                                  (void *) "iterator_test");
68    TEST_CHK(status == FDB_RESULT_SUCCESS);
69
70    // Create another KV store..
71    status = fdb_kvs_open(dbfile, &kv1, "kv1", &kvs_config);
72    TEST_CHK(status == FDB_RESULT_SUCCESS);
73
74    // insert using 'kv1' instance to ensure it does not interfere
75    for (i=0;i<n;++i){
76        sprintf(keybuf, "kEy%d", i);
77        sprintf(metabuf, "mEta%d", i);
78        sprintf(bodybuf, "bOdy%d", i);
79        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
80            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
81        status = fdb_set(kv1, doc[i]);
82        TEST_CHK(status == FDB_RESULT_SUCCESS);
83        fdb_doc_free(doc[i]);
84    }
85
86    // insert documents of even number
87    for (i=0;i<n;i+=2){
88        sprintf(keybuf, "key%d", i);
89        sprintf(metabuf, "meta%d", i);
90        sprintf(bodybuf, "body%d", i);
91        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
92            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
93        fdb_set(db, doc[i]);
94    }
95    // manually flush WAL & commit
96    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
97
98    // insert documents of odd number
99    for (i=1;i<n;i+=2){
100        sprintf(keybuf, "key%d", i);
101        sprintf(metabuf, "meta%d", i);
102        sprintf(bodybuf, "body%d", i);
103        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
104            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
105        fdb_set(db, doc[i]);
106    }
107    // commit without WAL flush
108    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
109
110    // now even number docs are in hb-trie & odd number docs are in WAL
111
112    // create an iterator for full range
113    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, FDB_ITR_NONE);
114
115    // repeat until fail
116    i=0;
117    do {
118        status = fdb_iterator_get(iterator, &rdoc);
119        TEST_CHK(status == FDB_RESULT_SUCCESS);
120
121        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
122        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
123        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
124
125        fdb_doc_free(rdoc);
126        rdoc = NULL;
127        i++;
128    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
129    TEST_CHK(i==10);
130    fdb_iterator_close(iterator);
131
132    // create an iterator.
133    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, FDB_ITR_NONE);
134
135    // repeat until fail
136    i=0;
137    do {
138        // retrieve the next doc and get the byte offset of the returned doc
139        status = fdb_iterator_get_metaonly(iterator, &rdoc);
140        TEST_CHK(status == FDB_RESULT_SUCCESS);
141
142        TEST_CHK(rdoc->offset != BLK_NOT_FOUND);
143        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
144        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
145        TEST_CHK(rdoc->body == NULL);
146
147        fdb_doc_free(rdoc);
148        rdoc = NULL;
149        i++;
150    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
151    TEST_CHK(i==10);
152    fdb_iterator_close(iterator);
153
154    // create another iterator starts from doc[3]
155    sprintf(keybuf, "key%d", 3);
156    fdb_iterator_init(db, &iterator, (void*)keybuf, strlen(keybuf), NULL, 0,
157                      FDB_ITR_NONE);
158
159    // repeat until fail
160    i=3;
161    do {
162        status = fdb_iterator_get(iterator, &rdoc);
163        TEST_CHK(status == FDB_RESULT_SUCCESS);
164
165        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
166        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
167        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
168
169        fdb_doc_free(rdoc);
170        rdoc = NULL;
171        i++;
172    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
173    TEST_CHK(i==10);
174    fdb_iterator_close(iterator);
175
176    // create another iterator for the range of doc[4] ~ doc[8]
177    sprintf(keybuf, "key%d", 4);
178    sprintf(temp, "key%d", 8);
179    fdb_iterator_init(db, &iterator, (void*)keybuf, strlen(keybuf),
180        (void*)temp, strlen(temp), FDB_ITR_NONE);
181
182    // repeat until fail
183    i=4;
184    do {
185        status = fdb_iterator_get(iterator, &rdoc);
186        TEST_CHK(status == FDB_RESULT_SUCCESS);
187
188        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
189        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
190        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
191
192        fdb_doc_free(rdoc);
193        rdoc = NULL;
194        i++;
195    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
196    TEST_CHK(i==9);
197    fdb_iterator_close(iterator);
198
199    // create another iterator for the range of doc[5] ~ doc[7]
200    fdb_iterator_init(db, &iterator, (void*)keybuf, strlen(keybuf),
201        (void*)temp, strlen(temp), FDB_ITR_SKIP_MIN_KEY | FDB_ITR_SKIP_MAX_KEY);
202
203    // repeat until fail
204    i=5;
205    do {
206        status = fdb_iterator_get(iterator, &rdoc);
207        TEST_CHK(status == FDB_RESULT_SUCCESS);
208
209        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
210        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
211        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
212
213        fdb_doc_free(rdoc);
214        rdoc = NULL;
215        i++;
216    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
217    TEST_CHK(i==8);
218    fdb_iterator_close(iterator);
219
220    // remove document #8 and #9
221    fdb_doc_create(&rdoc, doc[8]->key, doc[8]->keylen, doc[8]->meta,
222                   doc[8]->metalen, NULL, 0);
223    status = fdb_del(db, rdoc);
224    TEST_CHK(status == FDB_RESULT_SUCCESS);
225    fdb_doc_free(rdoc);
226    rdoc = NULL;
227    fdb_doc_create(&rdoc, doc[9]->key, doc[9]->keylen, doc[9]->meta,
228                   doc[9]->metalen, NULL, 0);
229    status = fdb_del(db, rdoc);
230    TEST_CHK(status == FDB_RESULT_SUCCESS);
231    fdb_doc_free(rdoc);
232    rdoc = NULL;
233    // commit
234    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
235
236    // create an iterator for full range
237    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, FDB_ITR_NONE);
238    // repeat until fail
239    i=0;
240    do {
241        status = fdb_iterator_get(iterator, &rdoc);
242        TEST_CHK(status == FDB_RESULT_SUCCESS);
243
244        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
245        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
246        if (i < 8) {
247            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
248        } else {
249            TEST_CHK(rdoc->deleted == true);
250        }
251
252        fdb_doc_free(rdoc);
253        rdoc = NULL;
254        i++;
255    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
256    TEST_CHK(i==10);
257    fdb_iterator_close(iterator);
258
259    // create an iterator for full range, but no deletes.
260    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, FDB_ITR_NO_DELETES);
261    // repeat until fail
262    i=0;
263    do {
264        status = fdb_iterator_get(iterator, &rdoc);
265        TEST_CHK(status == FDB_RESULT_SUCCESS);
266
267        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
268        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
269        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
270
271        fdb_doc_free(rdoc);
272        rdoc = NULL;
273        i++;
274    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
275    TEST_CHK(i==8);
276    fdb_iterator_close(iterator);
277
278    // create an iterator for range of doc[4] ~ doc[8], but no deletes.
279    sprintf(keybuf, "key%d", 4);
280    sprintf(temp, "key%d", 8);
281    fdb_iterator_init(db, &iterator, keybuf, strlen(keybuf), temp, strlen(temp),
282                      FDB_ITR_NO_DELETES);
283    // repeat until fail
284    i=4;
285    do {
286        status = fdb_iterator_get(iterator, &rdoc);
287        TEST_CHK(status == FDB_RESULT_SUCCESS);
288
289        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
290        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
291        TEST_CHK(rdoc->deleted == false);
292
293        fdb_doc_free(rdoc);
294        rdoc = NULL;
295        i++;
296    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
297    TEST_CHK(i==8);
298    fdb_iterator_close(iterator);
299
300    // close kvs1 instance
301    fdb_kvs_close(kv1);
302    // close db file
303    fdb_kvs_close(db);
304    fdb_close(dbfile);
305
306    // free all documents
307    for (i=0;i<n;++i){
308        fdb_doc_free(doc[i]);
309    }
310
311    // free all resources
312    fdb_shutdown();
313
314    memleak_end();
315
316    TEST_RESULT("iterator test");
317}
318
319void iterator_with_concurrent_updates_test()
320{
321    // unit test for MB-12287
322    TEST_INIT();
323    memleak_start();
324
325    int i, n=10;
326    int r;
327    fdb_file_handle *dbfile;
328    fdb_kvs_handle *db1, *db2, *db3;
329    fdb_iterator *itr;
330    fdb_config fconfig;
331    fdb_kvs_config kvs_config;
332    fdb_doc **doc = alca(fdb_doc*, n);
333    fdb_doc *rdoc = NULL;
334    fdb_status status;
335    char keybuf[256], bodybuf[256];
336
337    // remove previous iterator_test files
338    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
339    (void)r;
340
341    // open db1, db2, db3 on the same file
342    fconfig = fdb_get_default_config();
343    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
344    kvs_config = fdb_get_default_kvs_config();
345    fdb_open(&dbfile, "./iterator_test1", &fconfig);
346
347    fdb_kvs_open_default(dbfile, &db1, &kvs_config);
348    status = fdb_set_log_callback(db1, logCallbackFunc,
349                                  (void *) "iterator_concurrent_update_test");
350    TEST_CHK(status == FDB_RESULT_SUCCESS);
351
352    fdb_kvs_open_default(dbfile, &db2, &kvs_config);
353    status = fdb_set_log_callback(db2, logCallbackFunc,
354                                  (void *) "iterator_concurrent_update_test");
355    TEST_CHK(status == FDB_RESULT_SUCCESS);
356
357    fdb_kvs_open_default(dbfile, &db3, &kvs_config);
358    status = fdb_set_log_callback(db3, logCallbackFunc,
359                                  (void *) "iterator_concurrent_update_test");
360    TEST_CHK(status == FDB_RESULT_SUCCESS);
361
362    // insert docs using db1
363    for (i=0;i<10;++i){
364        sprintf(keybuf, "key%d", i);
365        sprintf(bodybuf, "body%d", i);
366        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf), NULL, 0,
367                       (void*)bodybuf, strlen(bodybuf));
368        fdb_set(db1, doc[i]);
369    }
370    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
371
372    // create an iterator using db2
373    fdb_iterator_init(db2, &itr, NULL, 0, NULL, 0, FDB_ITR_NONE);
374    r = 0;
375    do {
376        status = fdb_iterator_get(itr, &rdoc);
377        TEST_CHK(status == FDB_RESULT_SUCCESS);
378        TEST_CMP(rdoc->key, doc[r]->key, rdoc->keylen);
379        TEST_CMP(rdoc->body, doc[r]->body, rdoc->bodylen);
380        r++;
381        fdb_doc_free(rdoc);
382        rdoc = NULL;
383    } while (fdb_iterator_next(itr) == FDB_RESULT_SUCCESS);
384    fdb_iterator_close(itr);
385    TEST_CHK(r == n);
386
387    // same for sequence number
388    fdb_iterator_sequence_init(db3, &itr, 0, 0, FDB_ITR_NONE);
389    r = 0;
390    do {
391        status = fdb_iterator_get(itr, &rdoc);
392        TEST_CHK(status == FDB_RESULT_SUCCESS);
393        r++;
394        TEST_CHK(rdoc->seqnum == (fdb_seqnum_t)r);
395        fdb_doc_free(rdoc);
396        rdoc = NULL;
397    } while (fdb_iterator_next(itr) == FDB_RESULT_SUCCESS);
398    fdb_iterator_close(itr);
399    TEST_CHK(r == n);
400
401    fdb_close(dbfile);
402
403    // free all documents
404    for (i=0;i<n;++i){
405        fdb_doc_free(doc[i]);
406    }
407
408    fdb_shutdown();
409
410    memleak_end();
411    TEST_RESULT("iterator with concurrent updates test");
412}
413
414void iterator_compact_uncommitted_db()
415{
416    TEST_INIT();
417    memleak_start();
418
419    int i, r;
420    int n = 10;
421    char keybuf[256], bodybuf[256];
422    fdb_file_handle *dbfile;
423    fdb_kvs_handle *db;
424    fdb_status status;
425    fdb_iterator *it;
426    fdb_config fconfig = fdb_get_default_config();
427    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
428    fconfig.wal_threshold = 1024;
429    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
430    fconfig.flags = FDB_OPEN_FLAG_CREATE;
431
432    // remove previous iterator_test files
433    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
434    (void)r;
435
436    // open db
437    fdb_open(&dbfile, "./iterator_test1", &fconfig);
438    fdb_kvs_open(dbfile, &db, "db1", &kvs_config);
439
440
441    // set docs
442    for (i=0;i<n;++i){
443        sprintf(keybuf, "key%d", i);
444        sprintf(bodybuf, "body%d", i);
445        fdb_set_kv(db, keybuf, strlen(keybuf),
446                bodybuf, strlen(bodybuf));
447    }
448
449    // count number of iteratable docs
450    status = fdb_iterator_sequence_init(db, &it, 0, 0, FDB_ITR_NONE);
451    TEST_CHK(status == FDB_RESULT_SUCCESS);
452    i=0;
453    do { ++i; }
454    while (fdb_iterator_next(it) != FDB_RESULT_ITERATOR_FAIL);
455    fdb_iterator_close(it);
456    TEST_CHK(i == n);
457
458    // compact
459    fdb_compact(dbfile, NULL);
460
461    // set again
462    for (i=0;i<n;++i){
463        sprintf(keybuf, "key%d", i);
464        sprintf(bodybuf, "body%d", i);
465        fdb_set_kv(db, keybuf, strlen(keybuf),
466                bodybuf, strlen(bodybuf));
467    }
468
469    // count number of iteratable docs
470    status = fdb_iterator_sequence_init(db, &it, 0, 0, FDB_ITR_NONE);
471    TEST_CHK(status == FDB_RESULT_SUCCESS);
472    i=0;
473    do { ++i; }
474    while (fdb_iterator_next(it) != FDB_RESULT_ITERATOR_FAIL);
475    fdb_iterator_close(it);
476    TEST_CHK(i == n);
477
478
479    fdb_kvs_close(db);
480    fdb_close(dbfile);
481    fdb_shutdown();
482
483    memleak_end();
484    TEST_RESULT("iterator compact uncommitted db");
485}
486
487void iterator_seek_test()
488{
489    TEST_INIT();
490
491    memleak_start();
492
493    int i, r;
494    int n = 10;
495    fdb_file_handle *dbfile;
496    fdb_kvs_handle *db, *kv1;
497    fdb_doc **doc = alca(fdb_doc*, n);
498    fdb_doc *rdoc = NULL;
499    fdb_status status;
500    fdb_iterator *iterator;
501
502    char keybuf[256], metabuf[256], bodybuf[256];
503
504    // remove previous iterator_test files
505    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
506    (void)r;
507
508    fdb_config fconfig = fdb_get_default_config();
509    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
510    fconfig.buffercache_size = 0;
511    fconfig.wal_threshold = 1024;
512    fconfig.flags = FDB_OPEN_FLAG_CREATE;
513    fconfig.compaction_threshold = 0;
514    fconfig.purging_interval = 1; // retain deletes before compaction
515
516    // open db
517    fdb_open(&dbfile, "./iterator_test1", &fconfig);
518    fdb_kvs_open_default(dbfile, &db, &kvs_config);
519    status = fdb_set_log_callback(db, logCallbackFunc,
520                                  (void *) "iterator_seek_test");
521    TEST_CHK(status == FDB_RESULT_SUCCESS);
522
523    // Create another KV store..
524    status = fdb_kvs_open(dbfile, &kv1, "kv1", &kvs_config);
525    TEST_CHK(status == FDB_RESULT_SUCCESS);
526
527    // insert using 'kv1' instance to ensure it does not interfere
528    for (i=0;i<n;++i){
529        sprintf(keybuf, "kEy%d", i);
530        sprintf(metabuf, "mEta%d", i);
531        sprintf(bodybuf, "bOdy%d", i);
532        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
533            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
534        status = fdb_set(kv1, doc[i]);
535        TEST_CHK(status == FDB_RESULT_SUCCESS);
536        fdb_doc_free(doc[i]);
537    }
538
539    // insert documents of odd number into the main (default) KV store handle
540    for (i=1;i<n;i+=2){
541        sprintf(keybuf, "key%d", i);
542        sprintf(metabuf, "meta%d", i);
543        sprintf(bodybuf, "body%d", i);
544        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
545            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
546        fdb_set(db, doc[i]);
547    }
548    // manually flush WAL & commit
549    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
550
551    // insert documents of even number
552    for (i=0;i<n;i+=2){
553        sprintf(keybuf, "key%d", i);
554        sprintf(metabuf, "meta%d", i);
555        sprintf(bodybuf, "body%d", i);
556        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
557            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
558        fdb_set(db, doc[i]);
559    }
560    // commit without WAL flush
561    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
562
563    // now odd number docs are in hb-trie & even number docs are in WAL
564
565    // create an iterator for full range
566    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, FDB_ITR_NONE);
567
568    // seek current iterator to inside the WAL's avl tree..
569    status = fdb_iterator_get(iterator, &rdoc);
570    TEST_CHK(status == FDB_RESULT_SUCCESS);
571
572    TEST_CMP(rdoc->key, doc[0]->key, rdoc->keylen);
573    TEST_CMP(rdoc->meta, doc[0]->meta, rdoc->metalen);
574    TEST_CMP(rdoc->body, doc[0]->body, rdoc->bodylen);
575    fdb_doc_free(rdoc);
576    rdoc = NULL;
577
578    // seek forward to 2nd key ..
579    i=2;
580    status = fdb_iterator_seek(iterator, doc[i]->key, strlen(keybuf), 0);
581    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
582    status = fdb_iterator_get(iterator, &rdoc);
583    TEST_CHK(status == FDB_RESULT_SUCCESS);
584
585    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
586    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
587    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
588    fdb_doc_free(rdoc);
589    rdoc = NULL;
590
591    // iterator should be able to proceed forward
592    status = fdb_iterator_next(iterator);
593    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
594    status = fdb_iterator_get(iterator, &rdoc);
595    TEST_CHK(status == FDB_RESULT_SUCCESS);
596
597    i++;
598    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
599    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
600    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
601    fdb_doc_free(rdoc);
602    rdoc = NULL;
603
604    // seek forward to the last key.
605    status = fdb_iterator_seek(iterator, doc[n-1]->key, strlen(keybuf), 0);
606    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
607    status = fdb_iterator_get(iterator, &rdoc);
608    TEST_CHK(status == FDB_RESULT_SUCCESS);
609
610    TEST_CMP(rdoc->key, doc[n-1]->key, rdoc->keylen);
611    TEST_CMP(rdoc->meta, doc[n-1]->meta, rdoc->metalen);
612    TEST_CMP(rdoc->body, doc[n-1]->body, rdoc->bodylen);
613    fdb_doc_free(rdoc);
614    rdoc = NULL;
615
616    // seek backward to start key ..
617    i = 0;
618    status = fdb_iterator_seek(iterator, doc[i]->key, strlen(keybuf), 0);
619    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
620    status = fdb_iterator_get(iterator, &rdoc);
621    TEST_CHK(status == FDB_RESULT_SUCCESS);
622
623    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
624    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
625    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
626    fdb_doc_free(rdoc);
627    rdoc = NULL;
628
629    // seek forward to key2 ..
630    status = fdb_iterator_seek(iterator, doc[2]->key, strlen(keybuf), 0);
631    TEST_CHK(status == FDB_RESULT_SUCCESS);
632
633    i=2;
634    do {
635        status = fdb_iterator_get(iterator, &rdoc);
636        TEST_CHK(status == FDB_RESULT_SUCCESS);
637
638        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
639        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
640        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
641
642        fdb_doc_free(rdoc);
643        rdoc = NULL;
644        i++;
645    } while(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
646    TEST_CHK(i==10);
647
648    // Seek backward again to a key in the trie...
649    status = fdb_iterator_seek(iterator, doc[3]->key, strlen(keybuf), 0);
650    TEST_CHK(status == FDB_RESULT_SUCCESS);
651
652    i=3;
653    do {
654        status = fdb_iterator_get(iterator, &rdoc);
655        TEST_CHK(status == FDB_RESULT_SUCCESS);
656
657        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
658        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
659        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
660
661        fdb_doc_free(rdoc);
662        rdoc = NULL;
663        i++;
664    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
665    TEST_CHK(i==10);
666
667    fdb_iterator_close(iterator);
668
669    // Test fdb_iterator_seek_to_max with key range
670    // create an iterator for range of doc[4] ~ doc[8]
671    sprintf(keybuf, "key%d", 4); // reuse buffer for start key
672    sprintf(metabuf, "key%d", 8); // reuse buffer for end_key
673    status = fdb_iterator_init(db, &iterator, keybuf, strlen(keybuf),
674                      metabuf, strlen(metabuf),
675                      FDB_ITR_NO_DELETES);
676    TEST_CHK(status == FDB_RESULT_SUCCESS);
677    i = 8;
678    status = fdb_iterator_seek_to_max(iterator);
679    TEST_CHK(status == FDB_RESULT_SUCCESS);
680    status = fdb_iterator_get(iterator, &rdoc);
681    TEST_CHK(status == FDB_RESULT_SUCCESS);
682
683    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
684    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
685    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
686    fdb_doc_free(rdoc);
687    rdoc = NULL;
688
689    // test fdb_iterator_seek_to_min
690    i = 4;
691    status = fdb_iterator_seek_to_min(iterator);
692    TEST_CHK(status == FDB_RESULT_SUCCESS);
693    status = fdb_iterator_get(iterator, &rdoc);
694    TEST_CHK(status == FDB_RESULT_SUCCESS);
695
696    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
697    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
698    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
699    fdb_doc_free(rdoc);
700    rdoc = NULL;
701
702    status = fdb_iterator_close(iterator);
703    TEST_CHK(status == FDB_RESULT_SUCCESS);
704
705    // Test fdb_iterator_seek_to_max over full range
706    status = fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, FDB_ITR_NONE);
707    TEST_CHK(status == FDB_RESULT_SUCCESS);
708
709    i = n - 1;
710    status = fdb_iterator_seek_to_max(iterator);
711    TEST_CHK(status == FDB_RESULT_SUCCESS);
712    status = fdb_iterator_get(iterator, &rdoc);
713    TEST_CHK(status == FDB_RESULT_SUCCESS);
714
715    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
716    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
717    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
718    fdb_doc_free(rdoc);
719    rdoc = NULL;
720
721    // test fdb_iterator_seek_to_min
722    i = 0;
723    status = fdb_iterator_seek_to_min(iterator);
724    TEST_CHK(status == FDB_RESULT_SUCCESS);
725    status = fdb_iterator_get(iterator, &rdoc);
726    TEST_CHK(status == FDB_RESULT_SUCCESS);
727
728    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
729    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
730    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
731    fdb_doc_free(rdoc);
732    rdoc = NULL;
733
734    // seek to max using a non-existent FFFF key
735    if (status == FDB_RESULT_SUCCESS) {
736        uint8_t *big_seek_key = alca(uint8_t, FDB_MAX_KEYLEN);
737        memset(big_seek_key, 0xff, FDB_MAX_KEYLEN);
738        status = fdb_iterator_seek(iterator, big_seek_key, FDB_MAX_KEYLEN,
739                                   FDB_ITR_SEEK_LOWER);
740        TEST_CHK(status == FDB_RESULT_SUCCESS);
741        i = n - 1;
742        status = fdb_iterator_get(iterator, &rdoc);
743        TEST_CHK(status == FDB_RESULT_SUCCESS);
744
745        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
746        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
747        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
748        fdb_doc_free(rdoc);
749        rdoc = NULL;
750    }
751
752    status = fdb_iterator_close(iterator);
753    TEST_CHK(status == FDB_RESULT_SUCCESS);
754
755    // delete documents of even number so WAL only has deleted docs
756    for (i=0;i<n;i+=2){
757        status = fdb_del(db, doc[i]);
758        TEST_CHK(status == FDB_RESULT_SUCCESS);
759    }
760    // commit without WAL flush
761    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
762
763    // create an iterator for full range
764    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, FDB_ITR_NONE);
765
766    // seek forward to 2nd key ..
767    rdoc = NULL;
768    i=2;
769    status = fdb_iterator_seek(iterator, doc[i]->key, strlen(keybuf), 0);
770    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
771    status = fdb_iterator_get(iterator, &rdoc);
772    TEST_CHK(status == FDB_RESULT_SUCCESS);
773
774    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
775    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
776    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
777    fdb_doc_free(rdoc);
778    rdoc = NULL;
779
780    status = fdb_iterator_close(iterator);
781    TEST_CHK(status == FDB_RESULT_SUCCESS);
782
783    // close the kvs kv1
784    fdb_kvs_close(kv1);
785    // close db file
786    fdb_kvs_close(db);
787    fdb_close(dbfile);
788
789    // free all documents
790    for (i=0;i<n;++i){
791        fdb_doc_free(doc[i]);
792    }
793
794    // free all resources
795    fdb_shutdown();
796
797    memleak_end();
798
799    TEST_RESULT("iterator seek test");
800}
801
802void iterator_complete_test(int insert_opt, int delete_opt)
803{
804    TEST_INIT();
805
806    int n = 30;
807    int i, r, c;
808    int *doc_status = alca(int, n); // 0:HB+trie, 1:WAL, 2:deleted
809    char cmd[256];
810    char key[256], value[256];
811    char keystr[] = "key%06d";
812    char keystr_mid[] = "key%06d+";
813    char valuestr[] = "value%08d";
814    char valuestr2[] = "value%08d(WAL)";
815    fdb_file_handle *dbfile;
816    fdb_kvs_handle *db, *db_prev, *db_next;
817    fdb_config config;
818    fdb_kvs_config kvs_config;
819    fdb_doc *doc = NULL;
820    fdb_iterator *fit;
821    fdb_iterator_opt_t itr_opt;
822    fdb_status s;
823    uint64_t mask = 0x11111111111; //0x11111111111
824
825    sprintf(cmd, SHELL_DEL " iterator_test*");
826    r = system(cmd);
827    (void)r;
828
829    memleak_start();
830
831    config = fdb_get_default_config();
832    config.purging_interval = 1; // retain deletes before compaction
833    kvs_config = fdb_get_default_kvs_config();
834    s = fdb_open(&dbfile, "./iterator_test", &config);
835    TEST_CHK(s == FDB_RESULT_SUCCESS);
836
837    s = fdb_kvs_open(dbfile, &db_prev, "prev KVS", &kvs_config);
838    TEST_CHK(s == FDB_RESULT_SUCCESS);
839    s = fdb_kvs_open(dbfile, &db, "cur KVS", &kvs_config);
840    TEST_CHK(s == FDB_RESULT_SUCCESS);
841    s = fdb_kvs_open(dbfile, &db_next, "next KVS", &kvs_config);
842    TEST_CHK(s == FDB_RESULT_SUCCESS);
843
844    sprintf(key, "prev_key");
845    sprintf(value, "prev_value");
846    s = fdb_set_kv(db_prev, key, strlen(key)+1, value, strlen(value)+1);
847    TEST_CHK(s == FDB_RESULT_SUCCESS);
848
849    sprintf(key, "next_key");
850    sprintf(value, "next_value");
851    s = fdb_set_kv(db_next, key, strlen(key)+1, value, strlen(value)+1);
852    TEST_CHK(s == FDB_RESULT_SUCCESS);
853
854    sprintf(key, "next_key2");
855    sprintf(value, "next_value2");
856    s = fdb_set_kv(db_next, key, strlen(key)+1, value, strlen(value)+1);
857    TEST_CHK(s == FDB_RESULT_SUCCESS);
858
859    s = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
860    TEST_CHK(s == FDB_RESULT_SUCCESS);
861
862    if (insert_opt == 0) {
863        // HB+trie contains all keys
864        // WAL contains even number keys only
865        for (i=0;i<n;++i){
866            sprintf(key, keystr, (int)i);
867            sprintf(value, valuestr, (int)i);
868            s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
869            TEST_CHK(s == FDB_RESULT_SUCCESS);
870            doc_status[i] = 0;
871        }
872        s = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
873        TEST_CHK(s == FDB_RESULT_SUCCESS);
874        if (delete_opt) {
875            // remove doc #15
876            i = 15;
877            sprintf(key, keystr, (int)i);
878            s = fdb_del_kv(db, key, strlen(key)+1);
879            TEST_CHK(s == FDB_RESULT_SUCCESS);
880            s = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
881            TEST_CHK(s == FDB_RESULT_SUCCESS);
882            doc_status[i] = 2;
883        }
884
885        for (i=0;i<n;i+=2){
886            sprintf(key, keystr, (int)i);
887            sprintf(value, valuestr2, (int)i);
888            s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
889            TEST_CHK(s == FDB_RESULT_SUCCESS);
890            doc_status[i] = 1;
891        }
892        s = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
893        TEST_CHK(s == FDB_RESULT_SUCCESS);
894    } else if (insert_opt == 1) {
895        // HB+trie contains all keys
896        // WAL contains odd number keys only
897        for (i=0;i<n;++i){
898            sprintf(key, keystr, (int)i);
899            sprintf(value, valuestr, (int)i);
900            s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
901            TEST_CHK(s == FDB_RESULT_SUCCESS);
902            doc_status[i] = 0;
903        }
904        s = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
905        TEST_CHK(s == FDB_RESULT_SUCCESS);
906        for (i=1;i<n;i+=2){
907            sprintf(key, keystr, (int)i);
908            sprintf(value, valuestr2, (int)i);
909            s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
910            TEST_CHK(s == FDB_RESULT_SUCCESS);
911            doc_status[i] = 1;
912        }
913        if (delete_opt) {
914            // remove doc #15
915            i = 15;
916            sprintf(key, keystr, (int)i);
917            s = fdb_del_kv(db, key, strlen(key)+1);
918            TEST_CHK(s == FDB_RESULT_SUCCESS);
919            doc_status[i] = 2;
920        }
921        s = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
922        TEST_CHK(s == FDB_RESULT_SUCCESS);
923    } else if (insert_opt == 2) {
924        // HB+trie contains odd number keys
925        // WAL contains even number keys
926        for (i=1;i<n;i+=2){
927            sprintf(key, keystr, (int)i);
928            sprintf(value, valuestr, (int)i);
929            s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
930            TEST_CHK(s == FDB_RESULT_SUCCESS);
931            doc_status[i] = 0;
932        }
933        s = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
934        TEST_CHK(s == FDB_RESULT_SUCCESS);
935        if (delete_opt) {
936            // remove doc #15
937            i = 15;
938            sprintf(key, keystr, (int)i);
939            s = fdb_del_kv(db, key, strlen(key)+1);
940            TEST_CHK(s == FDB_RESULT_SUCCESS);
941            s = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
942            TEST_CHK(s == FDB_RESULT_SUCCESS);
943            doc_status[i] = 2;
944        }
945
946        for (i=0;i<n;i+=2){
947            sprintf(key, keystr, (int)i);
948            sprintf(value, valuestr2, (int)i);
949            s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
950            TEST_CHK(s == FDB_RESULT_SUCCESS);
951            doc_status[i] = 1;
952        }
953        s = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
954        TEST_CHK(s == FDB_RESULT_SUCCESS);
955    } else if (insert_opt == 3) {
956        // HB+trie contains even number keys
957        // WAL contains odd number keys
958        for (i=0;i<n;i+=2){
959            sprintf(key, keystr, (int)i);
960            sprintf(value, valuestr, (int)i);
961            s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
962            TEST_CHK(s == FDB_RESULT_SUCCESS);
963            doc_status[i] = 0;
964        }
965        s = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
966        TEST_CHK(s == FDB_RESULT_SUCCESS);
967        for (i=1;i<n;i+=2){
968            sprintf(key, keystr, (int)i);
969            sprintf(value, valuestr2, (int)i);
970            s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
971            TEST_CHK(s == FDB_RESULT_SUCCESS);
972            doc_status[i] = 1;
973        }
974        if (delete_opt) {
975            // remove doc #15
976            i = 15;
977            sprintf(key, keystr, (int)i);
978            s = fdb_del_kv(db, key, strlen(key)+1);
979            TEST_CHK(s == FDB_RESULT_SUCCESS);
980            doc_status[i] = 2;
981        }
982        s = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
983        TEST_CHK(s == FDB_RESULT_SUCCESS);
984    } else if (insert_opt == 4) {
985        // HB+trie contains all keys
986        // WAL is empty
987        for (i=0;i<n;i+=1){
988            sprintf(key, keystr, (int)i);
989            sprintf(value, valuestr, (int)i);
990            s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
991            TEST_CHK(s == FDB_RESULT_SUCCESS);
992            doc_status[i] = 0;
993        }
994        s = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
995        TEST_CHK(s == FDB_RESULT_SUCCESS);
996        if (delete_opt) {
997            // remove doc #15
998            i = 15;
999            sprintf(key, keystr, (int)i);
1000            s = fdb_del_kv(db, key, strlen(key)+1);
1001            TEST_CHK(s == FDB_RESULT_SUCCESS);
1002            doc_status[i] = 2;
1003        }
1004        s = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1005        TEST_CHK(s == FDB_RESULT_SUCCESS);
1006    } else if (insert_opt == 5) {
1007        // HB+trie is empty
1008        // WAL contains all keys
1009        for (i=0;i<n;i+=1){
1010            sprintf(key, keystr, (int)i);
1011            sprintf(value, valuestr, (int)i);
1012            s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
1013            TEST_CHK(s == FDB_RESULT_SUCCESS);
1014            doc_status[i] = 0;
1015        }
1016        s = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1017        TEST_CHK(s == FDB_RESULT_SUCCESS);
1018        if (delete_opt) {
1019            // remove doc #15
1020            i = 15;
1021            sprintf(key, keystr, (int)i);
1022            s = fdb_del_kv(db, key, strlen(key)+1);
1023            TEST_CHK(s == FDB_RESULT_SUCCESS);
1024            doc_status[i] = 2;
1025        }
1026        s = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1027        TEST_CHK(s == FDB_RESULT_SUCCESS);
1028    } else { // if (insert_opt == 6) {
1029        // Both HB+trie and WAL contains all keys
1030        for (i=0;i<n;++i){
1031            sprintf(key, keystr, (int)i);
1032            sprintf(value, valuestr, (int)i);
1033            s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
1034            TEST_CHK(s == FDB_RESULT_SUCCESS);
1035            doc_status[i] = 0;
1036        }
1037        s = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1038        TEST_CHK(s == FDB_RESULT_SUCCESS);
1039        for (i=0;i<n;i++){
1040            sprintf(key, keystr, (int)i);
1041            sprintf(value, valuestr2, (int)i);
1042            s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
1043            TEST_CHK(s == FDB_RESULT_SUCCESS);
1044            doc_status[i] = 1;
1045        }
1046        if (delete_opt) {
1047            // remove doc #15
1048            i = 15;
1049            sprintf(key, keystr, (int)i);
1050            s = fdb_del_kv(db, key, strlen(key)+1);
1051            TEST_CHK(s == FDB_RESULT_SUCCESS);
1052            doc_status[i] = 2;
1053        }
1054        s = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1055        TEST_CHK(s == FDB_RESULT_SUCCESS);
1056    }
1057
1058    if (delete_opt) {
1059        itr_opt = FDB_ITR_NO_DELETES;
1060    } else {
1061        itr_opt = FDB_ITR_NONE;
1062    }
1063
1064    s = fdb_iterator_init(db, &fit, NULL, 0, NULL, 0, itr_opt);
1065    TEST_CHK(s == FDB_RESULT_SUCCESS);
1066    if (mask & 0x1) {
1067        c = 0;
1068        do {
1069            s = fdb_iterator_get(fit, &doc);
1070            if (s != FDB_RESULT_SUCCESS) {
1071                if (s == FDB_RESULT_KEY_NOT_FOUND) {
1072                    continue;
1073                } else {
1074                    break;
1075                }
1076            }
1077            c += ((doc_status[c] == 2)?(1):(0));
1078            sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1079            TEST_CMP(doc->body, value, doc->bodylen);
1080            fdb_doc_free(doc);
1081            doc = NULL;
1082            c++;
1083        } while (fdb_iterator_next(fit) != FDB_RESULT_ITERATOR_FAIL);
1084        TEST_CHK(c == n);
1085        while (fdb_iterator_prev(fit) != FDB_RESULT_ITERATOR_FAIL) {
1086            c--;
1087            s = fdb_iterator_get(fit, &doc);
1088            if (s != FDB_RESULT_SUCCESS) {
1089                if (s == FDB_RESULT_KEY_NOT_FOUND) {
1090                    continue;
1091                } else {
1092                    break;
1093                }
1094            }
1095            c -= ((doc_status[c] == 2)?(1):(0));
1096            sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1097            TEST_CMP(doc->body, value, doc->bodylen);
1098            fdb_doc_free(doc);
1099            doc = NULL;
1100        }
1101        TEST_CHK(c == 0);
1102    }
1103
1104    if (mask & 0x10) {
1105        for (i=0;i<n;++i){
1106            sprintf(key, keystr, (int)i);
1107            s = fdb_iterator_seek(fit, key, strlen(key)+1, 0x0);
1108            TEST_CHK(s == FDB_RESULT_SUCCESS);
1109            c = i;
1110            if (doc_status[c] == 2) {
1111                c++; // higher mode
1112            }
1113            do {
1114                s = fdb_iterator_get(fit, &doc);
1115                if (s != FDB_RESULT_SUCCESS) break;
1116                sprintf(key, keystr, (int)c);
1117                c += ((doc_status[c] == 2)?(1):(0));
1118                sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1119                TEST_CMP(doc->body, value, doc->bodylen);
1120                fdb_doc_free(doc);
1121                doc = NULL;
1122                c++;
1123            } while (fdb_iterator_next(fit) != FDB_RESULT_ITERATOR_FAIL);
1124            TEST_CHK(c == n);
1125        }
1126    }
1127
1128    if (mask & 0x100) {
1129        for (i=0;i<n;++i){
1130            sprintf(key, keystr, (int)i);
1131            s = fdb_iterator_seek(fit, key, strlen(key)+1, 0x0);
1132            TEST_CHK(s == FDB_RESULT_SUCCESS);
1133            c = i;
1134            if (doc_status[c] == 2) {
1135                c++; // higher mode
1136            }
1137            do {
1138                s = fdb_iterator_get(fit, &doc);
1139                if (s != FDB_RESULT_SUCCESS) break;
1140                sprintf(key, keystr, (int)c);
1141                c -= ((doc_status[c] == 2)?(1):(0));
1142                sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1143                TEST_CMP(doc->body, value, doc->bodylen);
1144                fdb_doc_free(doc);
1145                doc = NULL;
1146                c--;
1147            } while (fdb_iterator_prev(fit) != FDB_RESULT_ITERATOR_FAIL);
1148            TEST_CHK(c == -1);
1149        }
1150    }
1151
1152    if (mask & 0x1000) {
1153        for (i=0;i<n;++i){
1154            sprintf(key, keystr, (int)i);
1155            s = fdb_iterator_seek(fit, key, strlen(key)+1, FDB_ITR_SEEK_LOWER);
1156            TEST_CHK(s == FDB_RESULT_SUCCESS);
1157            c = i;
1158            if (doc_status[c] == 2) {
1159                c--; // lower mode
1160            }
1161            do {
1162                s = fdb_iterator_get(fit, &doc);
1163                if (s != FDB_RESULT_SUCCESS) break;
1164                sprintf(key, keystr, (int)c);
1165                c += ((doc_status[c] == 2)?(1):(0));
1166                sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1167                TEST_CMP(doc->body, value, doc->bodylen);
1168                fdb_doc_free(doc);
1169                doc = NULL;
1170                c++;
1171            } while (fdb_iterator_next(fit) != FDB_RESULT_ITERATOR_FAIL);
1172            TEST_CHK(c == n);
1173        }
1174    }
1175
1176    if (mask & 0x10000) {
1177        for (i=0;i<n;++i){
1178            sprintf(key, keystr, (int)i);
1179            s = fdb_iterator_seek(fit, key, strlen(key)+1, FDB_ITR_SEEK_LOWER);
1180            TEST_CHK(s == FDB_RESULT_SUCCESS);
1181            c = i;
1182            if (doc_status[c] == 2) {
1183                c--; // lower mode
1184            }
1185            do {
1186                s = fdb_iterator_get(fit, &doc);
1187                if (s != FDB_RESULT_SUCCESS) break;
1188                sprintf(key, keystr, (int)c);
1189                c -= ((doc_status[c] == 2)?(1):(0));
1190                sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1191                TEST_CMP(doc->body, value, doc->bodylen);
1192                fdb_doc_free(doc);
1193                doc = NULL;
1194                c--;
1195            } while (fdb_iterator_prev(fit) != FDB_RESULT_ITERATOR_FAIL);
1196            TEST_CHK(c == -1);
1197        }
1198    }
1199
1200    if (mask & 0x100000) {
1201        for (i=0;i<n;++i){
1202            sprintf(key, keystr_mid, (int)i);
1203            s = fdb_iterator_seek(fit, key, strlen(key)+1, 0x0);
1204            (void)s;
1205            c = i+1;
1206            if (doc_status[c] == 2) {
1207                c++; // higher mode
1208            }
1209            do {
1210                s = fdb_iterator_get(fit, &doc);
1211                if (s != FDB_RESULT_SUCCESS) break;
1212                sprintf(key, keystr_mid, (int)c);
1213                c += ((doc_status[c] == 2)?(1):(0));
1214                sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1215                TEST_CMP(doc->body, value, doc->bodylen);
1216                fdb_doc_free(doc);
1217                doc = NULL;
1218                c++;
1219            } while (fdb_iterator_next(fit) != FDB_RESULT_ITERATOR_FAIL);
1220            TEST_CHK(c == n);
1221        }
1222    }
1223
1224    if (mask & 0x1000000) {
1225        for (i=0;i<n;++i){
1226            sprintf(key, keystr_mid, (int)i);
1227            s = fdb_iterator_seek(fit, key, strlen(key)+1, 0x0);
1228            (void)s;
1229            c = i+1;
1230            if (doc_status[c] == 2) {
1231                c++; // higher mode
1232            }
1233            do {
1234                s = fdb_iterator_get(fit, &doc);
1235                if (s != FDB_RESULT_SUCCESS) break;
1236                sprintf(key, keystr_mid, (int)c);
1237                c -= ((doc_status[c] == 2)?(1):(0));
1238                sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1239                TEST_CMP(doc->body, value, doc->bodylen);
1240                fdb_doc_free(doc);
1241                doc = NULL;
1242                c--;
1243            } while (fdb_iterator_prev(fit) != FDB_RESULT_ITERATOR_FAIL);
1244            if (i == n-1) {
1245                TEST_CHK(c == n);
1246            } else {
1247                TEST_CHK(c == -1);
1248            }
1249        }
1250    }
1251
1252    if (mask & 0x10000000) {
1253        for (i=0;i<n;++i){
1254            sprintf(key, keystr_mid, (int)i);
1255            s = fdb_iterator_seek(fit, key, strlen(key)+1, FDB_ITR_SEEK_LOWER);
1256            TEST_CHK(s == FDB_RESULT_SUCCESS);
1257            c = i;
1258            if (doc_status[c] == 2) {
1259                c--; // lower mode
1260            }
1261            do {
1262                s = fdb_iterator_get(fit, &doc);
1263                if (s != FDB_RESULT_SUCCESS) break;
1264                sprintf(key, keystr_mid, (int)c);
1265                c += ((doc_status[c] == 2)?(1):(0));
1266                sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1267                TEST_CMP(doc->body, value, doc->bodylen);
1268                fdb_doc_free(doc);
1269                doc = NULL;
1270                c++;
1271            } while (fdb_iterator_next(fit) != FDB_RESULT_ITERATOR_FAIL);
1272            TEST_CHK(c == n);
1273        }
1274    }
1275
1276    if (mask & 0x100000000) {
1277        for (i=0;i<n;++i){
1278            sprintf(key, keystr_mid, (int)i);
1279            s = fdb_iterator_seek(fit, key, strlen(key)+1, FDB_ITR_SEEK_LOWER);
1280            TEST_CHK(s == FDB_RESULT_SUCCESS);
1281            c = i;
1282            if (doc_status[c] == 2) {
1283                c--; // lower mode
1284            }
1285            do {
1286                s = fdb_iterator_get(fit, &doc);
1287                if (s != FDB_RESULT_SUCCESS) break;
1288                sprintf(key, keystr_mid, (int)c);
1289                c -= ((doc_status[c] == 2)?(1):(0));
1290                sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1291                TEST_CMP(doc->body, value, doc->bodylen);
1292                fdb_doc_free(doc);
1293                doc = NULL;
1294                c--;
1295            } while (fdb_iterator_prev(fit) != FDB_RESULT_ITERATOR_FAIL);
1296            TEST_CHK(c == -1);
1297        }
1298    }
1299
1300    if (mask & 0x1000000000) {
1301        s = fdb_iterator_seek_to_min(fit);
1302        TEST_CHK(s == FDB_RESULT_SUCCESS);
1303        c = 0;
1304        do {
1305            s = fdb_iterator_get(fit, &doc);
1306            if (s != FDB_RESULT_SUCCESS) break;
1307            sprintf(key, keystr_mid, (int)c);
1308            c += ((doc_status[c] == 2)?(1):(0));
1309            sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1310            TEST_CMP(doc->body, value, doc->bodylen);
1311            fdb_doc_free(doc);
1312            doc = NULL;
1313            c++;
1314        } while (fdb_iterator_next(fit) != FDB_RESULT_ITERATOR_FAIL);
1315        TEST_CHK(c == n);
1316
1317        s = fdb_iterator_seek_to_max(fit);
1318        TEST_CHK(s == FDB_RESULT_SUCCESS);
1319        c = n-1;
1320        do {
1321            s = fdb_iterator_get(fit, &doc);
1322            if (s != FDB_RESULT_SUCCESS) break;
1323            sprintf(key, keystr_mid, (int)c);
1324            c -= ((doc_status[c] == 2)?(1):(0));
1325            sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1326            TEST_CMP(doc->body, value, doc->bodylen);
1327            fdb_doc_free(doc);
1328            doc = NULL;
1329            c--;
1330        } while (fdb_iterator_prev(fit) != FDB_RESULT_ITERATOR_FAIL);
1331        TEST_CHK(c == -1);
1332    }
1333    s = fdb_iterator_close(fit);
1334    TEST_CHK(s == FDB_RESULT_SUCCESS);
1335
1336    if (mask & 0x10000000000) {
1337        // create an iterator with an end key and skip max key option
1338        i = n/3*2;
1339        sprintf(key, keystr, (int)i);
1340        s = fdb_iterator_init(db, &fit, NULL, 0, key, strlen(key)+1,
1341                              FDB_ITR_SKIP_MAX_KEY);
1342        TEST_CHK(s == FDB_RESULT_SUCCESS);
1343        s = fdb_iterator_seek_to_max(fit);
1344        TEST_CHK(s == FDB_RESULT_SUCCESS);
1345        s = fdb_iterator_get(fit, &doc);
1346        TEST_CHK(s == FDB_RESULT_SUCCESS);
1347        c = i-1;
1348        sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1349        TEST_CMP(doc->body, value, doc->bodylen);
1350        fdb_doc_free(doc);
1351        doc = NULL;
1352        s = fdb_iterator_close(fit);
1353        TEST_CHK(s == FDB_RESULT_SUCCESS);
1354
1355        // create an iterator with an start key and skip min key option
1356        i = n/3;
1357        sprintf(key, keystr, (int)i);
1358        s = fdb_iterator_init(db, &fit, key, strlen(key)+1, NULL, 0,
1359                              FDB_ITR_SKIP_MIN_KEY);
1360        TEST_CHK(s == FDB_RESULT_SUCCESS);
1361        s = fdb_iterator_seek_to_min(fit);
1362        TEST_CHK(s == FDB_RESULT_SUCCESS);
1363        s = fdb_iterator_get(fit, &doc);
1364        TEST_CHK(s == FDB_RESULT_SUCCESS);
1365        c = i+1;
1366        sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1367        TEST_CMP(doc->body, value, doc->bodylen);
1368        fdb_doc_free(doc);
1369        doc = NULL;
1370        s = fdb_iterator_close(fit);
1371        TEST_CHK(s == FDB_RESULT_SUCCESS);
1372
1373        s = fdb_close(dbfile);
1374        TEST_CHK(s == FDB_RESULT_SUCCESS);
1375        s = fdb_shutdown();
1376        TEST_CHK(s == FDB_RESULT_SUCCESS);
1377        memleak_end();
1378    }
1379
1380    sprintf(cmd, "iterator complete test ");
1381    if (insert_opt == 0) {
1382        strcat(cmd, "(HB+trie: all, WAL: even");
1383    } else if (insert_opt == 1) {
1384        strcat(cmd, "(HB+trie: all, WAL: odd");
1385    } else if (insert_opt == 2) {
1386        strcat(cmd, "(HB+trie: odd, WAL: even");
1387    } else if (insert_opt == 3) {
1388        strcat(cmd, "(HB+trie: even, WAL: odd");
1389    } else if (insert_opt == 4) {
1390        strcat(cmd, "(HB+trie: all, WAL: empty");
1391    } else if (insert_opt == 5) {
1392        strcat(cmd, "(HB+trie: empty, WAL: all");
1393    } else if (insert_opt == 6) {
1394        strcat(cmd, "(HB+trie: all, WAL: all");
1395    }
1396    if (delete_opt) {
1397        strcat(cmd, ", doc deletion)");
1398    } else {
1399        strcat(cmd, ")");
1400    }
1401    TEST_RESULT(cmd);
1402}
1403
1404void iterator_extreme_key_test()
1405{
1406    TEST_INIT();
1407
1408    int n = 30;
1409    int i, r, c;
1410    char cmd[256];
1411    char key[256], value[256];
1412    char keyBuf[256], valueBuf[256];
1413    fdb_file_handle *dbfile;
1414    fdb_kvs_handle *db;
1415    fdb_config config;
1416    fdb_kvs_config kvs_config;
1417    fdb_doc *doc = NULL;
1418    fdb_iterator *fit;
1419    fdb_status s;
1420
1421    sprintf(cmd, SHELL_DEL " iterator_test*");
1422    r = system(cmd);
1423    (void)r;
1424
1425    memleak_start();
1426
1427    config = fdb_get_default_config();
1428    kvs_config = fdb_get_default_kvs_config();
1429    s = fdb_open(&dbfile, "./iterator_test", &config);
1430    TEST_CHK(s == FDB_RESULT_SUCCESS);
1431    s = fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
1432    TEST_CHK(s == FDB_RESULT_SUCCESS);
1433
1434    memset(key, 0xff, 256);
1435    for (i=1;i<n;i+=1){
1436        sprintf(value, "0xff length %d", (int)i);
1437        fdb_set_kv(db, key, i, value, strlen(value)+1);
1438    }
1439    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1440
1441    // Pre-allocate iterator return document memory and re-use the same
1442    s = fdb_doc_create(&doc, NULL, 0, NULL, 0, NULL, 0);
1443    TEST_CHK(s == FDB_RESULT_SUCCESS);
1444    doc->key = &keyBuf[0];
1445    doc->meta = NULL;
1446    doc->body = &valueBuf[0];
1447
1448    s = fdb_iterator_init(db, &fit, NULL, 0, NULL, 0, 0x0);
1449    TEST_CHK(s == FDB_RESULT_SUCCESS);
1450    s = fdb_iterator_seek_to_max(fit);
1451    TEST_CHK(s == FDB_RESULT_SUCCESS);
1452    c = n-1;
1453    while (s == FDB_RESULT_SUCCESS) {
1454        s = fdb_iterator_get(fit, &doc);
1455        if (s != FDB_RESULT_SUCCESS) {
1456            break;
1457        }
1458        sprintf(value, "0xff length %d", (int)c);
1459        TEST_CMP(doc->body, value, doc->bodylen);
1460        s = fdb_iterator_prev(fit);
1461        c--;
1462    }
1463
1464    i = 8;
1465    c = i;
1466    s = fdb_iterator_seek(fit, key, i, FDB_ITR_SEEK_LOWER);
1467    while (s == FDB_RESULT_SUCCESS) {
1468        s = fdb_iterator_get(fit, &doc);
1469        if (s != FDB_RESULT_SUCCESS) {
1470            break;
1471        }
1472        sprintf(value, "0xff length %d", (int)c);
1473        TEST_CMP(doc->body, value, doc->bodylen);
1474        s = fdb_iterator_prev(fit);
1475        c--;
1476    }
1477
1478    i = 8;
1479    c = i;
1480    s = fdb_iterator_seek(fit, key, i, FDB_ITR_SEEK_LOWER);
1481    while (s == FDB_RESULT_SUCCESS) {
1482        s = fdb_iterator_get(fit, &doc);
1483        if (s != FDB_RESULT_SUCCESS) {
1484            break;
1485        }
1486        sprintf(value, "0xff length %d", (int)c);
1487        TEST_CMP(doc->body, value, doc->bodylen);
1488        s = fdb_iterator_next(fit);
1489        c++;
1490    }
1491
1492    s = fdb_iterator_close(fit);
1493    TEST_CHK(s == FDB_RESULT_SUCCESS);
1494
1495    // Initialize iterator to an extreme non-existent end_key
1496    s = fdb_iterator_init(db, &fit, NULL, 0, key, 256, 0x0);
1497    TEST_CHK(s == FDB_RESULT_SUCCESS);
1498    s = fdb_iterator_seek_to_max(fit);
1499    TEST_CHK(s == FDB_RESULT_SUCCESS);
1500
1501    c = n-1;
1502    while (s == FDB_RESULT_SUCCESS) {
1503        s = fdb_iterator_get(fit, &doc);
1504        if (s != FDB_RESULT_SUCCESS) {
1505            break;
1506        }
1507        sprintf(value, "0xff length %d", (int)c);
1508        TEST_CMP(doc->body, value, doc->bodylen);
1509        s = fdb_iterator_prev(fit);
1510        c--;
1511    }
1512
1513    s = fdb_iterator_close(fit);
1514    TEST_CHK(s == FDB_RESULT_SUCCESS);
1515    s = fdb_close(dbfile);
1516    TEST_CHK(s == FDB_RESULT_SUCCESS);
1517
1518    // Release pre-allocated iterator return document buffer space
1519    doc->key = NULL;
1520    doc->body = NULL;
1521    s = fdb_doc_free(doc);
1522    TEST_CHK(s == FDB_RESULT_SUCCESS);
1523
1524    s = fdb_shutdown();
1525    TEST_CHK(s == FDB_RESULT_SUCCESS);
1526    memleak_end();
1527    TEST_RESULT("iterator extreme key test");
1528}
1529
1530void iterator_inmem_snapshot_seek_test(bool flush_wal)
1531{
1532    TEST_INIT();
1533
1534    memleak_start();
1535
1536    int i, r;
1537    int n = 5;
1538    fdb_file_handle *dbfile;
1539    fdb_kvs_handle *db;
1540    fdb_kvs_handle *snap_db;
1541    fdb_doc **doc = alca(fdb_doc*, n);
1542    fdb_status status;
1543    fdb_iterator *iterator;
1544
1545    char keybuf[256], metabuf[256], bodybuf[256];
1546
1547    // remove previous mvcc_test files
1548    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
1549    (void)r;
1550
1551    fdb_config fconfig = fdb_get_default_config();
1552    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1553    fconfig.buffercache_size = 0;
1554    fconfig.wal_threshold = 1024;
1555    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1556    fconfig.compaction_threshold = 0;
1557
1558    // open db
1559    status = fdb_open(&dbfile, "./iterator_test1", &fconfig);
1560    TEST_CHK(status == FDB_RESULT_SUCCESS);
1561    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
1562    TEST_CHK(status == FDB_RESULT_SUCCESS);
1563
1564    // ------- Setup test ----------------------------------
1565    for (i=0; i<n; i++){
1566        sprintf(keybuf, "%c2",(char)i + 'a');
1567        sprintf(metabuf, "meta%d", i);
1568        sprintf(bodybuf, "body%d", i);
1569        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1570            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1571        fdb_set(db, doc[i]);
1572    }
1573
1574    if (flush_wal) {
1575        fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1576    } else {
1577        fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1578    }
1579    // ---------- Snapshot tests begin -----------------------
1580    // WAL items are not flushed...
1581    status = fdb_snapshot_open(db, &snap_db, FDB_SNAPSHOT_INMEM);
1582    TEST_CHK(status == FDB_RESULT_SUCCESS);
1583
1584    status = fdb_set_log_callback(db, logCallbackFunc,
1585                                  (void *) "iterator_inmem_snapshot_seek_test");
1586    TEST_CHK(status == FDB_RESULT_SUCCESS);
1587
1588    // create an iterator on the snapshot for full range
1589    fdb_iterator_init(snap_db, &iterator, (void*)"b2", 2, (void*)"d2", 2,
1590                      FDB_ITR_NO_DELETES|
1591                      FDB_ITR_SKIP_MAX_KEY|
1592                      FDB_ITR_SKIP_MIN_KEY);
1593
1594    // seek to non-existent key that happens to land on the start key which
1595    // should not be returned since we have passed ITR_SKIP_MIN_KEY
1596    status = fdb_iterator_seek(iterator, "c1", 2, FDB_ITR_SEEK_LOWER);
1597    TEST_CHK(status == FDB_RESULT_ITERATOR_FAIL);
1598
1599    // seek to non-existent key that happens to land on the end key which
1600    // should not be returned since we have passed ITR_SKIP_MAX_KEY
1601    status = fdb_iterator_seek(iterator, "c3", 2, FDB_ITR_SEEK_HIGHER);
1602    TEST_CHK(status == FDB_RESULT_ITERATOR_FAIL);
1603
1604    fdb_iterator_close(iterator);
1605
1606    // create an iterator on the snapshot for full range
1607    fdb_iterator_init(snap_db, &iterator, (void*)"b3", 2, (void*)"d1", 2,
1608                      FDB_ITR_NO_DELETES|
1609                      FDB_ITR_SKIP_MAX_KEY|
1610                      FDB_ITR_SKIP_MIN_KEY);
1611
1612    // seek to non-existent key that happens to land on key that is
1613    // smaller than the start key which should not be returned.
1614    status = fdb_iterator_seek(iterator, "c1", 2, FDB_ITR_SEEK_LOWER);
1615    TEST_CHK(status == FDB_RESULT_ITERATOR_FAIL);
1616
1617    // seek to non-existent key that happens to be land on key larger than
1618    // end key which should not be returned.
1619    status = fdb_iterator_seek(iterator, "c3", 2, FDB_ITR_SEEK_HIGHER);
1620    TEST_CHK(status == FDB_RESULT_ITERATOR_FAIL);
1621
1622    fdb_iterator_close(iterator);
1623
1624    // close db handle
1625    fdb_kvs_close(db);
1626    // close snapshot handle
1627    fdb_kvs_close(snap_db);
1628
1629    // close db file
1630    fdb_close(dbfile);
1631
1632    // free all documents
1633    for (i=0;i<n;++i){
1634        fdb_doc_free(doc[i]);
1635    }
1636
1637    // free all resources
1638    fdb_shutdown();
1639
1640    memleak_end();
1641
1642    TEST_RESULT("in-memory snapshot seek test");
1643}
1644
1645void iterator_no_deletes_test()
1646{
1647
1648    TEST_INIT();
1649    memleak_start();
1650    int i, r, n = 10;
1651    fdb_file_handle *dbfile;
1652    fdb_kvs_handle  *kv;
1653    char keybuf[256], bodybuf[256];
1654    fdb_doc **doc = alca(fdb_doc*, n);
1655    fdb_doc *rdoc = NULL;
1656    fdb_iterator *it;
1657    fdb_status status;
1658
1659    // remove previous iterator_test files
1660    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
1661    (void)r;
1662
1663    fdb_config fconfig = fdb_get_default_config();
1664    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1665    fconfig.wal_threshold = 1024;
1666    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1667    fconfig.compaction_threshold = 0;
1668
1669    // open db
1670    fdb_open(&dbfile, "./iterator_test", &fconfig);
1671    fdb_kvs_open(dbfile, &kv, "all_docs",  &kvs_config);
1672
1673    // insert docs to kv
1674    for (i=0;i<n;++i){
1675        sprintf(keybuf, "key%d", i);
1676        sprintf(bodybuf, "body%d", i);
1677        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf), NULL, 0,
1678                       (void*)bodybuf, strlen(bodybuf));
1679        fdb_set(kv, doc[i]);
1680    }
1681
1682    // delete all docs
1683    for (i=0;i<n;i++){
1684        status = fdb_del_kv(kv, doc[i]->key, doc[i]->keylen);
1685        TEST_CHK(status == FDB_RESULT_SUCCESS);
1686    }
1687
1688    // commit
1689    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1690
1691    // set doc that was deleted
1692    status = fdb_set(kv, doc[2]);
1693    TEST_CHK(status == FDB_RESULT_SUCCESS);
1694
1695    // get doc from db and verify not-deleted
1696    fdb_doc_create(&rdoc, doc[2]->key, doc[2]->keylen, NULL, 0, NULL, 0);
1697    status = fdb_get(kv, rdoc);
1698    TEST_CHK(status == FDB_RESULT_SUCCESS);
1699    TEST_CHK(rdoc->deleted == false);
1700    fdb_doc_free(rdoc);
1701    rdoc = NULL;
1702
1703    // iterate over all docs to retrieve undeleted key
1704    status = fdb_iterator_init(kv, &it, NULL, 0, NULL, 0, FDB_ITR_NO_DELETES);
1705    TEST_CHK(status == FDB_RESULT_SUCCESS);
1706    status = fdb_iterator_get(it, &rdoc);
1707    TEST_CHK(status == FDB_RESULT_SUCCESS);
1708    if (status == FDB_RESULT_SUCCESS){
1709        fdb_doc_free(rdoc);
1710    }
1711    fdb_iterator_close(it);
1712
1713    for (i=0;i<n;++i){
1714        fdb_doc_free(doc[i]);
1715    }
1716
1717    fdb_kvs_close(kv);
1718    fdb_close(dbfile);
1719    fdb_shutdown();
1720
1721    memleak_end();
1722    TEST_RESULT("iterator no deletes test");
1723}
1724
1725void iterator_set_del_docs_test()
1726{
1727    TEST_INIT();
1728    memleak_start();
1729
1730    int r;
1731    int i, j, k, n=100;
1732    int expected_doc_count=0;
1733    char keybuf[256], metabuf[256], bodybuf[256];
1734    int val2;
1735    fdb_file_handle *dbfile;
1736    fdb_iterator *it;
1737    fdb_kvs_handle *kv1;
1738    fdb_kvs_info info;
1739    fdb_config fconfig = fdb_get_default_config();
1740    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1741    fdb_doc **doc = alca(fdb_doc*, n);
1742    fdb_doc *vdoc;
1743    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
1744    (void)r;
1745
1746    fconfig.wal_threshold = 1024;
1747    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1748    fconfig.compaction_threshold = 10;
1749    fconfig.purging_interval = 1; //retain deletes until compaction
1750
1751    fdb_open(&dbfile, "./iterator_test1", &fconfig);
1752    fdb_kvs_open(dbfile, &kv1, "kv1", &kvs_config);
1753
1754    for(k=0;k<20;++k){
1755        // set n docs
1756        for(i=0;i<n;++i){
1757            sprintf(keybuf, "key%02d%03d", k, i);
1758            sprintf(metabuf, "meta%02d%03d", k, i);
1759            sprintf(bodybuf, "body%02d%03d", k, i);
1760            fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1761                (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1762            fdb_set(kv1, doc[i]);
1763            expected_doc_count++;
1764        }
1765
1766        // commit
1767        fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1768
1769        // delete subset of recently loaded docs
1770        for(j=n/4;j<n/2;j++){
1771            fdb_del(kv1, doc[j]);
1772            expected_doc_count--;
1773        }
1774        fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1775
1776        fdb_get_kvs_info(kv1, &info);
1777        if(info.doc_count != (size_t)expected_doc_count){
1778            // test already failed further debugging check info
1779            fdb_iterator_init(kv1, &it, NULL, 0,
1780                              NULL, 0, FDB_ITR_NONE);
1781            val2=0;
1782            do {
1783                fdb_iterator_get(it, &vdoc);
1784                if (!vdoc->deleted){
1785                    val2++;
1786                }
1787                fdb_doc_free(vdoc);
1788            } while (fdb_iterator_next(it) != FDB_RESULT_ITERATOR_FAIL);
1789            fdb_iterator_close(it);
1790            printf("dbdocs(%d) expected(%d)\n", val2, expected_doc_count);
1791        }
1792        TEST_CHK(info.doc_count == (size_t)expected_doc_count);
1793
1794        // preliminary cleanup
1795        for(i=0;i<n;++i){
1796            fdb_doc_free(doc[i]);
1797        }
1798    }
1799
1800    fdb_kvs_close(kv1);
1801    fdb_close(dbfile);
1802    fdb_shutdown();
1803    memleak_end();
1804
1805    TEST_RESULT("iterator set del docs");
1806}
1807
1808void iterator_del_next_test()
1809{
1810    TEST_INIT();
1811
1812    memleak_start();
1813
1814    int i, r;
1815    int n = 10;
1816    fdb_file_handle *dbfile;
1817    fdb_kvs_handle *db;
1818    fdb_doc **doc = alca(fdb_doc*, n);
1819    fdb_doc pre_alloc_doc;
1820    fdb_doc *rdoc = &pre_alloc_doc;
1821    fdb_status status;
1822    fdb_iterator *iterator;
1823
1824    char keybuf[256], metabuf[256], bodybuf[256];
1825    rdoc->key = keybuf;
1826    rdoc->meta = metabuf;
1827    rdoc->body = bodybuf;
1828    rdoc->flags = 0;
1829
1830    // remove previous iterator_test files
1831    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
1832    (void)r;
1833
1834    fdb_config fconfig = fdb_get_default_config();
1835    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1836    fconfig.buffercache_size = 0;
1837    fconfig.wal_threshold = 1024;
1838    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1839    fconfig.compaction_threshold = 0;
1840
1841    // open db
1842    fdb_open(&dbfile, "./iterator_test1", &fconfig);
1843    fdb_kvs_open(dbfile, &db, "kv1", &kvs_config);
1844
1845    status = fdb_set_log_callback(db, logCallbackFunc,
1846                                  (void *) "iterator_del_next_test");
1847    TEST_CHK(status == FDB_RESULT_SUCCESS);
1848
1849    // insert documents of even number
1850    for (i=0;i<n;i+=2){
1851        sprintf(keybuf, "key%d", i);
1852        sprintf(metabuf, "meta%d", i);
1853        sprintf(bodybuf, "body%d", i);
1854        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1855            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1856        fdb_set(db, doc[i]);
1857    }
1858    // manually flush WAL & commit
1859    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1860
1861    // insert documents of odd number
1862    for (i=1;i<n;i+=2){
1863        sprintf(keybuf, "key%d", i);
1864        sprintf(metabuf, "meta%d", i);
1865        sprintf(bodybuf, "body%d", i);
1866        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1867            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1868        fdb_set(db, doc[i]);
1869    }
1870    // commit without WAL flush
1871    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1872
1873    // now even number docs are in hb-trie & odd number docs are in WAL
1874
1875    // create an iterator for full range
1876    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, FDB_ITR_NONE);
1877
1878    // repeat until fail
1879    i=0;
1880    do {
1881        status = fdb_iterator_get(iterator, &rdoc);
1882        TEST_CHK(status == FDB_RESULT_SUCCESS);
1883
1884        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
1885        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
1886        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
1887        // Delete the document to ensure that the iteration is not affected
1888        status = fdb_del(db, rdoc);
1889        TEST_CHK(status == FDB_RESULT_SUCCESS);
1890        status = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1891        TEST_CHK(status == FDB_RESULT_SUCCESS);
1892        i++;
1893    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
1894    TEST_CHK(i==10);
1895
1896    // go back to start and retry iteration (should not be affected by deletes)
1897    status = fdb_iterator_seek_to_min(iterator);
1898    TEST_CHK(status == FDB_RESULT_SUCCESS);
1899
1900    // repeat full iteration until fail
1901    i=0;
1902    do {
1903        status = fdb_iterator_get(iterator, &rdoc);
1904        TEST_CHK(status == FDB_RESULT_SUCCESS);
1905
1906        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
1907        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
1908        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
1909        i++;
1910    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
1911    TEST_CHK(i==10);
1912
1913    fdb_iterator_close(iterator);
1914
1915    fdb_close(dbfile);
1916    fdb_shutdown();
1917
1918    // free all documents
1919    for (i=0;i<n;++i){
1920        fdb_doc_free(doc[i]);
1921    }
1922
1923    memleak_end();
1924
1925    TEST_RESULT("iterator del next test");
1926}
1927
1928void sequence_iterator_test()
1929{
1930    TEST_INIT();
1931
1932    memleak_start();
1933
1934    int i, r;
1935    int n = 10;
1936    int count;
1937    fdb_file_handle *dbfile;
1938    fdb_kvs_handle *db;
1939    fdb_doc **doc = alca(fdb_doc*, n);
1940    fdb_doc *rdoc = NULL;
1941    fdb_status status;
1942    fdb_iterator *iterator;
1943
1944    char keybuf[256], metabuf[256], bodybuf[256];
1945
1946    // remove previous iterator_test files
1947    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
1948    (void)r;
1949
1950    fdb_config fconfig = fdb_get_default_config();
1951    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1952    fconfig.buffercache_size = 0;
1953    fconfig.wal_threshold = 1024;
1954    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1955    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
1956    fconfig.compaction_threshold = 0;
1957    fconfig.purging_interval = 1; // retain deletes until compaction
1958
1959    // open db
1960    fdb_open(&dbfile, "./iterator_test1", &fconfig);
1961    fdb_kvs_open_default(dbfile, &db, &kvs_config);
1962    status = fdb_set_log_callback(db, logCallbackFunc,
1963                                  (void *) "sequence_iterator_test");
1964    TEST_CHK(status == FDB_RESULT_SUCCESS);
1965
1966    // insert documents of even number
1967    for (i=0;i<n;i+=2){
1968        sprintf(keybuf, "key%d", i);
1969        sprintf(metabuf, "meta%d", i);
1970        sprintf(bodybuf, "body%d", i);
1971        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1972            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1973        fdb_set(db, doc[i]);
1974    }
1975    // manually flush WAL & commit
1976    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1977
1978    // insert documents of odd number
1979    for (i=1;i<n;i+=2){
1980        sprintf(keybuf, "key%d", i);
1981        sprintf(metabuf, "meta%d", i);
1982        sprintf(bodybuf, "body%d", i);
1983        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1984            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1985        fdb_set(db, doc[i]);
1986    }
1987    // commit without WAL flush
1988    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1989
1990    // now even number docs are in hb-trie & odd number docs are in WAL
1991
1992    // create an iterator over sequence number range over FULL RANGE
1993    fdb_iterator_sequence_init(db, &iterator, 0, 0, FDB_ITR_NONE);
1994
1995    // repeat until fail
1996    i=0;
1997    count = 0;
1998    do {
1999        status = fdb_iterator_get(iterator, &rdoc);
2000        TEST_CHK(status == FDB_RESULT_SUCCESS);
2001
2002        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2003        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2004        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2005
2006        fdb_doc_free(rdoc);
2007        rdoc = NULL;
2008        i = (i + 2 >= n) ? 1: i + 2; // by-seq, first come even docs, then odd
2009        count++;
2010    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2011    TEST_CHK(count==n);
2012    fdb_iterator_close(iterator);
2013
2014    // create an iterator over sequence number.
2015    fdb_iterator_sequence_init(db, &iterator, 0, 0, FDB_ITR_NONE);
2016
2017    // repeat until fail
2018    i=0;
2019    count = 0;
2020    do {
2021        status = fdb_iterator_get_metaonly(iterator, &rdoc);
2022        TEST_CHK(status == FDB_RESULT_SUCCESS);
2023
2024        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2025        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2026        TEST_CHK(rdoc->body == NULL);
2027
2028        fdb_doc_free(rdoc);
2029        rdoc = NULL;
2030        i = (i + 2 >= n) ? 1: i + 2; // by-seq, first come even docs, then odd
2031        count++;
2032    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2033    TEST_CHK(count==n);
2034    fdb_iterator_close(iterator);
2035
2036    // create another iterator starts from seq number 2 and ends at 9
2037    fdb_iterator_sequence_init(db, &iterator, 2, 7, FDB_ITR_NONE);
2038
2039    // repeat until fail
2040    i=2;
2041    count = 0;
2042    do {
2043        status = fdb_iterator_get(iterator, &rdoc);
2044        TEST_CHK(status == FDB_RESULT_SUCCESS);
2045
2046        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2047        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2048        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2049
2050        fdb_doc_free(rdoc);
2051        rdoc = NULL;
2052        i = (i + 2 >= n) ? 1: i + 2; // by-seq, first come even docs, then odd
2053        count++;
2054    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2055    TEST_CHK(count==6);
2056    fdb_iterator_close(iterator);
2057    // remove document #8 and #9
2058    fdb_doc_create(&rdoc, doc[8]->key, doc[8]->keylen, doc[8]->meta, doc[8]->metalen, NULL, 0);
2059    status = fdb_del(db, rdoc);
2060    TEST_CHK(status == FDB_RESULT_SUCCESS);
2061    fdb_doc_free(rdoc);
2062    rdoc = NULL;
2063    fdb_doc_create(&rdoc, doc[9]->key, doc[9]->keylen, doc[9]->meta, doc[9]->metalen, NULL, 0);
2064    status = fdb_del(db, rdoc);
2065    TEST_CHK(status == FDB_RESULT_SUCCESS);
2066    fdb_doc_free(rdoc);
2067    rdoc = NULL;
2068    // commit
2069    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2070
2071    // create an iterator for full range
2072    fdb_iterator_sequence_init(db, &iterator, 0, 0, FDB_ITR_NONE);
2073    // repeat until fail
2074    i=0;
2075    count=0;
2076    do {
2077        status = fdb_iterator_get(iterator, &rdoc);
2078        TEST_CHK(status == FDB_RESULT_SUCCESS);
2079
2080        if (count != 8 && count != 9) { // do not look validate key8 and key9
2081            TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2082            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2083            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2084        } else {
2085            TEST_CHK(rdoc->deleted == true);
2086        }
2087
2088        fdb_doc_free(rdoc);
2089        rdoc = NULL;
2090        // Turn around when we hit 8 as the last items, key8 and key9 are gone
2091        i = (i + 2 >= 8) ? 1: i + 2; // by-seq, first come even docs, then odd
2092        count++;
2093    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2094    TEST_CHK(count==10); // 10 items, with 2 deletions
2095    fdb_iterator_close(iterator);
2096
2097    // create an iterator for full range, but no deletes.
2098    fdb_iterator_sequence_init(db, &iterator, 0, 0, FDB_ITR_NO_DELETES);
2099    // repeat until fail
2100    i=0;
2101    count=0;
2102    do {
2103        status = fdb_iterator_get(iterator, &rdoc);
2104        TEST_CHK(status == FDB_RESULT_SUCCESS);
2105
2106        if (i != 8 && i != 9) { // key8 and key9 are deleted
2107            TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2108            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2109            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2110        }
2111
2112        fdb_doc_free(rdoc);
2113        rdoc = NULL;
2114        i = (i + 2 >= 8) ? 1: i + 2; // by-seq, first come even docs, then odd
2115        count++;
2116    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2117    TEST_CHK(count==8); // 10 items, with 2 deletions
2118    fdb_iterator_close(iterator);
2119
2120    // Update first document and test for absence of duplicates
2121    *((char *)doc[0]->body) = 'K'; // update key0 to Key0
2122    fdb_set(db, doc[0]);
2123    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2124
2125    fdb_iterator_sequence_init(db, &iterator, 0, 0, FDB_ITR_NO_DELETES);
2126    // repeat until fail
2127    i=2; // i == 0 should not appear until the end
2128    count=0;
2129    do {
2130        status = fdb_iterator_get(iterator, &rdoc);
2131        TEST_CHK(status == FDB_RESULT_SUCCESS);
2132
2133        if (i != 8 && i != 9) { // key8 and key9 are deleted
2134            TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2135            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2136            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2137        }
2138
2139        fdb_doc_free(rdoc);
2140        rdoc = NULL;
2141        i = (i + 2 >= 8) ? 1: i + 2; // by-seq, first come even docs, then odd
2142        if (count == 6) i = 0; // go back to test for i=0 at the end
2143        count++;
2144    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2145    TEST_CHK(count==8); // 10 items, with 2 deletions
2146    fdb_iterator_close(iterator);
2147
2148    // close db file
2149    fdb_kvs_close(db);
2150    fdb_close(dbfile);
2151
2152    // free all documents
2153    for (i=0;i<n;++i){
2154        fdb_doc_free(doc[i]);
2155    }
2156
2157    // free all resources
2158    fdb_shutdown();
2159
2160    memleak_end();
2161
2162    TEST_RESULT("sequence iterator test");
2163}
2164
2165void sequence_iterator_duplicate_test()
2166{
2167    // Unit test for MB-12225
2168    TEST_INIT();
2169
2170    memleak_start();
2171
2172    int i, r;
2173    int n = 100;
2174    int count;
2175    fdb_file_handle *dbfile;
2176    fdb_kvs_handle *db;
2177    fdb_doc **doc = alca(fdb_doc*, n);
2178    fdb_doc *rdoc = NULL;
2179    fdb_status status;
2180    fdb_iterator *iterator;
2181    fdb_seqnum_t seqnum;
2182
2183    char keybuf[256], metabuf[256], bodybuf[256];
2184
2185    // remove previous iterator_test files
2186    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
2187    (void)r;
2188
2189    fdb_config fconfig = fdb_get_default_config();
2190    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2191    fconfig.buffercache_size = 0;
2192    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
2193    fconfig.wal_threshold = 1024;
2194    fconfig.compaction_threshold = 0;
2195    fconfig.purging_interval = 1; // retain deletes until compaction
2196
2197    // open db
2198    fdb_open(&dbfile, "./iterator_test1", &fconfig);
2199    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2200    status = fdb_set_log_callback(db, logCallbackFunc,
2201                                  (void *) "sequence_iterator_test");
2202    TEST_CHK(status == FDB_RESULT_SUCCESS);
2203
2204    // insert documents first time
2205    for (i=0;i<n;i++){
2206        sprintf(keybuf, "key%d", i);
2207        sprintf(metabuf, "meta%d", i);
2208        sprintf(bodybuf, "body%d(first)", i);
2209        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2210            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2211        fdb_set(db, doc[i]);
2212    }
2213    // manually flush WAL & commit
2214    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2215
2216    // insert documents second time
2217    for (i=0;i<n;i++){
2218        sprintf(bodybuf, "body%d(second)", i);
2219        fdb_doc_update(&doc[i], NULL, 0, bodybuf, strlen(bodybuf));
2220        fdb_set(db, doc[i]);
2221    }
2222    // manually flush WAL & commit
2223    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2224
2225    // insert documents third time (only even number documents)
2226    for (i=0;i<n;i+=2){
2227        sprintf(bodybuf, "body%d(third)", i);
2228        fdb_doc_update(&doc[i], NULL, 0, bodybuf, strlen(bodybuf));
2229        fdb_set(db, doc[i]);
2230    }
2231    // commit without WAL flushing
2232    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2233
2234    // create an iterator over sequence number
2235    fdb_iterator_sequence_init(db, &iterator, 0, 220, FDB_ITR_NONE);
2236
2237    // repeat until fail
2238    count = 0;
2239    seqnum = 100;
2240    do {
2241        status = fdb_iterator_get(iterator, &rdoc);
2242        TEST_CHK(status == FDB_RESULT_SUCCESS);
2243        if (seqnum < 140) { // point where WAL range & trie range overlap ends!
2244            seqnum += 2; // WAL overlap with trie, get unique trie keys only
2245        } else { // beyond this even keys in trie are also in WAL but outside..
2246            seqnum ++; // the iteration range, so they can be sequentially got
2247        }
2248
2249        if (seqnum <= 200) { // uptil WAL, unique trie items are returned...
2250            i = seqnum - 101;
2251            sprintf(bodybuf, "body%d(second)", i);
2252        } else { // once seqnum enters WAL range only WAL elements are returned..
2253            i = ((seqnum - 101) % n) * 2;
2254            sprintf(bodybuf, "body%d(third)", i);
2255        }
2256        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2257        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2258        TEST_CMP(rdoc->body, bodybuf, rdoc->bodylen);
2259        TEST_CHK(rdoc->seqnum == seqnum);
2260
2261        count++;
2262        fdb_doc_free(rdoc);
2263        rdoc = NULL;
2264    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2265    TEST_CHK(count==n); // since 220 > n all keys should be iterated
2266    fdb_iterator_close(iterator);
2267
2268    // close db file
2269    fdb_kvs_close(db);
2270    fdb_close(dbfile);
2271    // free all documents
2272    for (i=0;i<n;++i){
2273        fdb_doc_free(doc[i]);
2274    }
2275    // free all resources
2276    fdb_shutdown();
2277
2278    memleak_end();
2279
2280    TEST_RESULT("sequence iterator duplicate test");
2281}
2282
2283// MB-16406
2284void sequence_iterator_range_test()
2285{
2286    TEST_INIT();
2287
2288    memleak_start();
2289
2290    int i, r;
2291    int n = 10;
2292    int count;
2293    fdb_file_handle *dbfile;
2294    fdb_kvs_handle *db;
2295    fdb_doc **doc = alca(fdb_doc*, n);
2296    fdb_doc *rdoc = NULL;
2297    fdb_status status;
2298    fdb_iterator *iterator;
2299    fdb_seqnum_t seqnum;
2300
2301    char keybuf[256], metabuf[256], bodybuf[256];
2302
2303    // remove previous iterator_test files
2304    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
2305    (void)r;
2306
2307    fdb_config fconfig = fdb_get_default_config();
2308    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2309    fconfig.buffercache_size = 0;
2310    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
2311
2312    // open db
2313    fdb_open(&dbfile, "./iterator_test1", &fconfig);
2314    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2315    status = fdb_set_log_callback(db, logCallbackFunc,
2316                                  (void *) "sequence_iterator_test");
2317    TEST_CHK(status == FDB_RESULT_SUCCESS);
2318
2319    // insert docs
2320    for (i=0;i<n;i++){
2321        sprintf(keybuf, "key%d", i);
2322        sprintf(metabuf, "meta%d", i);
2323        sprintf(bodybuf, "body%d", i);
2324        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2325                                (void*)metabuf, strlen(metabuf),
2326                                (void*)bodybuf, strlen(bodybuf));
2327        fdb_set(db, doc[i]);
2328    }
2329    // manually flush WAL & commit
2330    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2331
2332    // create a full range seq iterator
2333    status = fdb_iterator_sequence_init(db, &iterator, 0, 0, FDB_ITR_NONE);
2334    TEST_CHK(status == FDB_RESULT_SUCCESS);
2335
2336    // repeat until fail
2337    count = 0;
2338    do {
2339        status = fdb_iterator_get(iterator, &rdoc);
2340        TEST_CHK(status == FDB_RESULT_SUCCESS);
2341        i = count;
2342        seqnum = i+1;
2343
2344        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2345        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2346        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2347        TEST_CHK(rdoc->seqnum == seqnum);
2348
2349        count++;
2350        fdb_doc_free(rdoc);
2351        rdoc = NULL;
2352    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2353    TEST_CHK(count == n);
2354    fdb_iterator_close(iterator);
2355
2356    // create a partial range seq iterator with the given max seq number
2357    status = fdb_iterator_sequence_init(db, &iterator, 0, n/2, FDB_ITR_NONE);
2358    TEST_CHK(status == FDB_RESULT_SUCCESS);
2359
2360    // repeat until fail
2361    count = 0;
2362    do {
2363        status = fdb_iterator_get(iterator, &rdoc);
2364        TEST_CHK(status == FDB_RESULT_SUCCESS);
2365        i = count;
2366        seqnum = i+1;
2367
2368        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2369        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2370        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2371        TEST_CHK(rdoc->seqnum == seqnum);
2372
2373        count++;
2374        fdb_doc_free(rdoc);
2375        rdoc = NULL;
2376    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2377    TEST_CHK(count == n/2);
2378    fdb_iterator_close(iterator);
2379
2380    // create a partial range seq iterator with the given min seq number
2381    status = fdb_iterator_sequence_init(db, &iterator, (n/2)+1, 0, FDB_ITR_NONE);
2382    TEST_CHK(status == FDB_RESULT_SUCCESS);
2383
2384    // repeat until fail
2385    count = 0;
2386    do {
2387        status = fdb_iterator_get(iterator, &rdoc);
2388        TEST_CHK(status == FDB_RESULT_SUCCESS);
2389        i = count + n/2;
2390        seqnum = i+1;
2391
2392        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2393        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2394        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2395        TEST_CHK(rdoc->seqnum == seqnum);
2396
2397        count++;
2398        fdb_doc_free(rdoc);
2399        rdoc = NULL;
2400    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2401    TEST_CHK(count == n/2);
2402    fdb_iterator_close(iterator);
2403
2404    // close db file
2405    fdb_kvs_close(db);
2406    fdb_close(dbfile);
2407    // free all documents
2408    for (i=0;i<n;++i){
2409        fdb_doc_free(doc[i]);
2410    }
2411    // free all resources
2412    fdb_shutdown();
2413
2414    memleak_end();
2415
2416    TEST_RESULT("sequence iterator range test");
2417}
2418
2419void reverse_sequence_iterator_test()
2420{
2421    TEST_INIT();
2422
2423    memleak_start();
2424
2425    int i, r, count;
2426    int n = 10;
2427    fdb_file_handle *dbfile;
2428    fdb_kvs_handle *db;
2429    fdb_doc **doc = alca(fdb_doc*, n);
2430    fdb_doc *rdoc = NULL;
2431    fdb_status status;
2432    fdb_iterator *iterator;
2433
2434    char keybuf[256], metabuf[256], bodybuf[256];
2435
2436    // remove previous iterator_test files
2437    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
2438    (void)r;
2439
2440    fdb_config fconfig = fdb_get_default_config();
2441    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2442    fconfig.buffercache_size = 0;
2443    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
2444    fconfig.wal_threshold = 1024;
2445    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2446    fconfig.compaction_threshold = 0;
2447
2448    // open db
2449    fdb_open(&dbfile, "./iterator_test1", &fconfig);
2450    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2451    status = fdb_set_log_callback(db, logCallbackFunc,
2452                                  (void *) "reverse_sequence_iterator_test");
2453    TEST_CHK(status == FDB_RESULT_SUCCESS);
2454
2455    // insert documents of even number
2456    for (i=0;i<n;i+=2){
2457        sprintf(keybuf, "key%d", i);
2458        sprintf(metabuf, "meta%d", i);
2459        sprintf(bodybuf, "body%d", i);
2460        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2461            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2462        fdb_set(db, doc[i]);
2463    }
2464    // manually flush WAL & commit
2465    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2466
2467    // insert documents of odd number
2468    for (i=1;i<n;i+=2){
2469        sprintf(keybuf, "key%d", i);
2470        sprintf(metabuf, "meta%d", i);
2471        sprintf(bodybuf, "body%d", i);
2472        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2473            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2474        fdb_set(db, doc[i]);
2475    }
2476    // commit without WAL flush
2477    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2478
2479    // now even number docs are in hb-trie & odd number docs are in WAL
2480
2481    // First test reverse sequence iteration as it only involves btrees
2482    // create an iterator over sequence number range over FULL RANGE
2483    fdb_iterator_sequence_init(db, &iterator, 0, 0, FDB_ITR_NONE);
2484
2485    // move iterator forward up till middle...
2486    i=0;
2487    count = 0;
2488    do {
2489        status = fdb_iterator_get(iterator, &rdoc);
2490        TEST_CHK(status == FDB_RESULT_SUCCESS);
2491
2492        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2493        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2494        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2495
2496        fdb_doc_free(rdoc);
2497        rdoc = NULL;
2498        count++;
2499        if (i + 2 >= n) break;
2500        i = i + 2; // by-seq, first come even docs, then odd
2501    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2502    TEST_CHK(count==n/2);
2503
2504    // Now test reverse sequence iterator from mid-way..
2505
2506    i = i - 2;
2507    status = fdb_iterator_prev(iterator);
2508    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
2509    status = fdb_iterator_get(iterator, &rdoc);
2510    TEST_CHK(status == FDB_RESULT_SUCCESS);
2511
2512    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2513    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2514    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2515
2516    fdb_doc_free(rdoc);
2517    rdoc = NULL;
2518    count++;
2519
2520    // change direction to forward again...
2521    TEST_CHK(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2522    i = i + 2;
2523    do {
2524        status = fdb_iterator_get(iterator, &rdoc);
2525        TEST_CHK(status == FDB_RESULT_SUCCESS);
2526
2527        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2528        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2529        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2530
2531        fdb_doc_free(rdoc);
2532        rdoc = NULL;
2533        i = (i + 2 >= n) ? 1 : i + 2;// by-seq, first come even docs, then odd
2534        count++;
2535    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2536
2537    TEST_CHK(count==n+2); // two items were double counted due to reverse
2538
2539    // Reached End, now reverse iterate till start
2540    i = n - 1;
2541    count = n;
2542    status = fdb_iterator_prev(iterator);
2543    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
2544    do {
2545        status = fdb_iterator_get(iterator, &rdoc);
2546        TEST_CHK(status == FDB_RESULT_SUCCESS);
2547
2548        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2549        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2550        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2551
2552        fdb_doc_free(rdoc);
2553        rdoc = NULL;
2554        i = (i - 2 < 0) ? n - 2 : i - 2;
2555        if (count) count--;
2556    } while (fdb_iterator_prev(iterator) != FDB_RESULT_ITERATOR_FAIL);
2557    TEST_CHK(count == 0);
2558    fdb_iterator_close(iterator);
2559
2560    // close db file
2561    fdb_kvs_close(db);
2562    fdb_close(dbfile);
2563
2564    // free all documents
2565    for (i=0;i<n;++i){
2566        fdb_doc_free(doc[i]);
2567    }
2568
2569    // free all resources
2570    fdb_shutdown();
2571
2572    memleak_end();
2573
2574    TEST_RESULT("reverse sequence iterator test");
2575}
2576
2577
2578void reverse_sequence_iterator_kvs_test()
2579{
2580    TEST_INIT();
2581    memleak_start();
2582
2583    int i, r, count;
2584    int n = 10;
2585    fdb_file_handle *dbfile;
2586    fdb_kvs_handle *kv1, *kv2;
2587    fdb_doc **doc = alca(fdb_doc*, n);
2588    fdb_doc **doc2 = alca(fdb_doc*, n);
2589    fdb_doc *rdoc = NULL;
2590    fdb_status status;
2591    fdb_iterator *iterator;
2592    fdb_iterator *iterator2;
2593
2594    char keybuf[256], metabuf[256], bodybuf[256];
2595
2596    // remove previous iterator_test files
2597    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
2598    (void)r;
2599
2600    fdb_config fconfig = fdb_get_default_config();
2601    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2602    fconfig.buffercache_size = 0;
2603    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
2604    fconfig.wal_threshold = 1024;
2605    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2606    fconfig.compaction_threshold = 0;
2607
2608    // open db
2609    fdb_open(&dbfile, "./iterator_test1", &fconfig);
2610    fdb_kvs_open_default(dbfile, &kv1, &kvs_config);
2611    status = fdb_set_log_callback(kv1, logCallbackFunc,
2612                                  (void *) "reverse_sequence_iterator_kvs_test");
2613    TEST_CHK(status == FDB_RESULT_SUCCESS);
2614
2615    // Create another KV store...
2616    status = fdb_kvs_open(dbfile, &kv2, "kv2", &kvs_config);
2617    TEST_CHK(status == FDB_RESULT_SUCCESS);
2618
2619    // to 'kv2' with entire range
2620    for (i=0;i<n;++i) {
2621        sprintf(keybuf, "kEy%d", i);
2622        sprintf(metabuf, "mEta%d", i);
2623        sprintf(bodybuf, "bOdy%d", i);
2624        fdb_doc_create(&doc2[i], (void*)keybuf, strlen(keybuf),
2625            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2626        status = fdb_set(kv2, doc2[i]);
2627        TEST_CHK(status == FDB_RESULT_SUCCESS);
2628    }
2629
2630    // insert kv1 documents of even number
2631    for (i=0;i<n;i+=2) {
2632        sprintf(keybuf, "key%d", i);
2633        sprintf(metabuf, "meta%d", i);
2634        sprintf(bodybuf, "body%d", i);
2635        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2636            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2637        fdb_set(kv1, doc[i]);
2638    }
2639
2640    // manually flush WAL & commit
2641    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2642
2643    // insert kv1 documents of odd number
2644    for (i=1;i<n;i+=2) {
2645        sprintf(keybuf, "key%d", i);
2646        sprintf(metabuf, "meta%d", i);
2647        sprintf(bodybuf, "body%d", i);
2648        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2649            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2650        fdb_set(kv1, doc[i]);
2651    }
2652    // commit without WAL flush
2653    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2654
2655    // iterate even docs on kv1
2656    status = fdb_iterator_sequence_init(kv1, &iterator, 0, 0, FDB_ITR_NONE);
2657    TEST_CHK(status == FDB_RESULT_SUCCESS);
2658
2659    i=0;
2660    count = 0;
2661    while (1) {
2662        fdb_iterator_get(iterator, &rdoc);
2663        TEST_CHK(!memcmp(rdoc->key, doc[i]->key, rdoc->keylen));
2664        TEST_CHK(!memcmp(rdoc->meta, doc[i]->meta, rdoc->metalen));
2665        TEST_CHK(!memcmp(rdoc->body, doc[i]->body, rdoc->bodylen));
2666        fdb_doc_free(rdoc);
2667        rdoc = NULL;
2668        count++;
2669        if (i + 2 >= n) {
2670            break;
2671        }
2672        i = i + 2; // by-seq, first come even docs, then odd
2673        status = fdb_iterator_next(iterator);
2674        if (status == FDB_RESULT_ITERATOR_FAIL) break;
2675    }
2676    TEST_CHK(count==n/2);
2677
2678    // iterate all docs over kv2
2679    status = fdb_iterator_sequence_init(kv2, &iterator2, 0, 0, FDB_ITR_NONE);
2680    TEST_CHK(status == FDB_RESULT_SUCCESS);
2681    while(1) {
2682        status = fdb_iterator_get(iterator2, &rdoc);
2683        TEST_CHK(status == FDB_RESULT_SUCCESS);
2684        fdb_doc_free(rdoc);
2685        rdoc = NULL;
2686        status = fdb_iterator_next(iterator2);
2687        if (status == FDB_RESULT_ITERATOR_FAIL) {
2688            break;
2689        }
2690    }
2691
2692    // manually flush WAL & commit
2693    // iterators should be unaffected
2694    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2695
2696    // reverse iterate even docs over kv1
2697     i = n - 4;
2698    count = 0;
2699    while (1) {
2700        status = fdb_iterator_prev(iterator);
2701        if (status == FDB_RESULT_ITERATOR_FAIL) {
2702            break;
2703        }
2704        status = fdb_iterator_get(iterator, &rdoc);
2705        TEST_CHK(status == FDB_RESULT_SUCCESS);
2706        TEST_CHK(!memcmp(rdoc->key, doc[i]->key, rdoc->keylen));
2707        TEST_CHK(!memcmp(rdoc->meta, doc[i]->meta, rdoc->metalen));
2708        TEST_CHK(!memcmp(rdoc->body, doc[i]->body, rdoc->bodylen));
2709        fdb_doc_free(rdoc);
2710        rdoc = NULL;
2711        i-=2;
2712        count++;
2713    }
2714
2715    TEST_CHK(count==4);
2716    fdb_iterator_close(iterator);
2717
2718    i = n-1;
2719    count = 0;
2720    // reverse iterate all docs over kv2
2721    while (1) {
2722        status = fdb_iterator_prev(iterator2);
2723        if (status == FDB_RESULT_ITERATOR_FAIL) {
2724            break;
2725        }
2726        status = fdb_iterator_get(iterator2, &rdoc);
2727        TEST_CHK(status == FDB_RESULT_SUCCESS);
2728        TEST_CHK(!memcmp(rdoc->key, doc2[i]->key, rdoc->keylen));
2729        TEST_CHK(!memcmp(rdoc->meta, doc2[i]->meta, rdoc->metalen));
2730        TEST_CHK(!memcmp(rdoc->body, doc2[i]->body, rdoc->bodylen));
2731        fdb_doc_free(rdoc);
2732        rdoc = NULL;
2733        i--;
2734        count++;
2735    }
2736    TEST_CHK(count==n);
2737    fdb_iterator_close(iterator2);
2738
2739    // re-open iterator after commit should return all docs for kv1
2740    i = 0;
2741    count = 0;
2742    status = fdb_iterator_sequence_init(kv1, &iterator, 0, 0, FDB_ITR_NONE);
2743    TEST_CHK(status == FDB_RESULT_SUCCESS);
2744    while (1) {
2745        status = fdb_iterator_get(iterator, &rdoc);
2746        TEST_CHK(status == FDB_RESULT_SUCCESS);
2747        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2748        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2749        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2750        fdb_doc_free(rdoc);
2751        rdoc = NULL;
2752        if (i == 8) {
2753            i=1; // switch to odds
2754        } else {
2755            i+=2;
2756        }
2757        count++;
2758        status = fdb_iterator_next(iterator);
2759        if (status == FDB_RESULT_ITERATOR_FAIL) {
2760            break;
2761        }
2762    }
2763    TEST_CHK(count==n);
2764    fdb_iterator_close(iterator);
2765
2766    // free all documents
2767    for (i=0;i<n;++i) {
2768        fdb_doc_free(doc[i]);
2769        fdb_doc_free(doc2[i]);
2770    }
2771
2772    fdb_kvs_close(kv1);
2773    fdb_kvs_close(kv2);
2774    fdb_close(dbfile);
2775
2776    fdb_shutdown();
2777    memleak_end();
2778    TEST_RESULT("reverse sequence iterator kvs test");
2779
2780}
2781
2782void reverse_iterator_test()
2783{
2784    TEST_INIT();
2785
2786    memleak_start();
2787
2788    int i, r;
2789    int n = 10;
2790    fdb_file_handle *dbfile;
2791    fdb_kvs_handle *db;
2792    fdb_doc **doc = alca(fdb_doc*, n);
2793    fdb_doc *rdoc = NULL;
2794    fdb_status status;
2795    fdb_iterator *iterator;
2796
2797    char keybuf[256], metabuf[256], bodybuf[256];
2798
2799    // remove previous iterator_test files
2800    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
2801    (void)r;
2802
2803    fdb_config fconfig = fdb_get_default_config();
2804    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2805    fconfig.buffercache_size = 0;
2806    fconfig.wal_threshold = 1024;
2807    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2808    fconfig.compaction_threshold = 0;
2809
2810    // open db
2811    fdb_open(&dbfile, "./iterator_test1", &fconfig);
2812    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2813    status = fdb_set_log_callback(db, logCallbackFunc,
2814                                  (void *) "reverse_iterator_test");
2815    TEST_CHK(status == FDB_RESULT_SUCCESS);
2816
2817    // insert documents of even number
2818    for (i=0;i<n;i+=2){
2819        sprintf(keybuf, "key%d", i);
2820        sprintf(metabuf, "meta%d", i);
2821        sprintf(bodybuf, "body%d", i);
2822        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2823            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2824        fdb_set(db, doc[i]);
2825    }
2826    // manually flush WAL & commit
2827    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2828
2829    // insert documents of odd number
2830    for (i=1;i<n;i+=2){
2831        sprintf(keybuf, "key%d", i);
2832        sprintf(metabuf, "meta%d", i);
2833        sprintf(bodybuf, "body%d", i);
2834        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2835            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2836        fdb_set(db, doc[i]);
2837    }
2838    // commit without WAL flush
2839    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2840
2841    // now even number docs are in hb-trie & odd number docs are in WAL
2842
2843    // Reverse iteration over key range which involves hb-tries
2844    // create an iterator for full range
2845    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, FDB_ITR_NONE);
2846
2847    // first test forward iterator - repeat until fail
2848    i=0;
2849    do {
2850        status = fdb_iterator_get(iterator, &rdoc);
2851        TEST_CHK(status == FDB_RESULT_SUCCESS);
2852
2853        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2854        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2855        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2856
2857        fdb_doc_free(rdoc);
2858        rdoc = NULL;
2859        i++;
2860    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2861    TEST_CHK(i==10);
2862
2863    // Now test reverse iterator..
2864    for (--i; fdb_iterator_prev(iterator) != FDB_RESULT_ITERATOR_FAIL; --i) {
2865        status = fdb_iterator_get(iterator, &rdoc);
2866        TEST_CHK(status == FDB_RESULT_SUCCESS);
2867
2868        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2869        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2870        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2871
2872        fdb_doc_free(rdoc);
2873        rdoc = NULL;
2874        if (i == 5) break; // Change direction at half point
2875    }
2876    TEST_CHK(i == 5);
2877
2878    // Mid-way reverse direction, again test forward iterator...
2879    i++;
2880    status = fdb_iterator_next(iterator);
2881    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
2882    status = fdb_iterator_get(iterator, &rdoc);
2883    TEST_CHK(status == FDB_RESULT_SUCCESS);
2884
2885    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2886    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2887    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2888
2889    fdb_doc_free(rdoc);
2890    rdoc = NULL;
2891
2892    // Mid-way reverse direction, again test forward iterator...
2893    i++;
2894    status = fdb_iterator_next(iterator);
2895    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
2896    status = fdb_iterator_get(iterator, &rdoc);
2897    TEST_CHK(status == FDB_RESULT_SUCCESS);
2898
2899    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2900    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2901    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2902
2903    fdb_doc_free(rdoc);
2904    rdoc = NULL;
2905
2906    // Again change direction and test reverse iterator..
2907    for (--i; fdb_iterator_prev(iterator) != FDB_RESULT_ITERATOR_FAIL; --i) {
2908        status = fdb_iterator_get(iterator, &rdoc);
2909        TEST_CHK(status == FDB_RESULT_SUCCESS);
2910
2911        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2912        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2913        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2914
2915        fdb_doc_free(rdoc);
2916        rdoc = NULL;
2917    }
2918    TEST_CHK(i == -1);
2919
2920    // Reached end - now test forward iterator...
2921    TEST_CHK(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2922    i++;
2923    do {
2924        status = fdb_iterator_get(iterator, &rdoc);
2925        TEST_CHK(status == FDB_RESULT_SUCCESS);
2926
2927        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2928        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2929        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2930
2931        fdb_doc_free(rdoc);
2932        rdoc = NULL;
2933        i++;
2934    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2935    TEST_CHK(i==10);
2936
2937    fdb_iterator_close(iterator);
2938
2939    // close db file
2940    fdb_kvs_close(db);
2941    fdb_close(dbfile);
2942
2943    // free all documents
2944    for (i=0;i<n;++i){
2945        fdb_doc_free(doc[i]);
2946    }
2947
2948    // free all resources
2949    fdb_shutdown();
2950
2951    memleak_end();
2952
2953    TEST_RESULT("reverse iterator test");
2954}
2955
2956void reverse_seek_to_max_nokey(void)
2957{
2958    TEST_INIT();
2959
2960    memleak_start();
2961
2962    int i, r;
2963    int n = 100;
2964    fdb_file_handle *dbfile;
2965    fdb_kvs_handle *db;
2966    fdb_doc **doc = alca(fdb_doc*, n);
2967    fdb_doc *rdoc;
2968    fdb_status status;
2969    fdb_iterator *iterator;
2970
2971    char keybuf[16];
2972
2973    // remove previous mvcc_test files
2974    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
2975    (void)r;
2976
2977    fdb_config fconfig = fdb_get_default_config();
2978    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2979    fconfig.buffercache_size = 0;
2980
2981    // open db
2982    status = fdb_open(&dbfile, "./iterator_test1", &fconfig);
2983    TEST_CHK(status == FDB_RESULT_SUCCESS);
2984    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
2985    TEST_CHK(status == FDB_RESULT_SUCCESS);
2986
2987    // ------- Setup test ----------------------------------
2988    for (i=0; i<n; i++){
2989        sprintf(keybuf, "doc-%03d", i);
2990        keybuf[7] = '\0';
2991        keybuf[8] = '\0';
2992        fdb_doc_create(&doc[i], (void*)keybuf, 10,
2993            NULL, 0, (void*)keybuf, 10);
2994        fdb_set(db, doc[i]);
2995    }
2996
2997    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2998
2999    status = fdb_set_log_callback(db, logCallbackFunc,
3000                                  (void *) "reverse_seek_to_max_nokey");
3001    TEST_CHK(status == FDB_RESULT_SUCCESS);
3002
3003    // set range to have end key that does not exist
3004    status = fdb_iterator_init(db, &iterator, doc[24]->key, 10,
3005                       (void*)"doc-029b", 10,
3006                      FDB_ITR_NO_DELETES);
3007    TEST_CHK(status == FDB_RESULT_SUCCESS);
3008
3009    status = fdb_iterator_seek_to_max(iterator);
3010    TEST_CHK(status == FDB_RESULT_SUCCESS);
3011
3012    fdb_doc_create(&rdoc, NULL, 0, NULL, 0, NULL, 0);
3013    status = fdb_iterator_get(iterator, &rdoc);
3014    TEST_CHK(status == FDB_RESULT_SUCCESS);
3015    TEST_CMP(rdoc->key, "doc-029", 8);
3016    fdb_doc_free(rdoc);
3017
3018    fdb_iterator_close(iterator);
3019
3020    // set range to have start key that does not exist
3021    status = fdb_iterator_init(db, &iterator, (void*)"doc-024b", 10,
3022                               doc[30]->key, 10,
3023                      FDB_ITR_NO_DELETES);
3024    TEST_CHK(status == FDB_RESULT_SUCCESS);
3025
3026    status = fdb_iterator_seek_to_min(iterator);
3027    TEST_CHK(status == FDB_RESULT_SUCCESS);
3028
3029    fdb_doc_create(&rdoc, NULL, 0, NULL, 0, NULL, 0);
3030    status = fdb_iterator_get(iterator, &rdoc);
3031    TEST_CHK(status == FDB_RESULT_SUCCESS);
3032    TEST_CMP(rdoc->key, "doc-025", 8);
3033    fdb_doc_free(rdoc);
3034
3035    fdb_iterator_close(iterator);
3036
3037    // close db handle
3038    fdb_kvs_close(db);
3039
3040    // close db file
3041    fdb_close(dbfile);
3042
3043    // free all documents
3044    for (i=0;i<n;++i){
3045        fdb_doc_free(doc[i]);
3046    }
3047
3048    // free all resources
3049    fdb_shutdown();
3050
3051    memleak_end();
3052
3053    TEST_RESULT("reverse seek to max non-existent key test");
3054}
3055
3056void iterator_seek_wal_only_test()
3057{
3058    TEST_INIT();
3059
3060    memleak_start();
3061
3062    int i, r;
3063    int n = 10;
3064    fdb_file_handle *dbfile;
3065    fdb_kvs_handle *db, *kv1;
3066    fdb_doc **doc = alca(fdb_doc*, n);
3067    fdb_doc *rdoc = NULL;
3068    fdb_status status;
3069    fdb_iterator *iterator;
3070
3071    char keybuf[256], metabuf[256], bodybuf[256];
3072
3073    // remove previous iterator_test files
3074    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
3075    (void)r;
3076
3077    fdb_config fconfig = fdb_get_default_config();
3078    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
3079    fconfig.buffercache_size = 0;
3080    fconfig.wal_threshold = 1024;
3081    fconfig.flags = FDB_OPEN_FLAG_CREATE;
3082    fconfig.compaction_threshold = 0;
3083
3084    // open db
3085    fdb_open(&dbfile, "./iterator_test1", &fconfig);
3086    fdb_kvs_open_default(dbfile, &db, &kvs_config);
3087    status = fdb_set_log_callback(db, logCallbackFunc,
3088                                  (void *) "iterator_seek_wal_only_test");
3089    TEST_CHK(status == FDB_RESULT_SUCCESS);
3090
3091    // Create another KV store..
3092    status = fdb_kvs_open(dbfile, &kv1, "kv1", &kvs_config);
3093    TEST_CHK(status == FDB_RESULT_SUCCESS);
3094
3095    // insert using 'kv1' instance to ensure it does not interfere
3096    for (i=0;i<n;++i){
3097        sprintf(keybuf, "kEy%d", i);
3098        sprintf(metabuf, "mEta%d", i);
3099        sprintf(bodybuf, "bOdy%d", i);
3100        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
3101            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
3102        status = fdb_set(kv1, doc[i]);
3103        TEST_CHK(status == FDB_RESULT_SUCCESS);
3104        fdb_doc_free(doc[i]);
3105    }
3106
3107    // insert all documents into WAL only
3108    for (i=0;i<n;++i){
3109        sprintf(keybuf, "key%d", i);
3110        sprintf(metabuf, "meta%d", i);
3111        sprintf(bodybuf, "body%d", i);
3112        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
3113            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
3114        fdb_set(db, doc[i]);
3115    }
3116    // commit without WAL flush
3117    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
3118
3119    // create an iterator for full range
3120    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, FDB_ITR_NONE);
3121
3122    // seek current iterator to inside the WAL's avl tree..
3123    status = fdb_iterator_get(iterator, &rdoc);
3124    TEST_CHK(status == FDB_RESULT_SUCCESS);
3125
3126    TEST_CMP(rdoc->key, doc[0]->key, rdoc->keylen);
3127    TEST_CMP(rdoc->meta, doc[0]->meta, rdoc->metalen);
3128    TEST_CMP(rdoc->body, doc[0]->body, rdoc->bodylen);
3129    fdb_doc_free(rdoc);
3130    rdoc = NULL;
3131
3132    // seek forward to 2nd key ..
3133    i=2;
3134    status = fdb_iterator_seek(iterator, doc[i]->key, strlen(keybuf), 0);
3135    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
3136    status = fdb_iterator_get(iterator, &rdoc);
3137    TEST_CHK(status == FDB_RESULT_SUCCESS);
3138
3139    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
3140    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
3141    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
3142    fdb_doc_free(rdoc);
3143    rdoc = NULL;
3144
3145    // iterator should be able to proceed forward
3146    status = fdb_iterator_next(iterator);
3147    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
3148    status = fdb_iterator_get(iterator, &rdoc);
3149    TEST_CHK(status == FDB_RESULT_SUCCESS);
3150
3151    i++;
3152    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
3153    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
3154    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
3155    fdb_doc_free(rdoc);
3156    rdoc = NULL;
3157
3158    // seek forward to the last key.
3159    status = fdb_iterator_seek(iterator, doc[n-1]->key, strlen(keybuf), 0);
3160    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
3161    status = fdb_iterator_get(iterator, &rdoc);
3162    TEST_CHK(status == FDB_RESULT_SUCCESS);
3163
3164    TEST_CMP(rdoc->key, doc[n-1]->key, rdoc->keylen);
3165    TEST_CMP(rdoc->meta, doc[n-1]->meta, rdoc->metalen);
3166    TEST_CMP(rdoc->body, doc[n-1]->body, rdoc->bodylen);
3167    fdb_doc_free(rdoc);
3168    rdoc = NULL;
3169
3170    // seek backward to start key ..
3171    i = 0;
3172    status = fdb_iterator_seek(iterator, doc[i]->key, strlen(keybuf), 0);
3173    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
3174    status = fdb_iterator_get(iterator, &rdoc);
3175    TEST_CHK(status == FDB_RESULT_SUCCESS);
3176
3177    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
3178    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
3179    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
3180    fdb_doc_free(rdoc);
3181    rdoc = NULL;
3182
3183    // seek forward to key2 ..
3184    status = fdb_iterator_seek(iterator, doc[2]->key, strlen(keybuf), 0);
3185    TEST_CHK(status == FDB_RESULT_SUCCESS);
3186
3187    i=2;
3188    do {
3189        status = fdb_iterator_get(iterator, &rdoc);
3190        TEST_CHK(status == FDB_RESULT_SUCCESS);
3191
3192        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
3193        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
3194        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
3195
3196        fdb_doc_free(rdoc);
3197        rdoc = NULL;
3198        i++;
3199    } while(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
3200    TEST_CHK(i==10);
3201
3202    // Seek backward again to a key...
3203    status = fdb_iterator_seek(iterator, doc[3]->key, strlen(keybuf), 0);
3204    TEST_CHK(status == FDB_RESULT_SUCCESS);
3205
3206    i=3;
3207    do {
3208        status = fdb_iterator_get(iterator, &rdoc);
3209        TEST_CHK(status == FDB_RESULT_SUCCESS);
3210
3211        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
3212        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
3213        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
3214
3215        fdb_doc_free(rdoc);
3216        rdoc = NULL;
3217        i++;
3218    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
3219    TEST_CHK(i==10);
3220
3221    fdb_iterator_close(iterator);
3222
3223    // Test fdb_iterator_seek_to_max with key range
3224    // create an iterator for range of doc[4] ~ doc[8]
3225    sprintf(keybuf, "key%d", 4); // reuse buffer for start key
3226    sprintf(metabuf, "key%d", 8); // reuse buffer for end_key
3227    status = fdb_iterator_init(db, &iterator, keybuf, strlen(keybuf),
3228                      metabuf, strlen(metabuf),
3229                      FDB_ITR_NO_DELETES);
3230    TEST_CHK(status == FDB_RESULT_SUCCESS);
3231    i = 8;
3232    status = fdb_iterator_seek_to_max(iterator);
3233    TEST_CHK(status == FDB_RESULT_SUCCESS);
3234    status = fdb_iterator_get(iterator, &rdoc);
3235    TEST_CHK(status == FDB_RESULT_SUCCESS);
3236
3237    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
3238    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
3239    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
3240    fdb_doc_free(rdoc);
3241    rdoc = NULL;
3242
3243    // test fdb_iterator_seek_to_min
3244    i = 4;
3245    status = fdb_iterator_seek_to_min(