1/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/*
3 *     Copyright 2010 Couchbase, Inc
4 *
5 *   Licensed under the Apache License, Version 2.0 (the "License");
6 *   you may not use this file except in compliance with the License.
7 *   You may obtain a copy of the License at
8 *
9 *       http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *   Unless required by applicable law or agreed to in writing, software
12 *   distributed under the License is distributed on an "AS IS" BASIS,
13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *   See the License for the specific language governing permissions and
15 *   limitations under the License.
16 */
17
18#include <stdio.h>
19#include <stdlib.h>
20#include <string.h>
21#include <stdint.h>
22#include <time.h>
23#if !defined(WIN32) && !defined(_WIN32)
24#include <unistd.h>
25#endif
26
27#include "libforestdb/forestdb.h"
28#include "test.h"
29
30#include "internal_types.h"
31#include "functional_util.h"
32
33
34void iterator_test()
35{
36    TEST_INIT();
37
38    memleak_start();
39
40    int i, r;
41    int n = 10;
42    fdb_file_handle *dbfile;
43    fdb_kvs_handle *db, *kv1;;
44    fdb_doc **doc = alca(fdb_doc*, n);
45    fdb_doc *rdoc = NULL;
46    fdb_status status;
47    fdb_iterator *iterator;
48
49    char keybuf[256], metabuf[256], bodybuf[256], temp[256];
50
51    // remove  all previous iterator_test files
52    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
53    (void)r;
54
55    fdb_config fconfig = fdb_get_default_config();
56    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
57    fconfig.buffercache_size = 0;
58    fconfig.wal_threshold = 1024;
59    fconfig.flags = FDB_OPEN_FLAG_CREATE;
60    fconfig.compaction_threshold = 0;
61    fconfig.purging_interval = 1; // retain deletes before compaction
62
63    // open db
64    fdb_open(&dbfile, "./iterator_test1", &fconfig);
65    fdb_kvs_open_default(dbfile, &db, &kvs_config);
66    status = fdb_set_log_callback(db, logCallbackFunc,
67                                  (void *) "iterator_test");
68    TEST_CHK(status == FDB_RESULT_SUCCESS);
69
70    // Create another KV store..
71    status = fdb_kvs_open(dbfile, &kv1, "kv1", &kvs_config);
72    TEST_CHK(status == FDB_RESULT_SUCCESS);
73
74    // insert using 'kv1' instance to ensure it does not interfere
75    for (i=0;i<n;++i){
76        sprintf(keybuf, "kEy%d", i);
77        sprintf(metabuf, "mEta%d", i);
78        sprintf(bodybuf, "bOdy%d", i);
79        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
80            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
81        status = fdb_set(kv1, doc[i]);
82        TEST_CHK(status == FDB_RESULT_SUCCESS);
83        fdb_doc_free(doc[i]);
84    }
85
86    // insert documents of even number
87    for (i=0;i<n;i+=2){
88        sprintf(keybuf, "key%d", i);
89        sprintf(metabuf, "meta%d", i);
90        sprintf(bodybuf, "body%d", i);
91        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
92            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
93        fdb_set(db, doc[i]);
94    }
95    // manually flush WAL & commit
96    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
97
98    // insert documents of odd number
99    for (i=1;i<n;i+=2){
100        sprintf(keybuf, "key%d", i);
101        sprintf(metabuf, "meta%d", i);
102        sprintf(bodybuf, "body%d", i);
103        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
104            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
105        fdb_set(db, doc[i]);
106    }
107    // commit without WAL flush
108    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
109
110    // now even number docs are in hb-trie & odd number docs are in WAL
111
112    // create an iterator for full range
113    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, FDB_ITR_NONE);
114
115    // repeat until fail
116    i=0;
117    do {
118        status = fdb_iterator_get(iterator, &rdoc);
119        TEST_CHK(status == FDB_RESULT_SUCCESS);
120
121        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
122        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
123        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
124
125        fdb_doc_free(rdoc);
126        rdoc = NULL;
127        i++;
128    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
129    TEST_CHK(i==10);
130    fdb_iterator_close(iterator);
131
132    // create an iterator.
133    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, FDB_ITR_NONE);
134
135    // repeat until fail
136    i=0;
137    do {
138        // retrieve the next doc and get the byte offset of the returned doc
139        status = fdb_iterator_get_metaonly(iterator, &rdoc);
140        TEST_CHK(status == FDB_RESULT_SUCCESS);
141
142        TEST_CHK(rdoc->offset != BLK_NOT_FOUND);
143        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
144        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
145        TEST_CHK(rdoc->body == NULL);
146
147        fdb_doc_free(rdoc);
148        rdoc = NULL;
149        i++;
150    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
151    TEST_CHK(i==10);
152    fdb_iterator_close(iterator);
153
154    // create another iterator starts from doc[3]
155    sprintf(keybuf, "key%d", 3);
156    fdb_iterator_init(db, &iterator, (void*)keybuf, strlen(keybuf), NULL, 0,
157                      FDB_ITR_NONE);
158
159    // repeat until fail
160    i=3;
161    do {
162        status = fdb_iterator_get(iterator, &rdoc);
163        TEST_CHK(status == FDB_RESULT_SUCCESS);
164
165        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
166        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
167        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
168
169        fdb_doc_free(rdoc);
170        rdoc = NULL;
171        i++;
172    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
173    TEST_CHK(i==10);
174    fdb_iterator_close(iterator);
175
176    // create another iterator for the range of doc[4] ~ doc[8]
177    sprintf(keybuf, "key%d", 4);
178    sprintf(temp, "key%d", 8);
179    fdb_iterator_init(db, &iterator, (void*)keybuf, strlen(keybuf),
180        (void*)temp, strlen(temp), FDB_ITR_NONE);
181
182    // repeat until fail
183    i=4;
184    do {
185        status = fdb_iterator_get(iterator, &rdoc);
186        TEST_CHK(status == FDB_RESULT_SUCCESS);
187
188        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
189        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
190        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
191
192        fdb_doc_free(rdoc);
193        rdoc = NULL;
194        i++;
195    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
196    TEST_CHK(i==9);
197    fdb_iterator_close(iterator);
198
199    // create another iterator for the range of doc[5] ~ doc[7]
200    fdb_iterator_init(db, &iterator, (void*)keybuf, strlen(keybuf),
201        (void*)temp, strlen(temp), FDB_ITR_SKIP_MIN_KEY | FDB_ITR_SKIP_MAX_KEY);
202
203    // repeat until fail
204    i=5;
205    do {
206        status = fdb_iterator_get(iterator, &rdoc);
207        TEST_CHK(status == FDB_RESULT_SUCCESS);
208
209        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
210        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
211        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
212
213        fdb_doc_free(rdoc);
214        rdoc = NULL;
215        i++;
216    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
217    TEST_CHK(i==8);
218    fdb_iterator_close(iterator);
219
220    // remove document #8 and #9
221    fdb_doc_create(&rdoc, doc[8]->key, doc[8]->keylen, doc[8]->meta,
222                   doc[8]->metalen, NULL, 0);
223    status = fdb_del(db, rdoc);
224    TEST_CHK(status == FDB_RESULT_SUCCESS);
225    fdb_doc_free(rdoc);
226    rdoc = NULL;
227    fdb_doc_create(&rdoc, doc[9]->key, doc[9]->keylen, doc[9]->meta,
228                   doc[9]->metalen, NULL, 0);
229    status = fdb_del(db, rdoc);
230    TEST_CHK(status == FDB_RESULT_SUCCESS);
231    fdb_doc_free(rdoc);
232    rdoc = NULL;
233    // commit
234    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
235
236    // create an iterator for full range
237    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, FDB_ITR_NONE);
238    // repeat until fail
239    i=0;
240    do {
241        status = fdb_iterator_get(iterator, &rdoc);
242        TEST_CHK(status == FDB_RESULT_SUCCESS);
243
244        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
245        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
246        if (i < 8) {
247            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
248        } else {
249            TEST_CHK(rdoc->deleted == true);
250        }
251
252        fdb_doc_free(rdoc);
253        rdoc = NULL;
254        i++;
255    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
256    TEST_CHK(i==10);
257    fdb_iterator_close(iterator);
258
259    // create an iterator for full range, but no deletes.
260    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, FDB_ITR_NO_DELETES);
261    // repeat until fail
262    i=0;
263    do {
264        status = fdb_iterator_get(iterator, &rdoc);
265        TEST_CHK(status == FDB_RESULT_SUCCESS);
266
267        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
268        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
269        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
270
271        fdb_doc_free(rdoc);
272        rdoc = NULL;
273        i++;
274    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
275    TEST_CHK(i==8);
276    fdb_iterator_close(iterator);
277
278    // create an iterator for range of doc[4] ~ doc[8], but no deletes.
279    sprintf(keybuf, "key%d", 4);
280    sprintf(temp, "key%d", 8);
281    fdb_iterator_init(db, &iterator, keybuf, strlen(keybuf), temp, strlen(temp),
282                      FDB_ITR_NO_DELETES);
283    // repeat until fail
284    i=4;
285    do {
286        status = fdb_iterator_get(iterator, &rdoc);
287        TEST_CHK(status == FDB_RESULT_SUCCESS);
288
289        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
290        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
291        TEST_CHK(rdoc->deleted == false);
292
293        fdb_doc_free(rdoc);
294        rdoc = NULL;
295        i++;
296    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
297    TEST_CHK(i==8);
298    fdb_iterator_close(iterator);
299
300    // close kvs1 instance
301    fdb_kvs_close(kv1);
302    // close db file
303    fdb_kvs_close(db);
304    fdb_close(dbfile);
305
306    // free all documents
307    for (i=0;i<n;++i){
308        fdb_doc_free(doc[i]);
309    }
310
311    // free all resources
312    fdb_shutdown();
313
314    memleak_end();
315
316    TEST_RESULT("iterator test");
317}
318
319void iterator_with_concurrent_updates_test()
320{
321    // unit test for MB-12287
322    TEST_INIT();
323    memleak_start();
324
325    int i, n=10;
326    int r;
327    fdb_file_handle *dbfile;
328    fdb_kvs_handle *db1, *db2, *db3;
329    fdb_iterator *itr;
330    fdb_config fconfig;
331    fdb_kvs_config kvs_config;
332    fdb_doc **doc = alca(fdb_doc*, n);
333    fdb_doc *rdoc = NULL;
334    fdb_status status;
335    char keybuf[256], bodybuf[256];
336
337    // remove previous iterator_test files
338    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
339    (void)r;
340
341    // open db1, db2, db3 on the same file
342    fconfig = fdb_get_default_config();
343    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
344    kvs_config = fdb_get_default_kvs_config();
345    fdb_open(&dbfile, "./iterator_test1", &fconfig);
346
347    fdb_kvs_open_default(dbfile, &db1, &kvs_config);
348    status = fdb_set_log_callback(db1, logCallbackFunc,
349                                  (void *) "iterator_concurrent_update_test");
350    TEST_CHK(status == FDB_RESULT_SUCCESS);
351
352    fdb_kvs_open_default(dbfile, &db2, &kvs_config);
353    status = fdb_set_log_callback(db2, logCallbackFunc,
354                                  (void *) "iterator_concurrent_update_test");
355    TEST_CHK(status == FDB_RESULT_SUCCESS);
356
357    fdb_kvs_open_default(dbfile, &db3, &kvs_config);
358    status = fdb_set_log_callback(db3, logCallbackFunc,
359                                  (void *) "iterator_concurrent_update_test");
360    TEST_CHK(status == FDB_RESULT_SUCCESS);
361
362    // insert docs using db1
363    for (i=0;i<10;++i){
364        sprintf(keybuf, "key%d", i);
365        sprintf(bodybuf, "body%d", i);
366        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf), NULL, 0,
367                       (void*)bodybuf, strlen(bodybuf));
368        fdb_set(db1, doc[i]);
369    }
370    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
371
372    // create an iterator using db2
373    fdb_iterator_init(db2, &itr, NULL, 0, NULL, 0, FDB_ITR_NONE);
374    r = 0;
375    do {
376        status = fdb_iterator_get(itr, &rdoc);
377        TEST_CHK(status == FDB_RESULT_SUCCESS);
378        TEST_CMP(rdoc->key, doc[r]->key, rdoc->keylen);
379        TEST_CMP(rdoc->body, doc[r]->body, rdoc->bodylen);
380        r++;
381        fdb_doc_free(rdoc);
382        rdoc = NULL;
383    } while (fdb_iterator_next(itr) == FDB_RESULT_SUCCESS);
384    fdb_iterator_close(itr);
385    TEST_CHK(r == n);
386
387    // same for sequence number
388    fdb_iterator_sequence_init(db3, &itr, 0, 0, FDB_ITR_NONE);
389    r = 0;
390    do {
391        status = fdb_iterator_get(itr, &rdoc);
392        TEST_CHK(status == FDB_RESULT_SUCCESS);
393        r++;
394        TEST_CHK(rdoc->seqnum == (fdb_seqnum_t)r);
395        fdb_doc_free(rdoc);
396        rdoc = NULL;
397    } while (fdb_iterator_next(itr) == FDB_RESULT_SUCCESS);
398    fdb_iterator_close(itr);
399    TEST_CHK(r == n);
400
401    fdb_close(dbfile);
402
403    // free all documents
404    for (i=0;i<n;++i){
405        fdb_doc_free(doc[i]);
406    }
407
408    fdb_shutdown();
409
410    memleak_end();
411    TEST_RESULT("iterator with concurrent updates test");
412}
413
414void iterator_compact_uncommitted_db()
415{
416    TEST_INIT();
417    memleak_start();
418
419    int i, r;
420    int n = 10;
421    char keybuf[256], bodybuf[256];
422    fdb_file_handle *dbfile;
423    fdb_kvs_handle *db;
424    fdb_status status;
425    fdb_iterator *it;
426    fdb_config fconfig = fdb_get_default_config();
427    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
428    fconfig.wal_threshold = 1024;
429    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
430    fconfig.flags = FDB_OPEN_FLAG_CREATE;
431
432    // remove previous iterator_test files
433    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
434    (void)r;
435
436    // open db
437    fdb_open(&dbfile, "./iterator_test1", &fconfig);
438    fdb_kvs_open(dbfile, &db, "db1", &kvs_config);
439
440
441    // set docs
442    for (i=0;i<n;++i){
443        sprintf(keybuf, "key%d", i);
444        sprintf(bodybuf, "body%d", i);
445        fdb_set_kv(db, keybuf, strlen(keybuf),
446                bodybuf, strlen(bodybuf));
447    }
448
449    // count number of iteratable docs
450    status = fdb_iterator_sequence_init(db, &it, 0, 0, FDB_ITR_NONE);
451    TEST_CHK(status == FDB_RESULT_SUCCESS);
452    i=0;
453    do { ++i; }
454    while (fdb_iterator_next(it) != FDB_RESULT_ITERATOR_FAIL);
455    fdb_iterator_close(it);
456    TEST_CHK(i == n);
457
458    // compact
459    fdb_compact(dbfile, NULL);
460
461    // set again
462    for (i=0;i<n;++i){
463        sprintf(keybuf, "key%d", i);
464        sprintf(bodybuf, "body%d", i);
465        fdb_set_kv(db, keybuf, strlen(keybuf),
466                bodybuf, strlen(bodybuf));
467    }
468
469    // count number of iteratable docs
470    status = fdb_iterator_sequence_init(db, &it, 0, 0, FDB_ITR_NONE);
471    TEST_CHK(status == FDB_RESULT_SUCCESS);
472    i=0;
473    do { ++i; }
474    while (fdb_iterator_next(it) != FDB_RESULT_ITERATOR_FAIL);
475    fdb_iterator_close(it);
476    TEST_CHK(i == n);
477
478
479    fdb_kvs_close(db);
480    fdb_close(dbfile);
481    fdb_shutdown();
482
483    memleak_end();
484    TEST_RESULT("iterator compact uncommitted db");
485}
486
487void iterator_seek_test()
488{
489    TEST_INIT();
490
491    memleak_start();
492
493    int i, r;
494    int n = 10;
495    fdb_file_handle *dbfile;
496    fdb_kvs_handle *db, *kv1;
497    fdb_doc **doc = alca(fdb_doc*, n);
498    fdb_doc *rdoc = NULL;
499    fdb_status status;
500    fdb_iterator *iterator;
501
502    char keybuf[256], metabuf[256], bodybuf[256];
503
504    // remove previous iterator_test files
505    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
506    (void)r;
507
508    fdb_config fconfig = fdb_get_default_config();
509    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
510    fconfig.buffercache_size = 0;
511    fconfig.wal_threshold = 1024;
512    fconfig.flags = FDB_OPEN_FLAG_CREATE;
513    fconfig.compaction_threshold = 0;
514    fconfig.purging_interval = 1; // retain deletes before compaction
515
516    // open db
517    fdb_open(&dbfile, "./iterator_test1", &fconfig);
518    fdb_kvs_open_default(dbfile, &db, &kvs_config);
519    status = fdb_set_log_callback(db, logCallbackFunc,
520                                  (void *) "iterator_seek_test");
521    TEST_CHK(status == FDB_RESULT_SUCCESS);
522
523    // Create another KV store..
524    status = fdb_kvs_open(dbfile, &kv1, "kv1", &kvs_config);
525    TEST_CHK(status == FDB_RESULT_SUCCESS);
526
527    // insert using 'kv1' instance to ensure it does not interfere
528    for (i=0;i<n;++i){
529        sprintf(keybuf, "kEy%d", i);
530        sprintf(metabuf, "mEta%d", i);
531        sprintf(bodybuf, "bOdy%d", i);
532        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
533            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
534        status = fdb_set(kv1, doc[i]);
535        TEST_CHK(status == FDB_RESULT_SUCCESS);
536        fdb_doc_free(doc[i]);
537    }
538
539    // insert documents of odd number into the main (default) KV store handle
540    for (i=1;i<n;i+=2){
541        sprintf(keybuf, "key%d", i);
542        sprintf(metabuf, "meta%d", i);
543        sprintf(bodybuf, "body%d", i);
544        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
545            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
546        fdb_set(db, doc[i]);
547    }
548    // manually flush WAL & commit
549    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
550
551    // insert documents of even number
552    for (i=0;i<n;i+=2){
553        sprintf(keybuf, "key%d", i);
554        sprintf(metabuf, "meta%d", i);
555        sprintf(bodybuf, "body%d", i);
556        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
557            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
558        fdb_set(db, doc[i]);
559    }
560    // commit without WAL flush
561    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
562
563    // now odd number docs are in hb-trie & even number docs are in WAL
564
565    // create an iterator for full range
566    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, FDB_ITR_NONE);
567
568    // seek current iterator to inside the WAL's avl tree..
569    status = fdb_iterator_get(iterator, &rdoc);
570    TEST_CHK(status == FDB_RESULT_SUCCESS);
571
572    TEST_CMP(rdoc->key, doc[0]->key, rdoc->keylen);
573    TEST_CMP(rdoc->meta, doc[0]->meta, rdoc->metalen);
574    TEST_CMP(rdoc->body, doc[0]->body, rdoc->bodylen);
575    fdb_doc_free(rdoc);
576    rdoc = NULL;
577
578    // seek forward to 2nd key ..
579    i=2;
580    status = fdb_iterator_seek(iterator, doc[i]->key, strlen(keybuf), 0);
581    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
582    status = fdb_iterator_get(iterator, &rdoc);
583    TEST_CHK(status == FDB_RESULT_SUCCESS);
584
585    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
586    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
587    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
588    fdb_doc_free(rdoc);
589    rdoc = NULL;
590
591    // iterator should be able to proceed forward
592    status = fdb_iterator_next(iterator);
593    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
594    status = fdb_iterator_get(iterator, &rdoc);
595    TEST_CHK(status == FDB_RESULT_SUCCESS);
596
597    i++;
598    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
599    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
600    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
601    fdb_doc_free(rdoc);
602    rdoc = NULL;
603
604    // seek forward to the last key.
605    status = fdb_iterator_seek(iterator, doc[n-1]->key, strlen(keybuf), 0);
606    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
607    status = fdb_iterator_get(iterator, &rdoc);
608    TEST_CHK(status == FDB_RESULT_SUCCESS);
609
610    TEST_CMP(rdoc->key, doc[n-1]->key, rdoc->keylen);
611    TEST_CMP(rdoc->meta, doc[n-1]->meta, rdoc->metalen);
612    TEST_CMP(rdoc->body, doc[n-1]->body, rdoc->bodylen);
613    fdb_doc_free(rdoc);
614    rdoc = NULL;
615
616    // seek backward to start key ..
617    i = 0;
618    status = fdb_iterator_seek(iterator, doc[i]->key, strlen(keybuf), 0);
619    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
620    status = fdb_iterator_get(iterator, &rdoc);
621    TEST_CHK(status == FDB_RESULT_SUCCESS);
622
623    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
624    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
625    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
626    fdb_doc_free(rdoc);
627    rdoc = NULL;
628
629    // seek forward to key2 ..
630    status = fdb_iterator_seek(iterator, doc[2]->key, strlen(keybuf), 0);
631    TEST_CHK(status == FDB_RESULT_SUCCESS);
632
633    i=2;
634    do {
635        status = fdb_iterator_get(iterator, &rdoc);
636        TEST_CHK(status == FDB_RESULT_SUCCESS);
637
638        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
639        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
640        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
641
642        fdb_doc_free(rdoc);
643        rdoc = NULL;
644        i++;
645    } while(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
646    TEST_CHK(i==10);
647
648    // Seek backward again to a key in the trie...
649    status = fdb_iterator_seek(iterator, doc[3]->key, strlen(keybuf), 0);
650    TEST_CHK(status == FDB_RESULT_SUCCESS);
651
652    i=3;
653    do {
654        status = fdb_iterator_get(iterator, &rdoc);
655        TEST_CHK(status == FDB_RESULT_SUCCESS);
656
657        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
658        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
659        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
660
661        fdb_doc_free(rdoc);
662        rdoc = NULL;
663        i++;
664    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
665    TEST_CHK(i==10);
666
667    fdb_iterator_close(iterator);
668
669    // Test fdb_iterator_seek_to_max with key range
670    // create an iterator for range of doc[4] ~ doc[8]
671    sprintf(keybuf, "key%d", 4); // reuse buffer for start key
672    sprintf(metabuf, "key%d", 8); // reuse buffer for end_key
673    status = fdb_iterator_init(db, &iterator, keybuf, strlen(keybuf),
674                      metabuf, strlen(metabuf),
675                      FDB_ITR_NO_DELETES);
676    TEST_CHK(status == FDB_RESULT_SUCCESS);
677    i = 8;
678    status = fdb_iterator_seek_to_max(iterator);
679    TEST_CHK(status == FDB_RESULT_SUCCESS);
680    status = fdb_iterator_get(iterator, &rdoc);
681    TEST_CHK(status == FDB_RESULT_SUCCESS);
682
683    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
684    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
685    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
686    fdb_doc_free(rdoc);
687    rdoc = NULL;
688
689    // test fdb_iterator_seek_to_min
690    i = 4;
691    status = fdb_iterator_seek_to_min(iterator);
692    TEST_CHK(status == FDB_RESULT_SUCCESS);
693    status = fdb_iterator_get(iterator, &rdoc);
694    TEST_CHK(status == FDB_RESULT_SUCCESS);
695
696    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
697    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
698    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
699    fdb_doc_free(rdoc);
700    rdoc = NULL;
701
702    status = fdb_iterator_close(iterator);
703    TEST_CHK(status == FDB_RESULT_SUCCESS);
704
705    // Test fdb_iterator_seek_to_max over full range
706    status = fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, FDB_ITR_NONE);
707    TEST_CHK(status == FDB_RESULT_SUCCESS);
708
709    i = n - 1;
710    status = fdb_iterator_seek_to_max(iterator);
711    TEST_CHK(status == FDB_RESULT_SUCCESS);
712    status = fdb_iterator_get(iterator, &rdoc);
713    TEST_CHK(status == FDB_RESULT_SUCCESS);
714
715    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
716    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
717    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
718    fdb_doc_free(rdoc);
719    rdoc = NULL;
720
721    // test fdb_iterator_seek_to_min
722    i = 0;
723    status = fdb_iterator_seek_to_min(iterator);
724    TEST_CHK(status == FDB_RESULT_SUCCESS);
725    status = fdb_iterator_get(iterator, &rdoc);
726    TEST_CHK(status == FDB_RESULT_SUCCESS);
727
728    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
729    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
730    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
731    fdb_doc_free(rdoc);
732    rdoc = NULL;
733
734    // seek to max using a non-existent FFFF key
735    if (status == FDB_RESULT_SUCCESS) {
736        uint8_t *big_seek_key = alca(uint8_t, FDB_MAX_KEYLEN);
737        memset(big_seek_key, 0xff, FDB_MAX_KEYLEN);
738        status = fdb_iterator_seek(iterator, big_seek_key, FDB_MAX_KEYLEN,
739                                   FDB_ITR_SEEK_LOWER);
740        TEST_CHK(status == FDB_RESULT_SUCCESS);
741        i = n - 1;
742        status = fdb_iterator_get(iterator, &rdoc);
743        TEST_CHK(status == FDB_RESULT_SUCCESS);
744
745        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
746        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
747        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
748        fdb_doc_free(rdoc);
749        rdoc = NULL;
750    }
751
752    status = fdb_iterator_close(iterator);
753    TEST_CHK(status == FDB_RESULT_SUCCESS);
754
755    // delete documents of even number so WAL only has deleted docs
756    for (i=0;i<n;i+=2){
757        status = fdb_del(db, doc[i]);
758        TEST_CHK(status == FDB_RESULT_SUCCESS);
759    }
760    // commit without WAL flush
761    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
762
763    // create an iterator for full range
764    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, FDB_ITR_NONE);
765
766    // seek forward to 2nd key ..
767    rdoc = NULL;
768    i=2;
769    status = fdb_iterator_seek(iterator, doc[i]->key, strlen(keybuf), 0);
770    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
771    status = fdb_iterator_get(iterator, &rdoc);
772    TEST_CHK(status == FDB_RESULT_SUCCESS);
773
774    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
775    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
776    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
777    fdb_doc_free(rdoc);
778    rdoc = NULL;
779
780    status = fdb_iterator_close(iterator);
781    TEST_CHK(status == FDB_RESULT_SUCCESS);
782
783    // close the kvs kv1
784    fdb_kvs_close(kv1);
785    // close db file
786    fdb_kvs_close(db);
787    fdb_close(dbfile);
788
789    // free all documents
790    for (i=0;i<n;++i){
791        fdb_doc_free(doc[i]);
792    }
793
794    // free all resources
795    fdb_shutdown();
796
797    memleak_end();
798
799    TEST_RESULT("iterator seek test");
800}
801
802void iterator_complete_test(int insert_opt, int delete_opt)
803{
804    TEST_INIT();
805
806    int n = 30;
807    int i, r, c;
808    int *doc_status = alca(int, n); // 0:HB+trie, 1:WAL, 2:deleted
809    char cmd[256];
810    char key[256], value[256];
811    char keystr[] = "key%06d";
812    char keystr_mid[] = "key%06d+";
813    char valuestr[] = "value%08d";
814    char valuestr2[] = "value%08d(WAL)";
815    fdb_file_handle *dbfile;
816    fdb_kvs_handle *db, *db_prev, *db_next;
817    fdb_config config;
818    fdb_kvs_config kvs_config;
819    fdb_doc *doc = NULL;
820    fdb_iterator *fit;
821    fdb_iterator_opt_t itr_opt;
822    fdb_status s;
823    uint64_t mask = 0x11111111111; //0x11111111111
824
825    sprintf(cmd, SHELL_DEL " iterator_test*");
826    r = system(cmd);
827    (void)r;
828
829    memleak_start();
830
831    config = fdb_get_default_config();
832    config.purging_interval = 1; // retain deletes before compaction
833    kvs_config = fdb_get_default_kvs_config();
834    s = fdb_open(&dbfile, "./iterator_test", &config);
835    TEST_CHK(s == FDB_RESULT_SUCCESS);
836
837    s = fdb_kvs_open(dbfile, &db_prev, "prev KVS", &kvs_config);
838    TEST_CHK(s == FDB_RESULT_SUCCESS);
839    s = fdb_kvs_open(dbfile, &db, "cur KVS", &kvs_config);
840    TEST_CHK(s == FDB_RESULT_SUCCESS);
841    s = fdb_kvs_open(dbfile, &db_next, "next KVS", &kvs_config);
842    TEST_CHK(s == FDB_RESULT_SUCCESS);
843
844    sprintf(key, "prev_key");
845    sprintf(value, "prev_value");
846    s = fdb_set_kv(db_prev, key, strlen(key)+1, value, strlen(value)+1);
847    TEST_CHK(s == FDB_RESULT_SUCCESS);
848
849    sprintf(key, "next_key");
850    sprintf(value, "next_value");
851    s = fdb_set_kv(db_next, key, strlen(key)+1, value, strlen(value)+1);
852    TEST_CHK(s == FDB_RESULT_SUCCESS);
853
854    sprintf(key, "next_key2");
855    sprintf(value, "next_value2");
856    s = fdb_set_kv(db_next, key, strlen(key)+1, value, strlen(value)+1);
857    TEST_CHK(s == FDB_RESULT_SUCCESS);
858
859    s = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
860    TEST_CHK(s == FDB_RESULT_SUCCESS);
861
862    if (insert_opt == 0) {
863        // HB+trie contains all keys
864        // WAL contains even number keys only
865        for (i=0;i<n;++i){
866            sprintf(key, keystr, (int)i);
867            sprintf(value, valuestr, (int)i);
868            s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
869            TEST_CHK(s == FDB_RESULT_SUCCESS);
870            doc_status[i] = 0;
871        }
872        s = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
873        TEST_CHK(s == FDB_RESULT_SUCCESS);
874        if (delete_opt) {
875            // remove doc #15
876            i = 15;
877            sprintf(key, keystr, (int)i);
878            s = fdb_del_kv(db, key, strlen(key)+1);
879            TEST_CHK(s == FDB_RESULT_SUCCESS);
880            s = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
881            TEST_CHK(s == FDB_RESULT_SUCCESS);
882            doc_status[i] = 2;
883        }
884
885        for (i=0;i<n;i+=2){
886            sprintf(key, keystr, (int)i);
887            sprintf(value, valuestr2, (int)i);
888            s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
889            TEST_CHK(s == FDB_RESULT_SUCCESS);
890            doc_status[i] = 1;
891        }
892        s = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
893        TEST_CHK(s == FDB_RESULT_SUCCESS);
894    } else if (insert_opt == 1) {
895        // HB+trie contains all keys
896        // WAL contains odd number keys only
897        for (i=0;i<n;++i){
898            sprintf(key, keystr, (int)i);
899            sprintf(value, valuestr, (int)i);
900            s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
901            TEST_CHK(s == FDB_RESULT_SUCCESS);
902            doc_status[i] = 0;
903        }
904        s = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
905        TEST_CHK(s == FDB_RESULT_SUCCESS);
906        for (i=1;i<n;i+=2){
907            sprintf(key, keystr, (int)i);
908            sprintf(value, valuestr2, (int)i);
909            s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
910            TEST_CHK(s == FDB_RESULT_SUCCESS);
911            doc_status[i] = 1;
912        }
913        if (delete_opt) {
914            // remove doc #15
915            i = 15;
916            sprintf(key, keystr, (int)i);
917            s = fdb_del_kv(db, key, strlen(key)+1);
918            TEST_CHK(s == FDB_RESULT_SUCCESS);
919            doc_status[i] = 2;
920        }
921        s = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
922        TEST_CHK(s == FDB_RESULT_SUCCESS);
923    } else if (insert_opt == 2) {
924        // HB+trie contains odd number keys
925        // WAL contains even number keys
926        for (i=1;i<n;i+=2){
927            sprintf(key, keystr, (int)i);
928            sprintf(value, valuestr, (int)i);
929            s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
930            TEST_CHK(s == FDB_RESULT_SUCCESS);
931            doc_status[i] = 0;
932        }
933        s = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
934        TEST_CHK(s == FDB_RESULT_SUCCESS);
935        if (delete_opt) {
936            // remove doc #15
937            i = 15;
938            sprintf(key, keystr, (int)i);
939            s = fdb_del_kv(db, key, strlen(key)+1);
940            TEST_CHK(s == FDB_RESULT_SUCCESS);
941            s = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
942            TEST_CHK(s == FDB_RESULT_SUCCESS);
943            doc_status[i] = 2;
944        }
945
946        for (i=0;i<n;i+=2){
947            sprintf(key, keystr, (int)i);
948            sprintf(value, valuestr2, (int)i);
949            s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
950            TEST_CHK(s == FDB_RESULT_SUCCESS);
951            doc_status[i] = 1;
952        }
953        s = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
954        TEST_CHK(s == FDB_RESULT_SUCCESS);
955    } else if (insert_opt == 3) {
956        // HB+trie contains even number keys
957        // WAL contains odd number keys
958        for (i=0;i<n;i+=2){
959            sprintf(key, keystr, (int)i);
960            sprintf(value, valuestr, (int)i);
961            s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
962            TEST_CHK(s == FDB_RESULT_SUCCESS);
963            doc_status[i] = 0;
964        }
965        s = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
966        TEST_CHK(s == FDB_RESULT_SUCCESS);
967        for (i=1;i<n;i+=2){
968            sprintf(key, keystr, (int)i);
969            sprintf(value, valuestr2, (int)i);
970            s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
971            TEST_CHK(s == FDB_RESULT_SUCCESS);
972            doc_status[i] = 1;
973        }
974        if (delete_opt) {
975            // remove doc #15
976            i = 15;
977            sprintf(key, keystr, (int)i);
978            s = fdb_del_kv(db, key, strlen(key)+1);
979            TEST_CHK(s == FDB_RESULT_SUCCESS);
980            doc_status[i] = 2;
981        }
982        s = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
983        TEST_CHK(s == FDB_RESULT_SUCCESS);
984    } else if (insert_opt == 4) {
985        // HB+trie contains all keys
986        // WAL is empty
987        for (i=0;i<n;i+=1){
988            sprintf(key, keystr, (int)i);
989            sprintf(value, valuestr, (int)i);
990            s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
991            TEST_CHK(s == FDB_RESULT_SUCCESS);
992            doc_status[i] = 0;
993        }
994        s = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
995        TEST_CHK(s == FDB_RESULT_SUCCESS);
996        if (delete_opt) {
997            // remove doc #15
998            i = 15;
999            sprintf(key, keystr, (int)i);
1000            s = fdb_del_kv(db, key, strlen(key)+1);
1001            TEST_CHK(s == FDB_RESULT_SUCCESS);
1002            doc_status[i] = 2;
1003        }
1004        s = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1005        TEST_CHK(s == FDB_RESULT_SUCCESS);
1006    } else if (insert_opt == 5) {
1007        // HB+trie is empty
1008        // WAL contains all keys
1009        for (i=0;i<n;i+=1){
1010            sprintf(key, keystr, (int)i);
1011            sprintf(value, valuestr, (int)i);
1012            s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
1013            TEST_CHK(s == FDB_RESULT_SUCCESS);
1014            doc_status[i] = 0;
1015        }
1016        s = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1017        TEST_CHK(s == FDB_RESULT_SUCCESS);
1018        if (delete_opt) {
1019            // remove doc #15
1020            i = 15;
1021            sprintf(key, keystr, (int)i);
1022            s = fdb_del_kv(db, key, strlen(key)+1);
1023            TEST_CHK(s == FDB_RESULT_SUCCESS);
1024            doc_status[i] = 2;
1025        }
1026        s = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1027        TEST_CHK(s == FDB_RESULT_SUCCESS);
1028    } else { // if (insert_opt == 6) {
1029        // Both HB+trie and WAL contains all keys
1030        for (i=0;i<n;++i){
1031            sprintf(key, keystr, (int)i);
1032            sprintf(value, valuestr, (int)i);
1033            s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
1034            TEST_CHK(s == FDB_RESULT_SUCCESS);
1035            doc_status[i] = 0;
1036        }
1037        s = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1038        TEST_CHK(s == FDB_RESULT_SUCCESS);
1039        for (i=0;i<n;i++){
1040            sprintf(key, keystr, (int)i);
1041            sprintf(value, valuestr2, (int)i);
1042            s = fdb_set_kv(db, key, strlen(key)+1, value, strlen(value)+1);
1043            TEST_CHK(s == FDB_RESULT_SUCCESS);
1044            doc_status[i] = 1;
1045        }
1046        if (delete_opt) {
1047            // remove doc #15
1048            i = 15;
1049            sprintf(key, keystr, (int)i);
1050            s = fdb_del_kv(db, key, strlen(key)+1);
1051            TEST_CHK(s == FDB_RESULT_SUCCESS);
1052            doc_status[i] = 2;
1053        }
1054        s = fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1055        TEST_CHK(s == FDB_RESULT_SUCCESS);
1056    }
1057
1058    if (delete_opt) {
1059        itr_opt = FDB_ITR_NO_DELETES;
1060    } else {
1061        itr_opt = FDB_ITR_NONE;
1062    }
1063
1064    s = fdb_iterator_init(db, &fit, NULL, 0, NULL, 0, itr_opt);
1065    TEST_CHK(s == FDB_RESULT_SUCCESS);
1066    if (mask & 0x1) {
1067        c = 0;
1068        do {
1069            s = fdb_iterator_get(fit, &doc);
1070            if (s != FDB_RESULT_SUCCESS) {
1071                if (s == FDB_RESULT_KEY_NOT_FOUND) {
1072                    continue;
1073                } else {
1074                    break;
1075                }
1076            }
1077            c += ((doc_status[c] == 2)?(1):(0));
1078            sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1079            TEST_CMP(doc->body, value, doc->bodylen);
1080            fdb_doc_free(doc);
1081            doc = NULL;
1082            c++;
1083        } while (fdb_iterator_next(fit) != FDB_RESULT_ITERATOR_FAIL);
1084        TEST_CHK(c == n);
1085        while (fdb_iterator_prev(fit) != FDB_RESULT_ITERATOR_FAIL) {
1086            c--;
1087            s = fdb_iterator_get(fit, &doc);
1088            if (s != FDB_RESULT_SUCCESS) {
1089                if (s == FDB_RESULT_KEY_NOT_FOUND) {
1090                    continue;
1091                } else {
1092                    break;
1093                }
1094            }
1095            c -= ((doc_status[c] == 2)?(1):(0));
1096            sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1097            TEST_CMP(doc->body, value, doc->bodylen);
1098            fdb_doc_free(doc);
1099            doc = NULL;
1100        }
1101        TEST_CHK(c == 0);
1102    }
1103
1104    if (mask & 0x10) {
1105        for (i=0;i<n;++i){
1106            sprintf(key, keystr, (int)i);
1107            s = fdb_iterator_seek(fit, key, strlen(key)+1, 0x0);
1108            TEST_CHK(s == FDB_RESULT_SUCCESS);
1109            c = i;
1110            if (doc_status[c] == 2) {
1111                c++; // higher mode
1112            }
1113            do {
1114                s = fdb_iterator_get(fit, &doc);
1115                if (s != FDB_RESULT_SUCCESS) break;
1116                sprintf(key, keystr, (int)c);
1117                c += ((doc_status[c] == 2)?(1):(0));
1118                sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1119                TEST_CMP(doc->body, value, doc->bodylen);
1120                fdb_doc_free(doc);
1121                doc = NULL;
1122                c++;
1123            } while (fdb_iterator_next(fit) != FDB_RESULT_ITERATOR_FAIL);
1124            TEST_CHK(c == n);
1125        }
1126    }
1127
1128    if (mask & 0x100) {
1129        for (i=0;i<n;++i){
1130            sprintf(key, keystr, (int)i);
1131            s = fdb_iterator_seek(fit, key, strlen(key)+1, 0x0);
1132            TEST_CHK(s == FDB_RESULT_SUCCESS);
1133            c = i;
1134            if (doc_status[c] == 2) {
1135                c++; // higher mode
1136            }
1137            do {
1138                s = fdb_iterator_get(fit, &doc);
1139                if (s != FDB_RESULT_SUCCESS) break;
1140                sprintf(key, keystr, (int)c);
1141                c -= ((doc_status[c] == 2)?(1):(0));
1142                sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1143                TEST_CMP(doc->body, value, doc->bodylen);
1144                fdb_doc_free(doc);
1145                doc = NULL;
1146                c--;
1147            } while (fdb_iterator_prev(fit) != FDB_RESULT_ITERATOR_FAIL);
1148            TEST_CHK(c == -1);
1149        }
1150    }
1151
1152    if (mask & 0x1000) {
1153        for (i=0;i<n;++i){
1154            sprintf(key, keystr, (int)i);
1155            s = fdb_iterator_seek(fit, key, strlen(key)+1, FDB_ITR_SEEK_LOWER);
1156            TEST_CHK(s == FDB_RESULT_SUCCESS);
1157            c = i;
1158            if (doc_status[c] == 2) {
1159                c--; // lower mode
1160            }
1161            do {
1162                s = fdb_iterator_get(fit, &doc);
1163                if (s != FDB_RESULT_SUCCESS) break;
1164                sprintf(key, keystr, (int)c);
1165                c += ((doc_status[c] == 2)?(1):(0));
1166                sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1167                TEST_CMP(doc->body, value, doc->bodylen);
1168                fdb_doc_free(doc);
1169                doc = NULL;
1170                c++;
1171            } while (fdb_iterator_next(fit) != FDB_RESULT_ITERATOR_FAIL);
1172            TEST_CHK(c == n);
1173        }
1174    }
1175
1176    if (mask & 0x10000) {
1177        for (i=0;i<n;++i){
1178            sprintf(key, keystr, (int)i);
1179            s = fdb_iterator_seek(fit, key, strlen(key)+1, FDB_ITR_SEEK_LOWER);
1180            TEST_CHK(s == FDB_RESULT_SUCCESS);
1181            c = i;
1182            if (doc_status[c] == 2) {
1183                c--; // lower mode
1184            }
1185            do {
1186                s = fdb_iterator_get(fit, &doc);
1187                if (s != FDB_RESULT_SUCCESS) break;
1188                sprintf(key, keystr, (int)c);
1189                c -= ((doc_status[c] == 2)?(1):(0));
1190                sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1191                TEST_CMP(doc->body, value, doc->bodylen);
1192                fdb_doc_free(doc);
1193                doc = NULL;
1194                c--;
1195            } while (fdb_iterator_prev(fit) != FDB_RESULT_ITERATOR_FAIL);
1196            TEST_CHK(c == -1);
1197        }
1198    }
1199
1200    if (mask & 0x100000) {
1201        for (i=0;i<n;++i){
1202            sprintf(key, keystr_mid, (int)i);
1203            s = fdb_iterator_seek(fit, key, strlen(key)+1, 0x0);
1204            (void)s;
1205            c = i+1;
1206            if (doc_status[c] == 2) {
1207                c++; // higher mode
1208            }
1209            do {
1210                s = fdb_iterator_get(fit, &doc);
1211                if (s != FDB_RESULT_SUCCESS) break;
1212                sprintf(key, keystr_mid, (int)c);
1213                c += ((doc_status[c] == 2)?(1):(0));
1214                sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1215                TEST_CMP(doc->body, value, doc->bodylen);
1216                fdb_doc_free(doc);
1217                doc = NULL;
1218                c++;
1219            } while (fdb_iterator_next(fit) != FDB_RESULT_ITERATOR_FAIL);
1220            TEST_CHK(c == n);
1221        }
1222    }
1223
1224    if (mask & 0x1000000) {
1225        for (i=0;i<n;++i){
1226            sprintf(key, keystr_mid, (int)i);
1227            s = fdb_iterator_seek(fit, key, strlen(key)+1, 0x0);
1228            (void)s;
1229            c = i+1;
1230            if (doc_status[c] == 2) {
1231                c++; // higher mode
1232            }
1233            do {
1234                s = fdb_iterator_get(fit, &doc);
1235                if (s != FDB_RESULT_SUCCESS) break;
1236                sprintf(key, keystr_mid, (int)c);
1237                c -= ((doc_status[c] == 2)?(1):(0));
1238                sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1239                TEST_CMP(doc->body, value, doc->bodylen);
1240                fdb_doc_free(doc);
1241                doc = NULL;
1242                c--;
1243            } while (fdb_iterator_prev(fit) != FDB_RESULT_ITERATOR_FAIL);
1244            if (i == n-1) {
1245                TEST_CHK(c == n);
1246            } else {
1247                TEST_CHK(c == -1);
1248            }
1249        }
1250    }
1251
1252    if (mask & 0x10000000) {
1253        for (i=0;i<n;++i){
1254            sprintf(key, keystr_mid, (int)i);
1255            s = fdb_iterator_seek(fit, key, strlen(key)+1, FDB_ITR_SEEK_LOWER);
1256            TEST_CHK(s == FDB_RESULT_SUCCESS);
1257            c = i;
1258            if (doc_status[c] == 2) {
1259                c--; // lower mode
1260            }
1261            do {
1262                s = fdb_iterator_get(fit, &doc);
1263                if (s != FDB_RESULT_SUCCESS) break;
1264                sprintf(key, keystr_mid, (int)c);
1265                c += ((doc_status[c] == 2)?(1):(0));
1266                sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1267                TEST_CMP(doc->body, value, doc->bodylen);
1268                fdb_doc_free(doc);
1269                doc = NULL;
1270                c++;
1271            } while (fdb_iterator_next(fit) != FDB_RESULT_ITERATOR_FAIL);
1272            TEST_CHK(c == n);
1273        }
1274    }
1275
1276    if (mask & 0x100000000) {
1277        for (i=0;i<n;++i){
1278            sprintf(key, keystr_mid, (int)i);
1279            s = fdb_iterator_seek(fit, key, strlen(key)+1, FDB_ITR_SEEK_LOWER);
1280            TEST_CHK(s == FDB_RESULT_SUCCESS);
1281            c = i;
1282            if (doc_status[c] == 2) {
1283                c--; // lower mode
1284            }
1285            do {
1286                s = fdb_iterator_get(fit, &doc);
1287                if (s != FDB_RESULT_SUCCESS) break;
1288                sprintf(key, keystr_mid, (int)c);
1289                c -= ((doc_status[c] == 2)?(1):(0));
1290                sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1291                TEST_CMP(doc->body, value, doc->bodylen);
1292                fdb_doc_free(doc);
1293                doc = NULL;
1294                c--;
1295            } while (fdb_iterator_prev(fit) != FDB_RESULT_ITERATOR_FAIL);
1296            TEST_CHK(c == -1);
1297        }
1298    }
1299
1300    if (mask & 0x1000000000) {
1301        s = fdb_iterator_seek_to_min(fit);
1302        TEST_CHK(s == FDB_RESULT_SUCCESS);
1303        c = 0;
1304        do {
1305            s = fdb_iterator_get(fit, &doc);
1306            if (s != FDB_RESULT_SUCCESS) break;
1307            sprintf(key, keystr_mid, (int)c);
1308            c += ((doc_status[c] == 2)?(1):(0));
1309            sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1310            TEST_CMP(doc->body, value, doc->bodylen);
1311            fdb_doc_free(doc);
1312            doc = NULL;
1313            c++;
1314        } while (fdb_iterator_next(fit) != FDB_RESULT_ITERATOR_FAIL);
1315        TEST_CHK(c == n);
1316
1317        s = fdb_iterator_seek_to_max(fit);
1318        TEST_CHK(s == FDB_RESULT_SUCCESS);
1319        c = n-1;
1320        do {
1321            s = fdb_iterator_get(fit, &doc);
1322            if (s != FDB_RESULT_SUCCESS) break;
1323            sprintf(key, keystr_mid, (int)c);
1324            c -= ((doc_status[c] == 2)?(1):(0));
1325            sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1326            TEST_CMP(doc->body, value, doc->bodylen);
1327            fdb_doc_free(doc);
1328            doc = NULL;
1329            c--;
1330        } while (fdb_iterator_prev(fit) != FDB_RESULT_ITERATOR_FAIL);
1331        TEST_CHK(c == -1);
1332    }
1333    s = fdb_iterator_close(fit);
1334    TEST_CHK(s == FDB_RESULT_SUCCESS);
1335
1336    if (mask & 0x10000000000) {
1337        // create an iterator with an end key and skip max key option
1338        i = n/3*2;
1339        sprintf(key, keystr, (int)i);
1340        s = fdb_iterator_init(db, &fit, NULL, 0, key, strlen(key)+1,
1341                              FDB_ITR_SKIP_MAX_KEY);
1342        TEST_CHK(s == FDB_RESULT_SUCCESS);
1343        s = fdb_iterator_seek_to_max(fit);
1344        TEST_CHK(s == FDB_RESULT_SUCCESS);
1345        s = fdb_iterator_get(fit, &doc);
1346        TEST_CHK(s == FDB_RESULT_SUCCESS);
1347        c = i-1;
1348        sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1349        TEST_CMP(doc->body, value, doc->bodylen);
1350        fdb_doc_free(doc);
1351        doc = NULL;
1352        s = fdb_iterator_close(fit);
1353        TEST_CHK(s == FDB_RESULT_SUCCESS);
1354
1355        // create an iterator with an start key and skip min key option
1356        i = n/3;
1357        sprintf(key, keystr, (int)i);
1358        s = fdb_iterator_init(db, &fit, key, strlen(key)+1, NULL, 0,
1359                              FDB_ITR_SKIP_MIN_KEY);
1360        TEST_CHK(s == FDB_RESULT_SUCCESS);
1361        s = fdb_iterator_seek_to_min(fit);
1362        TEST_CHK(s == FDB_RESULT_SUCCESS);
1363        s = fdb_iterator_get(fit, &doc);
1364        TEST_CHK(s == FDB_RESULT_SUCCESS);
1365        c = i+1;
1366        sprintf(value, (doc_status[c]==0)?(valuestr):(valuestr2), c);
1367        TEST_CMP(doc->body, value, doc->bodylen);
1368        fdb_doc_free(doc);
1369        doc = NULL;
1370        s = fdb_iterator_close(fit);
1371        TEST_CHK(s == FDB_RESULT_SUCCESS);
1372
1373        s = fdb_close(dbfile);
1374        TEST_CHK(s == FDB_RESULT_SUCCESS);
1375        s = fdb_shutdown();
1376        TEST_CHK(s == FDB_RESULT_SUCCESS);
1377        memleak_end();
1378    }
1379
1380    sprintf(cmd, "iterator complete test ");
1381    if (insert_opt == 0) {
1382        strcat(cmd, "(HB+trie: all, WAL: even");
1383    } else if (insert_opt == 1) {
1384        strcat(cmd, "(HB+trie: all, WAL: odd");
1385    } else if (insert_opt == 2) {
1386        strcat(cmd, "(HB+trie: odd, WAL: even");
1387    } else if (insert_opt == 3) {
1388        strcat(cmd, "(HB+trie: even, WAL: odd");
1389    } else if (insert_opt == 4) {
1390        strcat(cmd, "(HB+trie: all, WAL: empty");
1391    } else if (insert_opt == 5) {
1392        strcat(cmd, "(HB+trie: empty, WAL: all");
1393    } else if (insert_opt == 6) {
1394        strcat(cmd, "(HB+trie: all, WAL: all");
1395    }
1396    if (delete_opt) {
1397        strcat(cmd, ", doc deletion)");
1398    } else {
1399        strcat(cmd, ")");
1400    }
1401    TEST_RESULT(cmd);
1402}
1403
1404void iterator_extreme_key_test()
1405{
1406    TEST_INIT();
1407
1408    int n = 30;
1409    int i, r, c;
1410    char cmd[256];
1411    char key[256], value[256];
1412    char keyBuf[256], valueBuf[256];
1413    fdb_file_handle *dbfile;
1414    fdb_kvs_handle *db;
1415    fdb_config config;
1416    fdb_kvs_config kvs_config;
1417    fdb_doc *doc = NULL;
1418    fdb_iterator *fit;
1419    fdb_status s;
1420
1421    sprintf(cmd, SHELL_DEL " iterator_test*");
1422    r = system(cmd);
1423    (void)r;
1424
1425    memleak_start();
1426
1427    config = fdb_get_default_config();
1428    kvs_config = fdb_get_default_kvs_config();
1429    s = fdb_open(&dbfile, "./iterator_test", &config);
1430    TEST_CHK(s == FDB_RESULT_SUCCESS);
1431    s = fdb_kvs_open(dbfile, &db, NULL, &kvs_config);
1432    TEST_CHK(s == FDB_RESULT_SUCCESS);
1433
1434    memset(key, 0xff, 256);
1435    for (i=1;i<n;i+=1){
1436        sprintf(value, "0xff length %d", (int)i);
1437        fdb_set_kv(db, key, i, value, strlen(value)+1);
1438    }
1439    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1440
1441    // Pre-allocate iterator return document memory and re-use the same
1442    s = fdb_doc_create(&doc, NULL, 0, NULL, 0, NULL, 0);
1443    TEST_CHK(s == FDB_RESULT_SUCCESS);
1444    doc->key = &keyBuf[0];
1445    doc->meta = NULL;
1446    doc->body = &valueBuf[0];
1447
1448    s = fdb_iterator_init(db, &fit, NULL, 0, NULL, 0, 0x0);
1449    TEST_CHK(s == FDB_RESULT_SUCCESS);
1450    s = fdb_iterator_seek_to_max(fit);
1451    TEST_CHK(s == FDB_RESULT_SUCCESS);
1452    c = n-1;
1453    while (s == FDB_RESULT_SUCCESS) {
1454        s = fdb_iterator_get(fit, &doc);
1455        if (s != FDB_RESULT_SUCCESS) {
1456            break;
1457        }
1458        sprintf(value, "0xff length %d", (int)c);
1459        TEST_CMP(doc->body, value, doc->bodylen);
1460        s = fdb_iterator_prev(fit);
1461        c--;
1462    }
1463
1464    i = 8;
1465    c = i;
1466    s = fdb_iterator_seek(fit, key, i, FDB_ITR_SEEK_LOWER);
1467    while (s == FDB_RESULT_SUCCESS) {
1468        s = fdb_iterator_get(fit, &doc);
1469        if (s != FDB_RESULT_SUCCESS) {
1470            break;
1471        }
1472        sprintf(value, "0xff length %d", (int)c);
1473        TEST_CMP(doc->body, value, doc->bodylen);
1474        s = fdb_iterator_prev(fit);
1475        c--;
1476    }
1477
1478    i = 8;
1479    c = i;
1480    s = fdb_iterator_seek(fit, key, i, FDB_ITR_SEEK_LOWER);
1481    while (s == FDB_RESULT_SUCCESS) {
1482        s = fdb_iterator_get(fit, &doc);
1483        if (s != FDB_RESULT_SUCCESS) {
1484            break;
1485        }
1486        sprintf(value, "0xff length %d", (int)c);
1487        TEST_CMP(doc->body, value, doc->bodylen);
1488        s = fdb_iterator_next(fit);
1489        c++;
1490    }
1491
1492    s = fdb_iterator_close(fit);
1493    TEST_CHK(s == FDB_RESULT_SUCCESS);
1494
1495    // Initialize iterator to an extreme non-existent end_key
1496    s = fdb_iterator_init(db, &fit, NULL, 0, key, 256, 0x0);
1497    TEST_CHK(s == FDB_RESULT_SUCCESS);
1498    s = fdb_iterator_seek_to_max(fit);
1499    TEST_CHK(s == FDB_RESULT_SUCCESS);
1500
1501    c = n-1;
1502    while (s == FDB_RESULT_SUCCESS) {
1503        s = fdb_iterator_get(fit, &doc);
1504        if (s != FDB_RESULT_SUCCESS) {
1505            break;
1506        }
1507        sprintf(value, "0xff length %d", (int)c);
1508        TEST_CMP(doc->body, value, doc->bodylen);
1509        s = fdb_iterator_prev(fit);
1510        c--;
1511    }
1512
1513    s = fdb_iterator_close(fit);
1514    TEST_CHK(s == FDB_RESULT_SUCCESS);
1515    s = fdb_close(dbfile);
1516    TEST_CHK(s == FDB_RESULT_SUCCESS);
1517
1518    // Release pre-allocated iterator return document buffer space
1519    doc->key = NULL;
1520    doc->body = NULL;
1521    s = fdb_doc_free(doc);
1522    TEST_CHK(s == FDB_RESULT_SUCCESS);
1523
1524    s = fdb_shutdown();
1525    TEST_CHK(s == FDB_RESULT_SUCCESS);
1526    memleak_end();
1527    TEST_RESULT("iterator extreme key test");
1528}
1529
1530void iterator_inmem_snapshot_seek_test(bool flush_wal)
1531{
1532    TEST_INIT();
1533
1534    memleak_start();
1535
1536    int i, r;
1537    int n = 5;
1538    fdb_file_handle *dbfile;
1539    fdb_kvs_handle *db;
1540    fdb_kvs_handle *snap_db;
1541    fdb_doc **doc = alca(fdb_doc*, n);
1542    fdb_doc *rdoc = NULL;
1543    fdb_status status;
1544    fdb_iterator *iterator;
1545
1546    char keybuf[256], metabuf[256], bodybuf[256];
1547
1548    // remove previous mvcc_test files
1549    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
1550    (void)r;
1551
1552    fdb_config fconfig = fdb_get_default_config();
1553    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1554    fconfig.buffercache_size = 0;
1555    fconfig.wal_threshold = 1024;
1556    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1557    fconfig.compaction_threshold = 0;
1558
1559    // open db
1560    status = fdb_open(&dbfile, "./iterator_test1", &fconfig);
1561    TEST_CHK(status == FDB_RESULT_SUCCESS);
1562    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
1563    TEST_CHK(status == FDB_RESULT_SUCCESS);
1564
1565    // ------- Setup test ----------------------------------
1566    for (i=0; i<n; i++){
1567        sprintf(keybuf, "%c2",(char)i + 'a');
1568        sprintf(metabuf, "meta%d", i);
1569        sprintf(bodybuf, "body%d", i);
1570        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1571            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1572        fdb_set(db, doc[i]);
1573    }
1574
1575    if (flush_wal) {
1576        fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1577    } else {
1578        fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1579    }
1580    // ---------- Snapshot tests begin -----------------------
1581    // WAL items are not flushed...
1582    status = fdb_snapshot_open(db, &snap_db, FDB_SNAPSHOT_INMEM);
1583    TEST_CHK(status == FDB_RESULT_SUCCESS);
1584
1585    status = fdb_set_log_callback(db, logCallbackFunc,
1586                                  (void *) "iterator_inmem_snapshot_seek_test");
1587    TEST_CHK(status == FDB_RESULT_SUCCESS);
1588
1589    // create an iterator on the snapshot for full range
1590    fdb_iterator_init(snap_db, &iterator, (void*)"b2", 2, (void*)"d2", 2,
1591                      FDB_ITR_NO_DELETES|
1592                      FDB_ITR_SKIP_MAX_KEY|
1593                      FDB_ITR_SKIP_MIN_KEY);
1594
1595    // seek to non-existent key that happens to land on the start key which
1596    // should not be returned since we have passed ITR_SKIP_MIN_KEY
1597    status = fdb_iterator_seek(iterator, "c1", 2, FDB_ITR_SEEK_LOWER);
1598    TEST_CHK(status == FDB_RESULT_ITERATOR_FAIL);
1599
1600    // seek to non-existent key that happens to land on the end key which
1601    // should not be returned since we have passed ITR_SKIP_MAX_KEY
1602    status = fdb_iterator_seek(iterator, "c3", 2, FDB_ITR_SEEK_HIGHER);
1603    TEST_CHK(status == FDB_RESULT_ITERATOR_FAIL);
1604
1605    fdb_iterator_close(iterator);
1606
1607    // create an iterator on the snapshot for full range
1608    fdb_iterator_init(snap_db, &iterator, (void*)"b3", 2, (void*)"d1", 2,
1609                      FDB_ITR_NO_DELETES|
1610                      FDB_ITR_SKIP_MAX_KEY|
1611                      FDB_ITR_SKIP_MIN_KEY);
1612
1613    // seek to non-existent key that happens to land on key that is
1614    // smaller than the start key which should not be returned.
1615    status = fdb_iterator_seek(iterator, "c1", 2, FDB_ITR_SEEK_LOWER);
1616    TEST_CHK(status == FDB_RESULT_ITERATOR_FAIL);
1617
1618    // seek to non-existent key that happens to be land on key larger than
1619    // end key which should not be returned.
1620    status = fdb_iterator_seek(iterator, "c3", 2, FDB_ITR_SEEK_HIGHER);
1621    TEST_CHK(status == FDB_RESULT_ITERATOR_FAIL);
1622
1623    fdb_iterator_close(iterator);
1624
1625    // create an iterator on the snapshot for just 3 items within range
1626    fdb_iterator_init(snap_db, &iterator, (void*)"b0", 2, (void*)"e2", 2,
1627                      FDB_ITR_NO_DELETES|
1628                      FDB_ITR_SKIP_MAX_KEY|
1629                      FDB_ITR_SKIP_MIN_KEY);
1630
1631    // seek to max key but skip max key
1632    // should return a key for fdb_iterator_seek_to_max
1633    status = fdb_iterator_seek_to_max(iterator);
1634    TEST_CHK(status == FDB_RESULT_SUCCESS);
1635    status = fdb_iterator_get(iterator, &rdoc);
1636    TEST_CHK(status == FDB_RESULT_SUCCESS);
1637    TEST_CMP(rdoc->key, "d2", rdoc->keylen);
1638    fdb_doc_free(rdoc);
1639    rdoc = NULL;
1640
1641    // But same attempt with regular seek to max key
1642    // no key should be returned since we want to skip max key
1643    status = fdb_iterator_seek(iterator, "e2", 2, FDB_ITR_SEEK_HIGHER);
1644    TEST_CHK(status == FDB_RESULT_ITERATOR_FAIL);
1645
1646    // seek to min key but skip min key
1647    // should return a key for fdb_iterator_seek_to_min
1648    status = fdb_iterator_seek_to_min(iterator);
1649    TEST_CHK(status == FDB_RESULT_SUCCESS);
1650    status = fdb_iterator_get(iterator, &rdoc);
1651    TEST_CHK(status == FDB_RESULT_SUCCESS);
1652    TEST_CMP(rdoc->key, "c2", rdoc->keylen);
1653    fdb_doc_free(rdoc);
1654
1655    // But same attempt with regular seek to min key
1656    // no key should be returned since we want to skip min key
1657    status = fdb_iterator_seek(iterator, "b0", 2, FDB_ITR_SEEK_LOWER);
1658    TEST_CHK(status == FDB_RESULT_ITERATOR_FAIL);
1659
1660    // seek to key outside the range that happens to land on non-existent key
1661    // no key should be returned
1662    status = fdb_iterator_seek(iterator, "b0", 2, FDB_ITR_SEEK_LOWER);
1663    TEST_CHK(status == FDB_RESULT_ITERATOR_FAIL);
1664
1665    fdb_iterator_close(iterator);
1666
1667    // close db handle
1668    fdb_kvs_close(db);
1669    // close snapshot handle
1670    fdb_kvs_close(snap_db);
1671
1672    // close db file
1673    fdb_close(dbfile);
1674
1675    // free all documents
1676    for (i=0;i<n;++i){
1677        fdb_doc_free(doc[i]);
1678    }
1679
1680    // free all resources
1681    fdb_shutdown();
1682
1683    memleak_end();
1684
1685    TEST_RESULT("in-memory snapshot seek test");
1686}
1687
1688void iterator_no_deletes_test()
1689{
1690
1691    TEST_INIT();
1692    memleak_start();
1693    int i, r, n = 10;
1694    fdb_file_handle *dbfile;
1695    fdb_kvs_handle  *kv;
1696    char keybuf[256], bodybuf[256];
1697    fdb_doc **doc = alca(fdb_doc*, n);
1698    fdb_doc *rdoc = NULL;
1699    fdb_iterator *it;
1700    fdb_status status;
1701
1702    // remove previous iterator_test files
1703    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
1704    (void)r;
1705
1706    fdb_config fconfig = fdb_get_default_config();
1707    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1708    fconfig.wal_threshold = 1024;
1709    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1710    fconfig.compaction_threshold = 0;
1711
1712    // open db
1713    fdb_open(&dbfile, "./iterator_test", &fconfig);
1714    fdb_kvs_open(dbfile, &kv, "all_docs",  &kvs_config);
1715
1716    // insert docs to kv
1717    for (i=0;i<n;++i){
1718        sprintf(keybuf, "key%d", i);
1719        sprintf(bodybuf, "body%d", i);
1720        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf), NULL, 0,
1721                       (void*)bodybuf, strlen(bodybuf));
1722        fdb_set(kv, doc[i]);
1723    }
1724
1725    // delete all docs
1726    for (i=0;i<n;i++){
1727        status = fdb_del_kv(kv, doc[i]->key, doc[i]->keylen);
1728        TEST_CHK(status == FDB_RESULT_SUCCESS);
1729    }
1730
1731    // commit
1732    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1733
1734    // set doc that was deleted
1735    status = fdb_set(kv, doc[2]);
1736    TEST_CHK(status == FDB_RESULT_SUCCESS);
1737
1738    // get doc from db and verify not-deleted
1739    fdb_doc_create(&rdoc, doc[2]->key, doc[2]->keylen, NULL, 0, NULL, 0);
1740    status = fdb_get(kv, rdoc);
1741    TEST_CHK(status == FDB_RESULT_SUCCESS);
1742    TEST_CHK(rdoc->deleted == false);
1743    fdb_doc_free(rdoc);
1744    rdoc = NULL;
1745
1746    // iterate over all docs to retrieve undeleted key
1747    status = fdb_iterator_init(kv, &it, NULL, 0, NULL, 0, FDB_ITR_NO_DELETES);
1748    TEST_CHK(status == FDB_RESULT_SUCCESS);
1749    status = fdb_iterator_get(it, &rdoc);
1750    TEST_CHK(status == FDB_RESULT_SUCCESS);
1751    if (status == FDB_RESULT_SUCCESS){
1752        fdb_doc_free(rdoc);
1753    }
1754    fdb_iterator_close(it);
1755
1756    for (i=0;i<n;++i){
1757        fdb_doc_free(doc[i]);
1758    }
1759
1760    fdb_kvs_close(kv);
1761    fdb_close(dbfile);
1762    fdb_shutdown();
1763
1764    memleak_end();
1765    TEST_RESULT("iterator no deletes test");
1766}
1767
1768void iterator_set_del_docs_test()
1769{
1770    TEST_INIT();
1771    memleak_start();
1772
1773    int r;
1774    int i, j, k, n=100;
1775    int expected_doc_count=0;
1776    char keybuf[256], metabuf[256], bodybuf[256];
1777    int val2;
1778    fdb_file_handle *dbfile;
1779    fdb_iterator *it;
1780    fdb_kvs_handle *kv1;
1781    fdb_kvs_info info;
1782    fdb_config fconfig = fdb_get_default_config();
1783    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1784    fdb_doc **doc = alca(fdb_doc*, n);
1785    fdb_doc *vdoc;
1786    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
1787    (void)r;
1788
1789    fconfig.wal_threshold = 1024;
1790    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1791    fconfig.compaction_threshold = 10;
1792    fconfig.purging_interval = 1; //retain deletes until compaction
1793
1794    fdb_open(&dbfile, "./iterator_test1", &fconfig);
1795    fdb_kvs_open(dbfile, &kv1, "kv1", &kvs_config);
1796
1797    for(k=0;k<20;++k){
1798        // set n docs
1799        for(i=0;i<n;++i){
1800            sprintf(keybuf, "key%02d%03d", k, i);
1801            sprintf(metabuf, "meta%02d%03d", k, i);
1802            sprintf(bodybuf, "body%02d%03d", k, i);
1803            fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1804                (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1805            fdb_set(kv1, doc[i]);
1806            expected_doc_count++;
1807        }
1808
1809        // commit
1810        fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1811
1812        // delete subset of recently loaded docs
1813        for(j=n/4;j<n/2;j++){
1814            fdb_del(kv1, doc[j]);
1815            expected_doc_count--;
1816        }
1817        fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1818
1819        fdb_get_kvs_info(kv1, &info);
1820        if(info.doc_count != (size_t)expected_doc_count){
1821            // test already failed further debugging check info
1822            fdb_iterator_init(kv1, &it, NULL, 0,
1823                              NULL, 0, FDB_ITR_NONE);
1824            val2=0;
1825            do {
1826                fdb_iterator_get(it, &vdoc);
1827                if (!vdoc->deleted){
1828                    val2++;
1829                }
1830                fdb_doc_free(vdoc);
1831            } while (fdb_iterator_next(it) != FDB_RESULT_ITERATOR_FAIL);
1832            fdb_iterator_close(it);
1833            printf("dbdocs(%d) expected(%d)\n", val2, expected_doc_count);
1834        }
1835        TEST_CHK(info.doc_count == (size_t)expected_doc_count);
1836
1837        // preliminary cleanup
1838        for(i=0;i<n;++i){
1839            fdb_doc_free(doc[i]);
1840        }
1841    }
1842
1843    fdb_kvs_close(kv1);
1844    fdb_close(dbfile);
1845    fdb_shutdown();
1846    memleak_end();
1847
1848    TEST_RESULT("iterator set del docs");
1849}
1850
1851void iterator_del_next_test()
1852{
1853    TEST_INIT();
1854
1855    memleak_start();
1856
1857    int i, r;
1858    int n = 10;
1859    fdb_file_handle *dbfile;
1860    fdb_kvs_handle *db;
1861    fdb_doc **doc = alca(fdb_doc*, n);
1862    fdb_doc pre_alloc_doc;
1863    fdb_doc *rdoc = &pre_alloc_doc;
1864    fdb_status status;
1865    fdb_iterator *iterator;
1866
1867    char keybuf[256], metabuf[256], bodybuf[256];
1868    rdoc->key = keybuf;
1869    rdoc->meta = metabuf;
1870    rdoc->body = bodybuf;
1871    rdoc->flags = 0;
1872
1873    // remove previous iterator_test files
1874    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
1875    (void)r;
1876
1877    fdb_config fconfig = fdb_get_default_config();
1878    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1879    fconfig.buffercache_size = 0;
1880    fconfig.wal_threshold = 1024;
1881    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1882    fconfig.compaction_threshold = 0;
1883
1884    // open db
1885    fdb_open(&dbfile, "./iterator_test1", &fconfig);
1886    fdb_kvs_open(dbfile, &db, "kv1", &kvs_config);
1887
1888    status = fdb_set_log_callback(db, logCallbackFunc,
1889                                  (void *) "iterator_del_next_test");
1890    TEST_CHK(status == FDB_RESULT_SUCCESS);
1891
1892    // insert documents of even number
1893    for (i=0;i<n;i+=2){
1894        sprintf(keybuf, "key%d", i);
1895        sprintf(metabuf, "meta%d", i);
1896        sprintf(bodybuf, "body%d", i);
1897        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1898            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1899        fdb_set(db, doc[i]);
1900    }
1901    // manually flush WAL & commit
1902    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1903
1904    // insert documents of odd number
1905    for (i=1;i<n;i+=2){
1906        sprintf(keybuf, "key%d", i);
1907        sprintf(metabuf, "meta%d", i);
1908        sprintf(bodybuf, "body%d", i);
1909        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
1910            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
1911        fdb_set(db, doc[i]);
1912    }
1913    // commit without WAL flush
1914    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
1915
1916    // now even number docs are in hb-trie & odd number docs are in WAL
1917
1918    // create an iterator for full range
1919    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, FDB_ITR_NONE);
1920
1921    // repeat until fail
1922    i=0;
1923    do {
1924        status = fdb_iterator_get(iterator, &rdoc);
1925        TEST_CHK(status == FDB_RESULT_SUCCESS);
1926
1927        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
1928        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
1929        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
1930        // Delete the document to ensure that the iteration is not affected
1931        status = fdb_del(db, rdoc);
1932        TEST_CHK(status == FDB_RESULT_SUCCESS);
1933        status = fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
1934        TEST_CHK(status == FDB_RESULT_SUCCESS);
1935        i++;
1936    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
1937    TEST_CHK(i==10);
1938
1939    // go back to start and retry iteration (should not be affected by deletes)
1940    status = fdb_iterator_seek_to_min(iterator);
1941    TEST_CHK(status == FDB_RESULT_SUCCESS);
1942
1943    // repeat full iteration until fail
1944    i=0;
1945    do {
1946        status = fdb_iterator_get(iterator, &rdoc);
1947        TEST_CHK(status == FDB_RESULT_SUCCESS);
1948
1949        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
1950        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
1951        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
1952        i++;
1953    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
1954    TEST_CHK(i==10);
1955
1956    fdb_iterator_close(iterator);
1957
1958    fdb_close(dbfile);
1959    fdb_shutdown();
1960
1961    // free all documents
1962    for (i=0;i<n;++i){
1963        fdb_doc_free(doc[i]);
1964    }
1965
1966    memleak_end();
1967
1968    TEST_RESULT("iterator del next test");
1969}
1970
1971void sequence_iterator_test()
1972{
1973    TEST_INIT();
1974
1975    memleak_start();
1976
1977    int i, r;
1978    int n = 10;
1979    int count;
1980    fdb_file_handle *dbfile;
1981    fdb_kvs_handle *db;
1982    fdb_doc **doc = alca(fdb_doc*, n);
1983    fdb_doc *rdoc = NULL;
1984    fdb_status status;
1985    fdb_iterator *iterator;
1986
1987    char keybuf[256], metabuf[256], bodybuf[256];
1988
1989    // remove previous iterator_test files
1990    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
1991    (void)r;
1992
1993    fdb_config fconfig = fdb_get_default_config();
1994    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
1995    fconfig.buffercache_size = 0;
1996    fconfig.wal_threshold = 1024;
1997    fconfig.flags = FDB_OPEN_FLAG_CREATE;
1998    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
1999    fconfig.compaction_threshold = 0;
2000    fconfig.purging_interval = 1; // retain deletes until compaction
2001
2002    // open db
2003    fdb_open(&dbfile, "./iterator_test1", &fconfig);
2004    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2005    status = fdb_set_log_callback(db, logCallbackFunc,
2006                                  (void *) "sequence_iterator_test");
2007    TEST_CHK(status == FDB_RESULT_SUCCESS);
2008
2009    // insert documents of even number
2010    for (i=0;i<n;i+=2){
2011        sprintf(keybuf, "key%d", i);
2012        sprintf(metabuf, "meta%d", i);
2013        sprintf(bodybuf, "body%d", i);
2014        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2015            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2016        fdb_set(db, doc[i]);
2017    }
2018    // manually flush WAL & commit
2019    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2020
2021    // insert documents of odd number
2022    for (i=1;i<n;i+=2){
2023        sprintf(keybuf, "key%d", i);
2024        sprintf(metabuf, "meta%d", i);
2025        sprintf(bodybuf, "body%d", i);
2026        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2027            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2028        fdb_set(db, doc[i]);
2029    }
2030    // commit without WAL flush
2031    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2032
2033    // now even number docs are in hb-trie & odd number docs are in WAL
2034
2035    // create an iterator over sequence number range over FULL RANGE
2036    fdb_iterator_sequence_init(db, &iterator, 0, 0, FDB_ITR_NONE);
2037
2038    // repeat until fail
2039    i=0;
2040    count = 0;
2041    do {
2042        status = fdb_iterator_get(iterator, &rdoc);
2043        TEST_CHK(status == FDB_RESULT_SUCCESS);
2044
2045        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2046        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2047        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2048
2049        fdb_doc_free(rdoc);
2050        rdoc = NULL;
2051        i = (i + 2 >= n) ? 1: i + 2; // by-seq, first come even docs, then odd
2052        count++;
2053    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2054    TEST_CHK(count==n);
2055    fdb_iterator_close(iterator);
2056
2057    // create an iterator over sequence number.
2058    fdb_iterator_sequence_init(db, &iterator, 0, 0, FDB_ITR_NONE);
2059
2060    // repeat until fail
2061    i=0;
2062    count = 0;
2063    do {
2064        status = fdb_iterator_get_metaonly(iterator, &rdoc);
2065        TEST_CHK(status == FDB_RESULT_SUCCESS);
2066
2067        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2068        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2069        TEST_CHK(rdoc->body == NULL);
2070
2071        fdb_doc_free(rdoc);
2072        rdoc = NULL;
2073        i = (i + 2 >= n) ? 1: i + 2; // by-seq, first come even docs, then odd
2074        count++;
2075    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2076    TEST_CHK(count==n);
2077    fdb_iterator_close(iterator);
2078
2079    // create another iterator starts from seq number 2 and ends at 9
2080    fdb_iterator_sequence_init(db, &iterator, 2, 7, FDB_ITR_NONE);
2081
2082    // repeat until fail
2083    i=2;
2084    count = 0;
2085    do {
2086        status = fdb_iterator_get(iterator, &rdoc);
2087        TEST_CHK(status == FDB_RESULT_SUCCESS);
2088
2089        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2090        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2091        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2092
2093        fdb_doc_free(rdoc);
2094        rdoc = NULL;
2095        i = (i + 2 >= n) ? 1: i + 2; // by-seq, first come even docs, then odd
2096        count++;
2097    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2098    TEST_CHK(count==6);
2099    fdb_iterator_close(iterator);
2100    // remove document #8 and #9
2101    fdb_doc_create(&rdoc, doc[8]->key, doc[8]->keylen, doc[8]->meta, doc[8]->metalen, NULL, 0);
2102    status = fdb_del(db, rdoc);
2103    TEST_CHK(status == FDB_RESULT_SUCCESS);
2104    fdb_doc_free(rdoc);
2105    rdoc = NULL;
2106    fdb_doc_create(&rdoc, doc[9]->key, doc[9]->keylen, doc[9]->meta, doc[9]->metalen, NULL, 0);
2107    status = fdb_del(db, rdoc);
2108    TEST_CHK(status == FDB_RESULT_SUCCESS);
2109    fdb_doc_free(rdoc);
2110    rdoc = NULL;
2111    // commit
2112    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2113
2114    // create an iterator for full range
2115    fdb_iterator_sequence_init(db, &iterator, 0, 0, FDB_ITR_NONE);
2116    // repeat until fail
2117    i=0;
2118    count=0;
2119    do {
2120        status = fdb_iterator_get(iterator, &rdoc);
2121        TEST_CHK(status == FDB_RESULT_SUCCESS);
2122
2123        if (count != 8 && count != 9) { // do not look validate key8 and key9
2124            TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2125            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2126            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2127        } else {
2128            TEST_CHK(rdoc->deleted == true);
2129        }
2130
2131        fdb_doc_free(rdoc);
2132        rdoc = NULL;
2133        // Turn around when we hit 8 as the last items, key8 and key9 are gone
2134        i = (i + 2 >= 8) ? 1: i + 2; // by-seq, first come even docs, then odd
2135        count++;
2136    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2137    TEST_CHK(count==10); // 10 items, with 2 deletions
2138    fdb_iterator_close(iterator);
2139
2140    // create an iterator for full range, but no deletes.
2141    fdb_iterator_sequence_init(db, &iterator, 0, 0, FDB_ITR_NO_DELETES);
2142    // repeat until fail
2143    i=0;
2144    count=0;
2145    do {
2146        status = fdb_iterator_get(iterator, &rdoc);
2147        TEST_CHK(status == FDB_RESULT_SUCCESS);
2148
2149        if (i != 8 && i != 9) { // key8 and key9 are deleted
2150            TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2151            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2152            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2153        }
2154
2155        fdb_doc_free(rdoc);
2156        rdoc = NULL;
2157        i = (i + 2 >= 8) ? 1: i + 2; // by-seq, first come even docs, then odd
2158        count++;
2159    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2160    TEST_CHK(count==8); // 10 items, with 2 deletions
2161    fdb_iterator_close(iterator);
2162
2163    // Update first document and test for absence of duplicates
2164    *((char *)doc[0]->body) = 'K'; // update key0 to Key0
2165    fdb_set(db, doc[0]);
2166    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2167
2168    fdb_iterator_sequence_init(db, &iterator, 0, 0, FDB_ITR_NO_DELETES);
2169    // repeat until fail
2170    i=2; // i == 0 should not appear until the end
2171    count=0;
2172    do {
2173        status = fdb_iterator_get(iterator, &rdoc);
2174        TEST_CHK(status == FDB_RESULT_SUCCESS);
2175
2176        if (i != 8 && i != 9) { // key8 and key9 are deleted
2177            TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2178            TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2179            TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2180        }
2181
2182        fdb_doc_free(rdoc);
2183        rdoc = NULL;
2184        i = (i + 2 >= 8) ? 1: i + 2; // by-seq, first come even docs, then odd
2185        if (count == 6) i = 0; // go back to test for i=0 at the end
2186        count++;
2187    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2188    TEST_CHK(count==8); // 10 items, with 2 deletions
2189    fdb_iterator_close(iterator);
2190
2191    // close db file
2192    fdb_kvs_close(db);
2193    fdb_close(dbfile);
2194
2195    // free all documents
2196    for (i=0;i<n;++i){
2197        fdb_doc_free(doc[i]);
2198    }
2199
2200    // free all resources
2201    fdb_shutdown();
2202
2203    memleak_end();
2204
2205    TEST_RESULT("sequence iterator test");
2206}
2207
2208void sequence_iterator_duplicate_test()
2209{
2210    // Unit test for MB-12225
2211    TEST_INIT();
2212
2213    memleak_start();
2214
2215    int i, r;
2216    int n = 100;
2217    int count;
2218    fdb_file_handle *dbfile;
2219    fdb_kvs_handle *db;
2220    fdb_doc **doc = alca(fdb_doc*, n);
2221    fdb_doc *rdoc = NULL;
2222    fdb_status status;
2223    fdb_iterator *iterator;
2224    fdb_seqnum_t seqnum;
2225
2226    char keybuf[256], metabuf[256], bodybuf[256];
2227
2228    // remove previous iterator_test files
2229    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
2230    (void)r;
2231
2232    fdb_config fconfig = fdb_get_default_config();
2233    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2234    fconfig.buffercache_size = 0;
2235    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
2236    fconfig.wal_threshold = 1024;
2237    fconfig.compaction_threshold = 0;
2238    fconfig.purging_interval = 1; // retain deletes until compaction
2239
2240    // open db
2241    fdb_open(&dbfile, "./iterator_test1", &fconfig);
2242    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2243    status = fdb_set_log_callback(db, logCallbackFunc,
2244                                  (void *) "sequence_iterator_test");
2245    TEST_CHK(status == FDB_RESULT_SUCCESS);
2246
2247    // insert documents first time
2248    for (i=0;i<n;i++){
2249        sprintf(keybuf, "key%d", i);
2250        sprintf(metabuf, "meta%d", i);
2251        sprintf(bodybuf, "body%d(first)", i);
2252        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2253            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2254        fdb_set(db, doc[i]);
2255    }
2256    // manually flush WAL & commit
2257    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2258
2259    // insert documents second time
2260    for (i=0;i<n;i++){
2261        sprintf(bodybuf, "body%d(second)", i);
2262        fdb_doc_update(&doc[i], NULL, 0, bodybuf, strlen(bodybuf));
2263        fdb_set(db, doc[i]);
2264    }
2265    // manually flush WAL & commit
2266    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2267
2268    // insert documents third time (only even number documents)
2269    for (i=0;i<n;i+=2){
2270        sprintf(bodybuf, "body%d(third)", i);
2271        fdb_doc_update(&doc[i], NULL, 0, bodybuf, strlen(bodybuf));
2272        fdb_set(db, doc[i]);
2273    }
2274    // commit without WAL flushing
2275    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2276
2277    // create an iterator over sequence number
2278    fdb_iterator_sequence_init(db, &iterator, 0, 220, FDB_ITR_NONE);
2279
2280    // repeat until fail
2281    count = 0;
2282    seqnum = 100;
2283    do {
2284        status = fdb_iterator_get(iterator, &rdoc);
2285        TEST_CHK(status == FDB_RESULT_SUCCESS);
2286        if (seqnum < 140) { // point where WAL range & trie range overlap ends!
2287            seqnum += 2; // WAL overlap with trie, get unique trie keys only
2288        } else { // beyond this even keys in trie are also in WAL but outside..
2289            seqnum ++; // the iteration range, so they can be sequentially got
2290        }
2291
2292        if (seqnum <= 200) { // uptil WAL, unique trie items are returned...
2293            i = seqnum - 101;
2294            sprintf(bodybuf, "body%d(second)", i);
2295        } else { // once seqnum enters WAL range only WAL elements are returned..
2296            i = ((seqnum - 101) % n) * 2;
2297            sprintf(bodybuf, "body%d(third)", i);
2298        }
2299        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2300        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2301        TEST_CMP(rdoc->body, bodybuf, rdoc->bodylen);
2302        TEST_CHK(rdoc->seqnum == seqnum);
2303
2304        count++;
2305        fdb_doc_free(rdoc);
2306        rdoc = NULL;
2307    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2308    TEST_CHK(count==n); // since 220 > n all keys should be iterated
2309    fdb_iterator_close(iterator);
2310
2311    // close db file
2312    fdb_kvs_close(db);
2313    fdb_close(dbfile);
2314    // free all documents
2315    for (i=0;i<n;++i){
2316        fdb_doc_free(doc[i]);
2317    }
2318    // free all resources
2319    fdb_shutdown();
2320
2321    memleak_end();
2322
2323    TEST_RESULT("sequence iterator duplicate test");
2324}
2325
2326// MB-16406
2327void sequence_iterator_range_test()
2328{
2329    TEST_INIT();
2330
2331    memleak_start();
2332
2333    int i, r;
2334    int n = 10;
2335    int count;
2336    fdb_file_handle *dbfile;
2337    fdb_kvs_handle *db;
2338    fdb_doc **doc = alca(fdb_doc*, n);
2339    fdb_doc *rdoc = NULL;
2340    fdb_status status;
2341    fdb_iterator *iterator;
2342    fdb_seqnum_t seqnum;
2343
2344    char keybuf[256], metabuf[256], bodybuf[256];
2345
2346    // remove previous iterator_test files
2347    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
2348    (void)r;
2349
2350    fdb_config fconfig = fdb_get_default_config();
2351    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2352    fconfig.buffercache_size = 0;
2353    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
2354
2355    // open db
2356    fdb_open(&dbfile, "./iterator_test1", &fconfig);
2357    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2358    status = fdb_set_log_callback(db, logCallbackFunc,
2359                                  (void *) "sequence_iterator_test");
2360    TEST_CHK(status == FDB_RESULT_SUCCESS);
2361
2362    // insert docs
2363    for (i=0;i<n;i++){
2364        sprintf(keybuf, "key%d", i);
2365        sprintf(metabuf, "meta%d", i);
2366        sprintf(bodybuf, "body%d", i);
2367        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2368                                (void*)metabuf, strlen(metabuf),
2369                                (void*)bodybuf, strlen(bodybuf));
2370        fdb_set(db, doc[i]);
2371    }
2372    // manually flush WAL & commit
2373    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2374
2375    // create a full range seq iterator
2376    status = fdb_iterator_sequence_init(db, &iterator, 0, 0, FDB_ITR_NONE);
2377    TEST_CHK(status == FDB_RESULT_SUCCESS);
2378
2379    // repeat until fail
2380    count = 0;
2381    do {
2382        status = fdb_iterator_get(iterator, &rdoc);
2383        TEST_CHK(status == FDB_RESULT_SUCCESS);
2384        i = count;
2385        seqnum = i+1;
2386
2387        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2388        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2389        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2390        TEST_CHK(rdoc->seqnum == seqnum);
2391
2392        count++;
2393        fdb_doc_free(rdoc);
2394        rdoc = NULL;
2395    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2396    TEST_CHK(count == n);
2397    fdb_iterator_close(iterator);
2398
2399    // create a partial range seq iterator with the given max seq number
2400    status = fdb_iterator_sequence_init(db, &iterator, 0, n/2, FDB_ITR_NONE);
2401    TEST_CHK(status == FDB_RESULT_SUCCESS);
2402
2403    // repeat until fail
2404    count = 0;
2405    do {
2406        status = fdb_iterator_get(iterator, &rdoc);
2407        TEST_CHK(status == FDB_RESULT_SUCCESS);
2408        i = count;
2409        seqnum = i+1;
2410
2411        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2412        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2413        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2414        TEST_CHK(rdoc->seqnum == seqnum);
2415
2416        count++;
2417        fdb_doc_free(rdoc);
2418        rdoc = NULL;
2419    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2420    TEST_CHK(count == n/2);
2421    fdb_iterator_close(iterator);
2422
2423    // create a partial range seq iterator with the given min seq number
2424    status = fdb_iterator_sequence_init(db, &iterator, (n/2)+1, 0, FDB_ITR_NONE);
2425    TEST_CHK(status == FDB_RESULT_SUCCESS);
2426
2427    // repeat until fail
2428    count = 0;
2429    do {
2430        status = fdb_iterator_get(iterator, &rdoc);
2431        TEST_CHK(status == FDB_RESULT_SUCCESS);
2432        i = count + n/2;
2433        seqnum = i+1;
2434
2435        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2436        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2437        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2438        TEST_CHK(rdoc->seqnum == seqnum);
2439
2440        count++;
2441        fdb_doc_free(rdoc);
2442        rdoc = NULL;
2443    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2444    TEST_CHK(count == n/2);
2445    fdb_iterator_close(iterator);
2446
2447    // close db file
2448    fdb_kvs_close(db);
2449    fdb_close(dbfile);
2450    // free all documents
2451    for (i=0;i<n;++i){
2452        fdb_doc_free(doc[i]);
2453    }
2454    // free all resources
2455    fdb_shutdown();
2456
2457    memleak_end();
2458
2459    TEST_RESULT("sequence iterator range test");
2460}
2461
2462void reverse_sequence_iterator_test()
2463{
2464    TEST_INIT();
2465
2466    memleak_start();
2467
2468    int i, r, count;
2469    int n = 10;
2470    fdb_file_handle *dbfile;
2471    fdb_kvs_handle *db;
2472    fdb_doc **doc = alca(fdb_doc*, n);
2473    fdb_doc *rdoc = NULL;
2474    fdb_status status;
2475    fdb_iterator *iterator;
2476
2477    char keybuf[256], metabuf[256], bodybuf[256];
2478
2479    // remove previous iterator_test files
2480    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
2481    (void)r;
2482
2483    fdb_config fconfig = fdb_get_default_config();
2484    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2485    fconfig.buffercache_size = 0;
2486    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
2487    fconfig.wal_threshold = 1024;
2488    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2489    fconfig.compaction_threshold = 0;
2490
2491    // open db
2492    fdb_open(&dbfile, "./iterator_test1", &fconfig);
2493    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2494    status = fdb_set_log_callback(db, logCallbackFunc,
2495                                  (void *) "reverse_sequence_iterator_test");
2496    TEST_CHK(status == FDB_RESULT_SUCCESS);
2497
2498    // insert documents of even number
2499    for (i=0;i<n;i+=2){
2500        sprintf(keybuf, "key%d", i);
2501        sprintf(metabuf, "meta%d", i);
2502        sprintf(bodybuf, "body%d", i);
2503        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2504            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2505        fdb_set(db, doc[i]);
2506    }
2507    // manually flush WAL & commit
2508    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2509
2510    // insert documents of odd number
2511    for (i=1;i<n;i+=2){
2512        sprintf(keybuf, "key%d", i);
2513        sprintf(metabuf, "meta%d", i);
2514        sprintf(bodybuf, "body%d", i);
2515        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2516            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2517        fdb_set(db, doc[i]);
2518    }
2519    // commit without WAL flush
2520    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2521
2522    // now even number docs are in hb-trie & odd number docs are in WAL
2523
2524    // First test reverse sequence iteration as it only involves btrees
2525    // create an iterator over sequence number range over FULL RANGE
2526    fdb_iterator_sequence_init(db, &iterator, 0, 0, FDB_ITR_NONE);
2527
2528    // move iterator forward up till middle...
2529    i=0;
2530    count = 0;
2531    do {
2532        status = fdb_iterator_get(iterator, &rdoc);
2533        TEST_CHK(status == FDB_RESULT_SUCCESS);
2534
2535        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2536        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2537        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2538
2539        fdb_doc_free(rdoc);
2540        rdoc = NULL;
2541        count++;
2542        if (i + 2 >= n) break;
2543        i = i + 2; // by-seq, first come even docs, then odd
2544    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2545    TEST_CHK(count==n/2);
2546
2547    // Now test reverse sequence iterator from mid-way..
2548
2549    i = i - 2;
2550    status = fdb_iterator_prev(iterator);
2551    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
2552    status = fdb_iterator_get(iterator, &rdoc);
2553    TEST_CHK(status == FDB_RESULT_SUCCESS);
2554
2555    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2556    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2557    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2558
2559    fdb_doc_free(rdoc);
2560    rdoc = NULL;
2561    count++;
2562
2563    // change direction to forward again...
2564    TEST_CHK(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2565    i = i + 2;
2566    do {
2567        status = fdb_iterator_get(iterator, &rdoc);
2568        TEST_CHK(status == FDB_RESULT_SUCCESS);
2569
2570        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2571        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2572        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2573
2574        fdb_doc_free(rdoc);
2575        rdoc = NULL;
2576        i = (i + 2 >= n) ? 1 : i + 2;// by-seq, first come even docs, then odd
2577        count++;
2578    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2579
2580    TEST_CHK(count==n+2); // two items were double counted due to reverse
2581
2582    // Reached End, now reverse iterate till start
2583    i = n - 1;
2584    count = n;
2585    status = fdb_iterator_prev(iterator);
2586    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
2587    do {
2588        status = fdb_iterator_get(iterator, &rdoc);
2589        TEST_CHK(status == FDB_RESULT_SUCCESS);
2590
2591        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2592        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2593        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2594
2595        fdb_doc_free(rdoc);
2596        rdoc = NULL;
2597        i = (i - 2 < 0) ? n - 2 : i - 2;
2598        if (count) count--;
2599    } while (fdb_iterator_prev(iterator) != FDB_RESULT_ITERATOR_FAIL);
2600    TEST_CHK(count == 0);
2601    fdb_iterator_close(iterator);
2602
2603    // close db file
2604    fdb_kvs_close(db);
2605    fdb_close(dbfile);
2606
2607    // free all documents
2608    for (i=0;i<n;++i){
2609        fdb_doc_free(doc[i]);
2610    }
2611
2612    // free all resources
2613    fdb_shutdown();
2614
2615    memleak_end();
2616
2617    TEST_RESULT("reverse sequence iterator test");
2618}
2619
2620
2621void reverse_sequence_iterator_kvs_test()
2622{
2623    TEST_INIT();
2624    memleak_start();
2625
2626    int i, r, count;
2627    int n = 10;
2628    fdb_file_handle *dbfile;
2629    fdb_kvs_handle *kv1, *kv2;
2630    fdb_doc **doc = alca(fdb_doc*, n);
2631    fdb_doc **doc2 = alca(fdb_doc*, n);
2632    fdb_doc *rdoc = NULL;
2633    fdb_status status;
2634    fdb_iterator *iterator;
2635    fdb_iterator *iterator2;
2636
2637    char keybuf[256], metabuf[256], bodybuf[256];
2638
2639    // remove previous iterator_test files
2640    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
2641    (void)r;
2642
2643    fdb_config fconfig = fdb_get_default_config();
2644    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2645    fconfig.buffercache_size = 0;
2646    fconfig.seqtree_opt = FDB_SEQTREE_USE; // enable seqtree since get_byseq
2647    fconfig.wal_threshold = 1024;
2648    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2649    fconfig.compaction_threshold = 0;
2650
2651    // open db
2652    fdb_open(&dbfile, "./iterator_test1", &fconfig);
2653    fdb_kvs_open_default(dbfile, &kv1, &kvs_config);
2654    status = fdb_set_log_callback(kv1, logCallbackFunc,
2655                                  (void *) "reverse_sequence_iterator_kvs_test");
2656    TEST_CHK(status == FDB_RESULT_SUCCESS);
2657
2658    // Create another KV store...
2659    status = fdb_kvs_open(dbfile, &kv2, "kv2", &kvs_config);
2660    TEST_CHK(status == FDB_RESULT_SUCCESS);
2661
2662    // to 'kv2' with entire range
2663    for (i=0;i<n;++i) {
2664        sprintf(keybuf, "kEy%d", i);
2665        sprintf(metabuf, "mEta%d", i);
2666        sprintf(bodybuf, "bOdy%d", i);
2667        fdb_doc_create(&doc2[i], (void*)keybuf, strlen(keybuf),
2668            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2669        status = fdb_set(kv2, doc2[i]);
2670        TEST_CHK(status == FDB_RESULT_SUCCESS);
2671    }
2672
2673    // insert kv1 documents of even number
2674    for (i=0;i<n;i+=2) {
2675        sprintf(keybuf, "key%d", i);
2676        sprintf(metabuf, "meta%d", i);
2677        sprintf(bodybuf, "body%d", i);
2678        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2679            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2680        fdb_set(kv1, doc[i]);
2681    }
2682
2683    // manually flush WAL & commit
2684    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2685
2686    // insert kv1 documents of odd number
2687    for (i=1;i<n;i+=2) {
2688        sprintf(keybuf, "key%d", i);
2689        sprintf(metabuf, "meta%d", i);
2690        sprintf(bodybuf, "body%d", i);
2691        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2692            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2693        fdb_set(kv1, doc[i]);
2694    }
2695    // commit without WAL flush
2696    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2697
2698    // iterate even docs on kv1
2699    status = fdb_iterator_sequence_init(kv1, &iterator, 0, 0, FDB_ITR_NONE);
2700    TEST_CHK(status == FDB_RESULT_SUCCESS);
2701
2702    i=0;
2703    count = 0;
2704    while (1) {
2705        fdb_iterator_get(iterator, &rdoc);
2706        TEST_CHK(!memcmp(rdoc->key, doc[i]->key, rdoc->keylen));
2707        TEST_CHK(!memcmp(rdoc->meta, doc[i]->meta, rdoc->metalen));
2708        TEST_CHK(!memcmp(rdoc->body, doc[i]->body, rdoc->bodylen));
2709        fdb_doc_free(rdoc);
2710        rdoc = NULL;
2711        count++;
2712        if (i + 2 >= n) {
2713            break;
2714        }
2715        i = i + 2; // by-seq, first come even docs, then odd
2716        status = fdb_iterator_next(iterator);
2717        if (status == FDB_RESULT_ITERATOR_FAIL) break;
2718    }
2719    TEST_CHK(count==n/2);
2720
2721    // iterate all docs over kv2
2722    status = fdb_iterator_sequence_init(kv2, &iterator2, 0, 0, FDB_ITR_NONE);
2723    TEST_CHK(status == FDB_RESULT_SUCCESS);
2724    while(1) {
2725        status = fdb_iterator_get(iterator2, &rdoc);
2726        TEST_CHK(status == FDB_RESULT_SUCCESS);
2727        fdb_doc_free(rdoc);
2728        rdoc = NULL;
2729        status = fdb_iterator_next(iterator2);
2730        if (status == FDB_RESULT_ITERATOR_FAIL) {
2731            break;
2732        }
2733    }
2734
2735    // manually flush WAL & commit
2736    // iterators should be unaffected
2737    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2738
2739    // reverse iterate even docs over kv1
2740     i = n - 4;
2741    count = 0;
2742    while (1) {
2743        status = fdb_iterator_prev(iterator);
2744        if (status == FDB_RESULT_ITERATOR_FAIL) {
2745            break;
2746        }
2747        status = fdb_iterator_get(iterator, &rdoc);
2748        TEST_CHK(status == FDB_RESULT_SUCCESS);
2749        TEST_CHK(!memcmp(rdoc->key, doc[i]->key, rdoc->keylen));
2750        TEST_CHK(!memcmp(rdoc->meta, doc[i]->meta, rdoc->metalen));
2751        TEST_CHK(!memcmp(rdoc->body, doc[i]->body, rdoc->bodylen));
2752        fdb_doc_free(rdoc);
2753        rdoc = NULL;
2754        i-=2;
2755        count++;
2756    }
2757
2758    TEST_CHK(count==4);
2759    fdb_iterator_close(iterator);
2760
2761    i = n-1;
2762    count = 0;
2763    // reverse iterate all docs over kv2
2764    while (1) {
2765        status = fdb_iterator_prev(iterator2);
2766        if (status == FDB_RESULT_ITERATOR_FAIL) {
2767            break;
2768        }
2769        status = fdb_iterator_get(iterator2, &rdoc);
2770        TEST_CHK(status == FDB_RESULT_SUCCESS);
2771        TEST_CHK(!memcmp(rdoc->key, doc2[i]->key, rdoc->keylen));
2772        TEST_CHK(!memcmp(rdoc->meta, doc2[i]->meta, rdoc->metalen));
2773        TEST_CHK(!memcmp(rdoc->body, doc2[i]->body, rdoc->bodylen));
2774        fdb_doc_free(rdoc);
2775        rdoc = NULL;
2776        i--;
2777        count++;
2778    }
2779    TEST_CHK(count==n);
2780    fdb_iterator_close(iterator2);
2781
2782    // re-open iterator after commit should return all docs for kv1
2783    i = 0;
2784    count = 0;
2785    status = fdb_iterator_sequence_init(kv1, &iterator, 0, 0, FDB_ITR_NONE);
2786    TEST_CHK(status == FDB_RESULT_SUCCESS);
2787    while (1) {
2788        status = fdb_iterator_get(iterator, &rdoc);
2789        TEST_CHK(status == FDB_RESULT_SUCCESS);
2790        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2791        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2792        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2793        fdb_doc_free(rdoc);
2794        rdoc = NULL;
2795        if (i == 8) {
2796            i=1; // switch to odds
2797        } else {
2798            i+=2;
2799        }
2800        count++;
2801        status = fdb_iterator_next(iterator);
2802        if (status == FDB_RESULT_ITERATOR_FAIL) {
2803            break;
2804        }
2805    }
2806    TEST_CHK(count==n);
2807    fdb_iterator_close(iterator);
2808
2809    // free all documents
2810    for (i=0;i<n;++i) {
2811        fdb_doc_free(doc[i]);
2812        fdb_doc_free(doc2[i]);
2813    }
2814
2815    fdb_kvs_close(kv1);
2816    fdb_kvs_close(kv2);
2817    fdb_close(dbfile);
2818
2819    fdb_shutdown();
2820    memleak_end();
2821    TEST_RESULT("reverse sequence iterator kvs test");
2822
2823}
2824
2825void reverse_iterator_test()
2826{
2827    TEST_INIT();
2828
2829    memleak_start();
2830
2831    int i, r;
2832    int n = 10;
2833    fdb_file_handle *dbfile;
2834    fdb_kvs_handle *db;
2835    fdb_doc **doc = alca(fdb_doc*, n);
2836    fdb_doc *rdoc = NULL;
2837    fdb_status status;
2838    fdb_iterator *iterator;
2839
2840    char keybuf[256], metabuf[256], bodybuf[256];
2841
2842    // remove previous iterator_test files
2843    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
2844    (void)r;
2845
2846    fdb_config fconfig = fdb_get_default_config();
2847    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
2848    fconfig.buffercache_size = 0;
2849    fconfig.wal_threshold = 1024;
2850    fconfig.flags = FDB_OPEN_FLAG_CREATE;
2851    fconfig.compaction_threshold = 0;
2852
2853    // open db
2854    fdb_open(&dbfile, "./iterator_test1", &fconfig);
2855    fdb_kvs_open_default(dbfile, &db, &kvs_config);
2856    status = fdb_set_log_callback(db, logCallbackFunc,
2857                                  (void *) "reverse_iterator_test");
2858    TEST_CHK(status == FDB_RESULT_SUCCESS);
2859
2860    // insert documents of even number
2861    for (i=0;i<n;i+=2){
2862        sprintf(keybuf, "key%d", i);
2863        sprintf(metabuf, "meta%d", i);
2864        sprintf(bodybuf, "body%d", i);
2865        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2866            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2867        fdb_set(db, doc[i]);
2868    }
2869    // manually flush WAL & commit
2870    fdb_commit(dbfile, FDB_COMMIT_MANUAL_WAL_FLUSH);
2871
2872    // insert documents of odd number
2873    for (i=1;i<n;i+=2){
2874        sprintf(keybuf, "key%d", i);
2875        sprintf(metabuf, "meta%d", i);
2876        sprintf(bodybuf, "body%d", i);
2877        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
2878            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
2879        fdb_set(db, doc[i]);
2880    }
2881    // commit without WAL flush
2882    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
2883
2884    // now even number docs are in hb-trie & odd number docs are in WAL
2885
2886    // Reverse iteration over key range which involves hb-tries
2887    // create an iterator for full range
2888    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, FDB_ITR_NONE);
2889
2890    // first test forward iterator - repeat until fail
2891    i=0;
2892    do {
2893        status = fdb_iterator_get(iterator, &rdoc);
2894        TEST_CHK(status == FDB_RESULT_SUCCESS);
2895
2896        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2897        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2898        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2899
2900        fdb_doc_free(rdoc);
2901        rdoc = NULL;
2902        i++;
2903    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2904    TEST_CHK(i==10);
2905
2906    // Now test reverse iterator..
2907    for (--i; fdb_iterator_prev(iterator) != FDB_RESULT_ITERATOR_FAIL; --i) {
2908        status = fdb_iterator_get(iterator, &rdoc);
2909        TEST_CHK(status == FDB_RESULT_SUCCESS);
2910
2911        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2912        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2913        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2914
2915        fdb_doc_free(rdoc);
2916        rdoc = NULL;
2917        if (i == 5) break; // Change direction at half point
2918    }
2919    TEST_CHK(i == 5);
2920
2921    // Mid-way reverse direction, again test forward iterator...
2922    i++;
2923    status = fdb_iterator_next(iterator);
2924    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
2925    status = fdb_iterator_get(iterator, &rdoc);
2926    TEST_CHK(status == FDB_RESULT_SUCCESS);
2927
2928    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2929    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2930    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2931
2932    fdb_doc_free(rdoc);
2933    rdoc = NULL;
2934
2935    // Mid-way reverse direction, again test forward iterator...
2936    i++;
2937    status = fdb_iterator_next(iterator);
2938    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
2939    status = fdb_iterator_get(iterator, &rdoc);
2940    TEST_CHK(status == FDB_RESULT_SUCCESS);
2941
2942    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2943    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2944    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2945
2946    fdb_doc_free(rdoc);
2947    rdoc = NULL;
2948
2949    // Again change direction and test reverse iterator..
2950    for (--i; fdb_iterator_prev(iterator) != FDB_RESULT_ITERATOR_FAIL; --i) {
2951        status = fdb_iterator_get(iterator, &rdoc);
2952        TEST_CHK(status == FDB_RESULT_SUCCESS);
2953
2954        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2955        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2956        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2957
2958        fdb_doc_free(rdoc);
2959        rdoc = NULL;
2960    }
2961    TEST_CHK(i == -1);
2962
2963    // Reached end - now test forward iterator...
2964    TEST_CHK(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2965    i++;
2966    do {
2967        status = fdb_iterator_get(iterator, &rdoc);
2968        TEST_CHK(status == FDB_RESULT_SUCCESS);
2969
2970        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
2971        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
2972        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
2973
2974        fdb_doc_free(rdoc);
2975        rdoc = NULL;
2976        i++;
2977    } while (fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
2978    TEST_CHK(i==10);
2979
2980    fdb_iterator_close(iterator);
2981
2982    // close db file
2983    fdb_kvs_close(db);
2984    fdb_close(dbfile);
2985
2986    // free all documents
2987    for (i=0;i<n;++i){
2988        fdb_doc_free(doc[i]);
2989    }
2990
2991    // free all resources
2992    fdb_shutdown();
2993
2994    memleak_end();
2995
2996    TEST_RESULT("reverse iterator test");
2997}
2998
2999void reverse_seek_to_max_nokey(void)
3000{
3001    TEST_INIT();
3002
3003    memleak_start();
3004
3005    int i, r;
3006    int n = 100;
3007    fdb_file_handle *dbfile;
3008    fdb_kvs_handle *db;
3009    fdb_doc **doc = alca(fdb_doc*, n);
3010    fdb_doc *rdoc;
3011    fdb_status status;
3012    fdb_iterator *iterator;
3013
3014    char keybuf[16];
3015
3016    // remove previous mvcc_test files
3017    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
3018    (void)r;
3019
3020    fdb_config fconfig = fdb_get_default_config();
3021    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
3022    fconfig.buffercache_size = 0;
3023
3024    // open db
3025    status = fdb_open(&dbfile, "./iterator_test1", &fconfig);
3026    TEST_CHK(status == FDB_RESULT_SUCCESS);
3027    status = fdb_kvs_open_default(dbfile, &db, &kvs_config);
3028    TEST_CHK(status == FDB_RESULT_SUCCESS);
3029
3030    // ------- Setup test ----------------------------------
3031    for (i=0; i<n; i++){
3032        sprintf(keybuf, "doc-%03d", i);
3033        keybuf[7] = '\0';
3034        keybuf[8] = '\0';
3035        fdb_doc_create(&doc[i], (void*)keybuf, 10,
3036            NULL, 0, (void*)keybuf, 10);
3037        fdb_set(db, doc[i]);
3038    }
3039
3040    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
3041
3042    status = fdb_set_log_callback(db, logCallbackFunc,
3043                                  (void *) "reverse_seek_to_max_nokey");
3044    TEST_CHK(status == FDB_RESULT_SUCCESS);
3045
3046    // set range to have end key that does not exist
3047    status = fdb_iterator_init(db, &iterator, doc[24]->key, 10,
3048                       (void*)"doc-029b", 10,
3049                      FDB_ITR_NO_DELETES);
3050    TEST_CHK(status == FDB_RESULT_SUCCESS);
3051
3052    status = fdb_iterator_seek_to_max(iterator);
3053    TEST_CHK(status == FDB_RESULT_SUCCESS);
3054
3055    fdb_doc_create(&rdoc, NULL, 0, NULL, 0, NULL, 0);
3056    status = fdb_iterator_get(iterator, &rdoc);
3057    TEST_CHK(status == FDB_RESULT_SUCCESS);
3058    TEST_CMP(rdoc->key, "doc-029", 8);
3059    fdb_doc_free(rdoc);
3060
3061    fdb_iterator_close(iterator);
3062
3063    // set range to have start key that does not exist
3064    status = fdb_iterator_init(db, &iterator, (void*)"doc-024b", 10,
3065                               doc[30]->key, 10,
3066                      FDB_ITR_NO_DELETES);
3067    TEST_CHK(status == FDB_RESULT_SUCCESS);
3068
3069    status = fdb_iterator_seek_to_min(iterator);
3070    TEST_CHK(status == FDB_RESULT_SUCCESS);
3071
3072    fdb_doc_create(&rdoc, NULL, 0, NULL, 0, NULL, 0);
3073    status = fdb_iterator_get(iterator, &rdoc);
3074    TEST_CHK(status == FDB_RESULT_SUCCESS);
3075    TEST_CMP(rdoc->key, "doc-025", 8);
3076    fdb_doc_free(rdoc);
3077
3078    fdb_iterator_close(iterator);
3079
3080    // close db handle
3081    fdb_kvs_close(db);
3082
3083    // close db file
3084    fdb_close(dbfile);
3085
3086    // free all documents
3087    for (i=0;i<n;++i){
3088        fdb_doc_free(doc[i]);
3089    }
3090
3091    // free all resources
3092    fdb_shutdown();
3093
3094    memleak_end();
3095
3096    TEST_RESULT("reverse seek to max non-existent key test");
3097}
3098
3099void iterator_seek_wal_only_test()
3100{
3101    TEST_INIT();
3102
3103    memleak_start();
3104
3105    int i, r;
3106    int n = 10;
3107    fdb_file_handle *dbfile;
3108    fdb_kvs_handle *db, *kv1;
3109    fdb_doc **doc = alca(fdb_doc*, n);
3110    fdb_doc *rdoc = NULL;
3111    fdb_status status;
3112    fdb_iterator *iterator;
3113
3114    char keybuf[256], metabuf[256], bodybuf[256];
3115
3116    // remove previous iterator_test files
3117    r = system(SHELL_DEL" iterator_test* > errorlog.txt");
3118    (void)r;
3119
3120    fdb_config fconfig = fdb_get_default_config();
3121    fdb_kvs_config kvs_config = fdb_get_default_kvs_config();
3122    fconfig.buffercache_size = 0;
3123    fconfig.wal_threshold = 1024;
3124    fconfig.flags = FDB_OPEN_FLAG_CREATE;
3125    fconfig.compaction_threshold = 0;
3126
3127    // open db
3128    fdb_open(&dbfile, "./iterator_test1", &fconfig);
3129    fdb_kvs_open_default(dbfile, &db, &kvs_config);
3130    status = fdb_set_log_callback(db, logCallbackFunc,
3131                                  (void *) "iterator_seek_wal_only_test");
3132    TEST_CHK(status == FDB_RESULT_SUCCESS);
3133
3134    // Create another KV store..
3135    status = fdb_kvs_open(dbfile, &kv1, "kv1", &kvs_config);
3136    TEST_CHK(status == FDB_RESULT_SUCCESS);
3137
3138    // insert using 'kv1' instance to ensure it does not interfere
3139    for (i=0;i<n;++i){
3140        sprintf(keybuf, "kEy%d", i);
3141        sprintf(metabuf, "mEta%d", i);
3142        sprintf(bodybuf, "bOdy%d", i);
3143        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
3144            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
3145        status = fdb_set(kv1, doc[i]);
3146        TEST_CHK(status == FDB_RESULT_SUCCESS);
3147        fdb_doc_free(doc[i]);
3148    }
3149
3150    // insert all documents into WAL only
3151    for (i=0;i<n;++i){
3152        sprintf(keybuf, "key%d", i);
3153        sprintf(metabuf, "meta%d", i);
3154        sprintf(bodybuf, "body%d", i);
3155        fdb_doc_create(&doc[i], (void*)keybuf, strlen(keybuf),
3156            (void*)metabuf, strlen(metabuf), (void*)bodybuf, strlen(bodybuf));
3157        fdb_set(db, doc[i]);
3158    }
3159    // commit without WAL flush
3160    fdb_commit(dbfile, FDB_COMMIT_NORMAL);
3161
3162    // create an iterator for full range
3163    fdb_iterator_init(db, &iterator, NULL, 0, NULL, 0, FDB_ITR_NONE);
3164
3165    // seek current iterator to inside the WAL's avl tree..
3166    status = fdb_iterator_get(iterator, &rdoc);
3167    TEST_CHK(status == FDB_RESULT_SUCCESS);
3168
3169    TEST_CMP(rdoc->key, doc[0]->key, rdoc->keylen);
3170    TEST_CMP(rdoc->meta, doc[0]->meta, rdoc->metalen);
3171    TEST_CMP(rdoc->body, doc[0]->body, rdoc->bodylen);
3172    fdb_doc_free(rdoc);
3173    rdoc = NULL;
3174
3175    // seek forward to 2nd key ..
3176    i=2;
3177    status = fdb_iterator_seek(iterator, doc[i]->key, strlen(keybuf), 0);
3178    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
3179    status = fdb_iterator_get(iterator, &rdoc);
3180    TEST_CHK(status == FDB_RESULT_SUCCESS);
3181
3182    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
3183    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
3184    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
3185    fdb_doc_free(rdoc);
3186    rdoc = NULL;
3187
3188    // iterator should be able to proceed forward
3189    status = fdb_iterator_next(iterator);
3190    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
3191    status = fdb_iterator_get(iterator, &rdoc);
3192    TEST_CHK(status == FDB_RESULT_SUCCESS);
3193
3194    i++;
3195    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
3196    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
3197    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
3198    fdb_doc_free(rdoc);
3199    rdoc = NULL;
3200
3201    // seek forward to the last key.
3202    status = fdb_iterator_seek(iterator, doc[n-1]->key, strlen(keybuf), 0);
3203    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
3204    status = fdb_iterator_get(iterator, &rdoc);
3205    TEST_CHK(status == FDB_RESULT_SUCCESS);
3206
3207    TEST_CMP(rdoc->key, doc[n-1]->key, rdoc->keylen);
3208    TEST_CMP(rdoc->meta, doc[n-1]->meta, rdoc->metalen);
3209    TEST_CMP(rdoc->body, doc[n-1]->body, rdoc->bodylen);
3210    fdb_doc_free(rdoc);
3211    rdoc = NULL;
3212
3213    // seek backward to start key ..
3214    i = 0;
3215    status = fdb_iterator_seek(iterator, doc[i]->key, strlen(keybuf), 0);
3216    TEST_CHK(status != FDB_RESULT_ITERATOR_FAIL);
3217    status = fdb_iterator_get(iterator, &rdoc);
3218    TEST_CHK(status == FDB_RESULT_SUCCESS);
3219
3220    TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
3221    TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
3222    TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
3223    fdb_doc_free(rdoc);
3224    rdoc = NULL;
3225
3226    // seek forward to key2 ..
3227    status = fdb_iterator_seek(iterator, doc[2]->key, strlen(keybuf), 0);
3228    TEST_CHK(status == FDB_RESULT_SUCCESS);
3229
3230    i=2;
3231    do {
3232        status = fdb_iterator_get(iterator, &rdoc);
3233        TEST_CHK(status == FDB_RESULT_SUCCESS);
3234
3235        TEST_CMP(rdoc->key, doc[i]->key, rdoc->keylen);
3236        TEST_CMP(rdoc->meta, doc[i]->meta, rdoc->metalen);
3237        TEST_CMP(rdoc->body, doc[i]->body, rdoc->bodylen);
3238
3239        fdb_doc_free(rdoc);
3240        rdoc = NULL;
3241        i++;
3242    } while(fdb_iterator_next(iterator) != FDB_RESULT_ITERATOR_FAIL);
3243    TEST_CHK(i==10);
3244
3245    // Seek backward again to a key...
3246    status = fdb_iterator_seek(iterator, doc[3]->key, strlen(keybuf), 0);
3247    TEST_CHK(status == FDB_RESULT_SUCCESS);
3248
3249    i=3;
3250    do {
3251        status = fdb_iterator_get(iterator, &rdoc);
3252        TEST_CHK(status == FDB_RESULT_SUCCESS);
3253
3254        TEST_CMP(rdoc->key, doc[i]->