xref: /6.6.0/couchstore/src/views/view_group.cc (revision f798e0b7)
1/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2
3/**
4 * @copyright 2013 Couchbase, Inc.
5 *
6 * @author Filipe Manana  <filipe@couchbase.com>
7 *
8 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
9 * use this file except in compliance with the License. You may obtain a copy of
10 * the License at
11 *
12 *  http://www.apache.org/licenses/LICENSE-2.0
13 *
14 * Unless required by applicable law or agreed to in writing, software
15 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17 * License for the specific language governing permissions and limitations under
18 * the License.
19 **/
20
21#include "couchstore_config.h"
22#include <errno.h>
23#include <stdlib.h>
24#include <string.h>
25
26#include "view_group.h"
27#include "reducers.h"
28#include "reductions.h"
29#include "values.h"
30#include "purgers.h"
31#include "util.h"
32#include "file_sorter.h"
33#include "spatial.h"
34#include "../arena.h"
35#include "../couch_btree.h"
36#include "../internal.h"
37#include "../util.h"
38#include <platform/cb_malloc.h>
39#include <platform/cbassert.h>
40
41#define VIEW_KV_CHUNK_THRESHOLD (7 * 1024)
42#define VIEW_KP_CHUNK_THRESHOLD (6 * 1024)
43#define MAX_ACTIONS_SIZE        (2 * 1024 * 1024)
44
45static couchstore_error_t read_btree_info(view_group_info_t *info,
46                                          FILE *in_stream,
47                                          FILE *error_stream);
48
49static couchstore_error_t read_spatial_info(view_group_info_t *info,
50                                            FILE *in_stream,
51                                            FILE *error_stream);
52
53static couchstore_error_t build_btree(const char *source_file,
54                                      tree_file *dest_file,
55                                      compare_info *cmp,
56                                      reduce_fn reduce_fun,
57                                      reduce_fn rereduce_fun,
58                                      const char *tmpdir,
59                                      sort_record_fn sort_fun,
60                                      void *reduce_ctx,
61                                      node_pointer **out_root);
62
63static couchstore_error_t build_id_btree(const char *source_file,
64                                         tree_file *dest_file,
65                                         const char *tmpdir,
66                                         node_pointer **out_root);
67
68static couchstore_error_t build_view_btree(const char *source_file,
69                                           const view_btree_info_t *info,
70                                           tree_file *dest_file,
71                                           const char *tmpdir,
72                                           node_pointer **out_root,
73                                           view_error_t *error_info);
74
75static void close_view_group_file(view_group_info_t *info);
76
77static int read_record(FILE *f, arena *a, sized_buf *k, sized_buf *v,
78                                                        uint8_t *op);
79
80extern "C" {
81    int view_btree_cmp(const sized_buf *key1, const sized_buf *key2);
82}
83static couchstore_error_t update_btree(const char *source_file,
84                                       tree_file *dest_file,
85                                       const node_pointer *root,
86                                       size_t batch_size,
87                                       compare_info *cmp,
88                                       reduce_fn reduce_fun,
89                                       reduce_fn rereduce_fun,
90                                       purge_kv_fn purge_kv,
91                                       purge_kp_fn purge_kp,
92                                       view_reducer_ctx_t *red_ctx,
93                                       view_purger_ctx_t *purge_ctx,
94                                       uint64_t *inserted,
95                                       uint64_t *removed,
96                                       node_pointer **out_root);
97
98static couchstore_error_t update_id_btree(const char *source_file,
99                                         tree_file *dest_file,
100                                         const node_pointer *root,
101                                         size_t batch_size,
102                                         view_purger_ctx_t *purge_ctx,
103                                         uint64_t *inserted,
104                                         uint64_t *removed,
105                                         node_pointer **out_root);
106
107static couchstore_error_t update_view_btree(const char *source_file,
108                                            const view_btree_info_t *info,
109                                            tree_file *dest_file,
110                                            const node_pointer *root,
111                                            size_t batch_size,
112                                            view_purger_ctx_t *purge_ctx,
113                                            uint64_t *inserted,
114                                            uint64_t *removed,
115                                            node_pointer **out_root,
116                                            view_error_t *error_info);
117
118static couchstore_error_t compact_view_fetchcb(couchfile_lookup_request *rq,
119                                        const sized_buf *k,
120                                        const sized_buf *v);
121
122static couchstore_error_t compact_btree(tree_file *source,
123                                 tree_file *target,
124                                 const node_pointer *root,
125                                 compare_info *cmp,
126                                 reduce_fn reduce_fun,
127                                 reduce_fn rereduce_fun,
128                                 compact_filter_fn filter_fun,
129                                 view_reducer_ctx_t *red_ctx,
130                                 const bitmap_t *filterbm,
131                                 compactor_stats_t *stats,
132                                 node_pointer **out_root);
133
134static couchstore_error_t compact_id_btree(tree_file *source,
135                                    tree_file *target,
136                                    const node_pointer *root,
137                                    const bitmap_t *filterbm,
138                                    compactor_stats_t *stats,
139                                    node_pointer **out_root);
140
141static couchstore_error_t compact_view_btree(tree_file *source,
142                                      tree_file *target,
143                                      const view_btree_info_t *info,
144                                      const node_pointer *root,
145                                      const bitmap_t *filterbm,
146                                      compactor_stats_t *stats,
147                                      node_pointer **out_root,
148                                      view_error_t *error_info);
149
150/* Some preparations before building the actual spatial index */
151static couchstore_error_t build_view_spatial(const char *source_file,
152                                             const view_spatial_info_t *info,
153                                             tree_file *dest_file,
154                                             const char *tmpdir,
155                                             node_pointer **out_root,
156                                             view_error_t *error_info);
157
158/* Build the actual spatial index */
159static couchstore_error_t build_spatial(const char *source_file,
160                                        tree_file *dest_file,
161                                        compare_info *cmp,
162                                        reduce_fn reduce_fun,
163                                        reduce_fn rereduce_fun,
164                                        const uint16_t dimension,
165                                        const double *mbb,
166                                        const char *tmpdir,
167                                        sort_record_fn sort_fun,
168                                        node_pointer **out_root);
169
170/* Callback for every item that got fetched from the original view */
171static couchstore_error_t compact_spatial_fetchcb(couchfile_lookup_request *rq,
172                                                  const sized_buf *k,
173                                                  const sized_buf *v);
174
175/* Some preparations before compacting the actual spatial index */
176static couchstore_error_t compact_view_spatial(tree_file *source,
177                                               tree_file *target,
178                                               const view_spatial_info_t *info,
179                                               const node_pointer *root,
180                                               const bitmap_t *filterbm,
181                                               compactor_stats_t *stats,
182                                               node_pointer **out_root,
183                                               view_error_t *error_info);
184
185/* Compact the actual spatial index */
186static couchstore_error_t compact_spatial(tree_file *source,
187                                          tree_file *target,
188                                          const node_pointer *root,
189                                          compare_info *cmp,
190                                          reduce_fn reduce_fun,
191                                          reduce_fn rereduce_fun,
192                                          compact_filter_fn filter_fun,
193                                          const bitmap_t *filterbm,
194                                          compactor_stats_t *stats,
195                                          node_pointer **out_root);
196
197view_group_info_t *couchstore_read_view_group_info(FILE *in_stream,
198                                                   FILE *error_stream)
199{
200    view_group_info_t *info;
201    char buf[4096];
202    char *dup;
203    couchstore_error_t ret;
204    uint64_t type;
205
206    info = (view_group_info_t *) cb_calloc(1, sizeof(*info));
207    if (info == NULL) {
208        fprintf(error_stream, "Memory allocation failure\n");
209        goto out_error;
210    }
211
212    type = couchstore_read_int(in_stream, buf, sizeof(buf), &ret);
213    if (ret != COUCHSTORE_SUCCESS) {
214        fprintf(stderr, "Error reading view file type\n");
215        goto out_error;
216    }
217    switch (type) {
218    case VIEW_INDEX_TYPE_MAPREDUCE:
219        info->type = VIEW_INDEX_TYPE_MAPREDUCE;
220        break;
221    case VIEW_INDEX_TYPE_SPATIAL:
222        info->type = VIEW_INDEX_TYPE_SPATIAL;
223        break;
224    default:
225        fprintf(stderr, "Invalid view file type: %" PRIu64 "\n", type);
226        goto out_error;
227    }
228
229    if (couchstore_read_line(in_stream, buf, sizeof(buf)) != buf) {
230        fprintf(stderr, "Error reading source index file path\n");
231        goto out_error;
232    }
233    dup = cb_strdup(buf);
234    if (dup == NULL) {
235        fprintf(error_stream, "Memory allocation failure\n");
236        goto out_error;
237    }
238    info->filepath = (const char *) dup;
239
240    info->header_pos = couchstore_read_int(in_stream, buf,
241                                                      sizeof(buf),
242                                                      &ret);
243    if (ret != COUCHSTORE_SUCCESS) {
244        fprintf(error_stream, "Error reading header position\n");
245        goto out_error;
246    }
247
248    info->num_btrees = couchstore_read_int(in_stream, buf,
249                                                      sizeof(buf),
250                                                      &ret);
251    if (ret != COUCHSTORE_SUCCESS) {
252        fprintf(error_stream, "Error reading number of btrees\n");
253        goto out_error;
254    }
255
256    switch (info->type) {
257    case VIEW_INDEX_TYPE_MAPREDUCE:
258        ret = read_btree_info(info, in_stream, error_stream);
259        break;
260    case VIEW_INDEX_TYPE_SPATIAL:
261        ret = read_spatial_info(info, in_stream, error_stream);
262        break;
263    }
264    if (ret != COUCHSTORE_SUCCESS) {
265        goto out_error;
266    }
267
268    return info;
269
270out_error:
271    couchstore_free_view_group_info(info);
272
273    return NULL;
274}
275
276
277/* Read in the information about the mapreduce view indexes */
278static couchstore_error_t read_btree_info(view_group_info_t *info,
279                                          FILE *in_stream,
280                                          FILE *error_stream)
281{
282    char buf[4096];
283    char *dup;
284    int i, j;
285    int reduce_len;
286    couchstore_error_t ret;
287
288    info->view_infos.btree = (view_btree_info_t *)
289        cb_calloc(info->num_btrees, sizeof(view_btree_info_t));
290    if (info->view_infos.btree == NULL) {
291        fprintf(error_stream, "Memory allocation failure on btree infos\n");
292        info->num_btrees = 0;
293        return COUCHSTORE_ERROR_ALLOC_FAIL;
294    }
295
296    for (i = 0; i < info->num_btrees; ++i) {
297        view_btree_info_t *bti = &info->view_infos.btree[i];
298
299        bti->view_id = i;
300        bti->num_reducers = couchstore_read_int(in_stream,
301                                                buf,
302                                                sizeof(buf),
303                                                &ret);
304
305        if (ret != COUCHSTORE_SUCCESS) {
306            fprintf(error_stream,
307                    "Error reading number of reducers for btree %d\n", i);
308            return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
309        }
310
311        bti->names = (const char **) cb_calloc(bti->num_reducers, sizeof(char *));
312        if (bti->names == NULL) {
313            fprintf(error_stream,
314                    "Memory allocation failure on btree %d reducer names\n",
315                    i);
316            bti->num_reducers = 0;
317            return COUCHSTORE_ERROR_ALLOC_FAIL;
318        }
319
320        bti->reducers = (const char **) cb_calloc(bti->num_reducers, sizeof(char *));
321        if (bti->reducers == NULL) {
322            fprintf(error_stream,
323                    "Memory allocation failure on btree %d reducers\n", i);
324            bti->num_reducers = 0;
325            cb_free(bti->names);
326            return COUCHSTORE_ERROR_ALLOC_FAIL;
327        }
328
329        for (j = 0; j < bti->num_reducers; ++j) {
330            if (couchstore_read_line(in_stream, buf, sizeof(buf)) != buf) {
331                fprintf(error_stream,
332                        "Error reading btree %d view %d name\n", i, j);
333                return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
334            }
335            dup = cb_strdup(buf);
336            if (dup == NULL) {
337                fprintf(error_stream,
338                        "Memory allocation failure on btree %d "
339                        "view %d name\n", i, j);
340                return COUCHSTORE_ERROR_ALLOC_FAIL;
341            }
342            bti->names[j] = (const char *) dup;
343
344            reduce_len = couchstore_read_int(in_stream,
345                                             buf,
346                                             sizeof(buf),
347                                             &ret);
348            if (ret != COUCHSTORE_SUCCESS) {
349                fprintf(error_stream,
350                        "Error reading btree %d view %d "
351                        "reduce function size\n", i, j);
352                return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
353            }
354
355            dup = (char *) cb_malloc(reduce_len + 1);
356            if (dup == NULL) {
357                fprintf(error_stream,
358                        "Memory allocation failure on btree %d "
359                        "view %d reducer\n", i, j);
360                return COUCHSTORE_ERROR_ALLOC_FAIL;
361            }
362
363            if (fread(dup, reduce_len, 1, in_stream) != 1) {
364                fprintf(error_stream,
365                        "Error reading btree %d view %d reducer\n", i, j);
366                cb_free(dup);
367                return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
368            }
369            dup[reduce_len] = '\0';
370            bti->reducers[j] = (const char *) dup;
371        }
372    }
373    return COUCHSTORE_SUCCESS;
374}
375
376
377/* Read in the information about the spatial view indexes */
378static couchstore_error_t read_spatial_info(view_group_info_t *info,
379                                            FILE *in_stream,
380                                            FILE *error_stream)
381{
382    char buf[24];
383    int i;
384    couchstore_error_t ret;
385
386    info->view_infos.spatial = (view_spatial_info_t *)
387        cb_calloc(info->num_btrees, sizeof(view_spatial_info_t));
388    if (info->view_infos.spatial == NULL) {
389        fprintf(error_stream, "Memory allocation failure on spatial infos\n");
390        info->num_btrees = 0;
391        return COUCHSTORE_ERROR_ALLOC_FAIL;
392    }
393
394    for (i = 0; i < info->num_btrees; ++i) {
395        view_spatial_info_t *si = &info->view_infos.spatial[i];
396
397        si->dimension = couchstore_read_int(in_stream, buf, sizeof(buf), &ret);
398
399        if (ret != COUCHSTORE_SUCCESS) {
400            fprintf(error_stream,
401                    "Error reading the dimension of spatial view %d\n", i);
402            return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
403        }
404
405        /* If the dimension is 0, no index exists or hasn't been built yet */
406        if (si->dimension == 0) {
407            continue;
408        }
409
410        si->mbb = (double *) cb_calloc(si->dimension * 2, sizeof(double));
411        if (si->mbb == NULL) {
412            fprintf(error_stream,
413                    "Memory allocation failure on spatial view %d\n", i);
414            si->dimension = 0;
415            return COUCHSTORE_ERROR_ALLOC_FAIL;
416        }
417
418        if (fread(si->mbb, sizeof(double), si->dimension * 2, in_stream) <
419                si->dimension * 2) {
420            fprintf(error_stream, "Error reading mbb of view %d\n", i);
421            return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
422        }
423    }
424    return COUCHSTORE_SUCCESS;
425}
426
427void couchstore_free_view_group_info(view_group_info_t *info)
428{
429    int i, j;
430
431    if (info == NULL)
432        return;
433
434    close_view_group_file(info);
435
436    switch (info->type) {
437    case VIEW_INDEX_TYPE_MAPREDUCE:
438        for (i = 0; i < info->num_btrees; ++i) {
439            view_btree_info_t vi = info->view_infos.btree[i];
440
441            for (j = 0; j < vi.num_reducers; ++j) {
442                cb_free((void *) vi.names[j]);
443                cb_free((void *) vi.reducers[j]);
444            }
445            cb_free(vi.names);
446            cb_free(vi.reducers);
447        }
448        cb_free(info->view_infos.btree);
449        break;
450    case VIEW_INDEX_TYPE_SPATIAL:
451        for (i = 0; i < info->num_btrees; ++i) {
452            cb_free((void *) info->view_infos.spatial[i].mbb);
453        }
454        cb_free(info->view_infos.spatial);
455        break;
456    }
457    cb_free((void *) info->filepath);
458    cb_free(info);
459}
460
461
462static void close_view_group_file(view_group_info_t *info)
463{
464    if (info->file.ops != NULL) {
465        info->file.ops->close(&info->file.lastError, info->file.handle);
466        info->file.ops->destructor(info->file.handle);
467        info->file.ops = NULL;
468        info->file.handle = NULL;
469    }
470    cb_free((void *) info->file.path);
471    info->file.path = NULL;
472}
473
474
475couchstore_error_t open_view_group_file(const char *path,
476                                               couchstore_open_flags open_flags,
477                                               tree_file *file)
478{
479    couchstore_error_t ret;
480    FileOpsInterface* file_ops = couchstore_get_default_file_ops();
481    int flags = 0;
482
483    if ((open_flags & COUCHSTORE_OPEN_FLAG_RDONLY) &&
484        (open_flags & COUCHSTORE_OPEN_FLAG_CREATE)) {
485        return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
486    }
487
488    if (open_flags & COUCHSTORE_OPEN_FLAG_RDONLY) {
489        flags |= O_RDONLY;
490    } else {
491        flags |= O_RDWR;
492    }
493
494    if (open_flags & COUCHSTORE_OPEN_FLAG_CREATE) {
495        flags |= O_CREAT;
496    }
497
498    ret = tree_file_open(file, path, flags, CRC32, file_ops,
499                         tree_file_options() /* Enable buffered IO by default. */);
500
501    return ret;
502}
503
504couchstore_error_t couchstore_build_view_group(view_group_info_t *info,
505                                               const char *id_records_file,
506                                               const char *kv_records_files[],
507                                               const char *dst_file,
508                                               const char *tmpdir,
509                                               uint64_t *header_pos,
510                                               view_error_t *error_info)
511{
512    couchstore_error_t ret;
513    tree_file index_file;
514    index_header_t *header = NULL;
515    node_pointer *id_root = NULL;
516    node_pointer **view_roots = NULL;
517    int i;
518
519    error_info->view_name = NULL;
520    error_info->error_msg = NULL;
521    index_file.handle = NULL;
522    index_file.ops = NULL;
523    index_file.path = NULL;
524
525    view_roots = (node_pointer **) cb_calloc(
526        info->num_btrees, sizeof(node_pointer *));
527    if (view_roots == NULL) {
528        return COUCHSTORE_ERROR_ALLOC_FAIL;
529    }
530
531    ret = open_view_group_file(info->filepath,
532                               COUCHSTORE_OPEN_FLAG_RDONLY,
533                               &info->file);
534    if (ret != COUCHSTORE_SUCCESS) {
535        goto out;
536    }
537
538    ret = read_view_group_header(info, &header);
539    if (ret != COUCHSTORE_SUCCESS) {
540        goto out;
541    }
542    cb_assert(info->num_btrees == header->num_views);
543
544    ret = open_view_group_file(dst_file,
545                               COUCHSTORE_OPEN_FLAG_CREATE,
546                               &index_file);
547    if (ret != COUCHSTORE_SUCCESS) {
548        goto out;
549    }
550
551    ret = build_id_btree(id_records_file, &index_file, tmpdir, &id_root);
552    if (ret != COUCHSTORE_SUCCESS) {
553        goto out;
554    }
555
556    cb_free(header->id_btree_state);
557    header->id_btree_state = id_root;
558    id_root = NULL;
559
560    for (i = 0; i < info->num_btrees; ++i) {
561        switch(info->type) {
562        case VIEW_INDEX_TYPE_MAPREDUCE:
563            ret = build_view_btree(kv_records_files[i],
564                                   &info->view_infos.btree[i],
565                                   &index_file,
566                                   tmpdir,
567                                   &view_roots[i],
568                                   error_info);
569            break;
570        case VIEW_INDEX_TYPE_SPATIAL:
571            ret = build_view_spatial(kv_records_files[i],
572                                     &info->view_infos.spatial[i],
573                                     &index_file,
574                                     tmpdir,
575                                     &view_roots[i],
576                                     error_info);
577            break;
578        }
579        if (ret != COUCHSTORE_SUCCESS) {
580            goto out;
581        }
582
583        cb_free(header->view_states[i]);
584        header->view_states[i] = view_roots[i];
585        view_roots[i] = NULL;
586    }
587
588    ret = write_view_group_header(&index_file, header_pos, header);
589    if (ret != COUCHSTORE_SUCCESS) {
590        goto out;
591    }
592
593    ret = COUCHSTORE_SUCCESS;
594
595out:
596    free_index_header(header);
597    close_view_group_file(info);
598    tree_file_close(&index_file);
599    cb_free(id_root);
600    if (view_roots != NULL) {
601        for (i = 0; i < info->num_btrees; ++i) {
602            cb_free(view_roots[i]);
603        }
604        cb_free(view_roots);
605    }
606
607    return ret;
608}
609
610
611/*
612 * Similar to util.c:read_view_record(), but it uses arena allocator, which is
613 * required for the existing semantics/api of btree bottom-up build in
614 * src/btree_modify.cc.
615 */
616static int read_record(FILE *f, arena *a, sized_buf *k, sized_buf *v,
617                                                        uint8_t *op)
618{
619    uint16_t klen;
620    uint8_t oplen = 0;
621    uint32_t vlen, len;
622
623    if (fread(&len, sizeof(len), 1, f) != 1) {
624        if (feof(f)) {
625            return 0;
626        } else {
627            return COUCHSTORE_ERROR_READ;
628        }
629    }
630
631    /* For incremental update, read optype */
632    if (op != NULL) {
633        if (fread(op, sizeof(uint8_t), 1, f) != 1) {
634            return COUCHSTORE_ERROR_READ;
635        }
636
637        oplen = 1;
638    }
639
640    if (fread(&klen, sizeof(klen), 1, f) != 1) {
641        return COUCHSTORE_ERROR_READ;
642    }
643
644    klen = ntohs(klen);
645    vlen = len - sizeof(klen) - klen - oplen;
646
647    k->size = klen;
648    k->buf = (char *) arena_alloc(a, k->size);
649    if (k->buf == NULL) {
650        return COUCHSTORE_ERROR_ALLOC_FAIL;
651    }
652
653    v->size = vlen;
654    /* Handle zero len vals */
655    if (v->size) {
656        v->buf = (char *) arena_alloc(a, v->size);
657        if (v->buf == NULL) {
658            return COUCHSTORE_ERROR_ALLOC_FAIL;
659        }
660    } else {
661        v->buf = NULL;
662    }
663
664    if (fread(k->buf, k->size, 1, f) != 1) {
665        return FILE_MERGER_ERROR_FILE_READ;
666    }
667
668    if (v->size && fread(v->buf, v->size, 1, f) != 1) {
669        return FILE_MERGER_ERROR_FILE_READ;
670    }
671
672    return len;
673}
674
675
676static int id_btree_cmp(const sized_buf *key1, const sized_buf *key2)
677{
678    return view_id_cmp(key1, key2, NULL);
679}
680
681
682int view_btree_cmp(const sized_buf *key1, const sized_buf *key2)
683{
684    return view_key_cmp(key1, key2, NULL);
685}
686
687
688/*
689 * For initial btree build, feed the btree builder as soon as
690 * sorted records are available.
691 */
692static file_merger_error_t build_btree_record_callback(void *buf, void *ctx)
693{
694    int ret;
695    sized_buf *k, *v;
696    view_file_merge_record_t *rec = (view_file_merge_record_t *) buf;
697    view_file_merge_ctx_t *merge_ctx = (view_file_merge_ctx_t *) ctx;
698    view_btree_builder_ctx_t *build_ctx =
699                    (view_btree_builder_ctx_t *) merge_ctx->user_ctx;
700    sized_buf src_k, src_v;
701
702    src_k.size = rec->ksize;
703    src_k.buf = VIEW_RECORD_KEY(rec);
704    src_v.size = rec->vsize;
705    src_v.buf = VIEW_RECORD_VAL(rec);
706
707    k = arena_copy_buf(build_ctx->transient_arena, &src_k);
708    v = arena_copy_buf(build_ctx->transient_arena, &src_v);
709    ret = mr_push_item(k, v, build_ctx->modify_result);
710
711    if (ret != COUCHSTORE_SUCCESS) {
712        return static_cast<file_merger_error_t>(ret);
713    }
714
715    if (build_ctx->modify_result->count == 0) {
716        arena_free_all(build_ctx->transient_arena);
717    }
718
719    return static_cast<file_merger_error_t>(ret);
720}
721
722
723static couchstore_error_t build_btree(const char *source_file,
724                                      tree_file *dest_file,
725                                      compare_info *cmp,
726                                      reduce_fn reduce_fun,
727                                      reduce_fn rereduce_fun,
728                                      const char *tmpdir,
729                                      sort_record_fn sort_fun,
730                                      void *reduce_ctx,
731                                      node_pointer **out_root)
732{
733    couchstore_error_t ret = COUCHSTORE_SUCCESS;
734    arena *transient_arena = new_arena(0);
735    arena *persistent_arena = new_arena(0);
736    couchfile_modify_result *mr;
737    view_btree_builder_ctx_t build_ctx;
738
739    if (transient_arena == NULL || persistent_arena == NULL) {
740        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
741        goto out;
742    }
743
744    mr = new_btree_modres(persistent_arena,
745                          transient_arena,
746                          dest_file,
747                          cmp,
748                          reduce_fun,
749                          rereduce_fun,
750                          reduce_ctx,
751                          VIEW_KV_CHUNK_THRESHOLD + (VIEW_KV_CHUNK_THRESHOLD / 3),
752                          VIEW_KP_CHUNK_THRESHOLD + (VIEW_KP_CHUNK_THRESHOLD / 3));
753    if (mr == NULL) {
754        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
755        goto out;
756    }
757
758    build_ctx.transient_arena = transient_arena;
759    build_ctx.modify_result = mr;
760
761    ret = (couchstore_error_t) sort_fun(source_file,
762                                        tmpdir,
763                                        build_btree_record_callback, &build_ctx);
764    if (ret != COUCHSTORE_SUCCESS) {
765        goto out;
766    }
767
768    *out_root = complete_new_btree(mr, &ret);
769    if (ret != COUCHSTORE_SUCCESS) {
770        goto out;
771    }
772
773    /* Don't care about success/failure. Erlang side will eventually delete it. */
774    remove(source_file);
775
776out:
777    if (transient_arena != NULL) {
778        delete_arena(transient_arena);
779    }
780    if (persistent_arena != NULL) {
781        delete_arena(persistent_arena);
782    }
783
784    return ret;
785}
786
787
788static couchstore_error_t build_id_btree(const char *source_file,
789                                         tree_file *dest_file,
790                                         const char *tmpdir,
791                                         node_pointer **out_root)
792{
793    couchstore_error_t ret;
794    compare_info cmp;
795
796    cmp.compare = id_btree_cmp;
797
798    ret = build_btree(source_file,
799                      dest_file,
800                      &cmp,
801                      view_id_btree_reduce,
802                      view_id_btree_rereduce,
803                      tmpdir,
804                      sort_view_ids_file,
805                      NULL,
806                      out_root);
807
808    return ret;
809}
810
811
812static couchstore_error_t build_view_btree(const char *source_file,
813                                           const view_btree_info_t *info,
814                                           tree_file *dest_file,
815                                           const char *tmpdir,
816                                           node_pointer **out_root,
817                                           view_error_t *error_info)
818{
819    couchstore_error_t ret = COUCHSTORE_SUCCESS;
820    compare_info cmp;
821    view_reducer_ctx_t *red_ctx = NULL;
822    char *error_msg = NULL;
823
824    cmp.compare = view_btree_cmp;
825    red_ctx = make_view_reducer_ctx(info->reducers,
826                                    info->num_reducers,
827                                    &error_msg);
828    if (red_ctx == NULL) {
829        set_error_info(info, (const char *) error_msg, ret, error_info);
830        cb_free(error_msg);
831        return COUCHSTORE_ERROR_REDUCER_FAILURE;
832    }
833
834    ret = build_btree(source_file,
835                      dest_file,
836                      &cmp,
837                      view_btree_reduce,
838                      view_btree_rereduce,
839                      tmpdir,
840                      sort_view_kvs_file,
841                      red_ctx,
842                      out_root);
843
844    if (ret != COUCHSTORE_SUCCESS) {
845        set_error_info(info, red_ctx->error, ret, error_info);
846    }
847
848    free_view_reducer_ctx(red_ctx);
849
850    return ret;
851}
852
853
854couchstore_error_t read_view_group_header(view_group_info_t *info,
855                                          index_header_t **header)
856{
857    couchstore_error_t ret;
858    cs_off_t pos = info->header_pos;
859    tree_file *file = &info->file;
860    char *header_buf = NULL;
861    int header_len;
862    char buf;
863
864    if (file->handle == NULL) {
865        return COUCHSTORE_ERROR_FILE_CLOSED;
866    }
867
868    if (info->file.ops->pread(NULL, file->handle, &buf, 1, pos) != 1) {
869        return COUCHSTORE_ERROR_READ;
870    }
871    if (buf == 0) {
872        return COUCHSTORE_ERROR_NO_HEADER;
873    } else if (buf != 1) {
874        return COUCHSTORE_ERROR_CORRUPT;
875    }
876
877    header_len = pread_header(file, pos, &header_buf, MAX_VIEW_HEADER_SIZE);
878    if (header_len < 0) {
879        return (couchstore_error_t) header_len;
880    }
881
882    ret = decode_index_header(header_buf, (size_t) header_len, header);
883    cb_free(header_buf);
884
885    return ret;
886}
887
888couchstore_error_t write_view_group_header(tree_file *file,
889                                           uint64_t *pos,
890                                           const index_header_t *header)
891{
892    couchstore_error_t ret;
893    sized_buf buf = { NULL, 0 };
894    cs_off_t p;
895
896    if (file->handle == NULL) {
897        return COUCHSTORE_ERROR_FILE_CLOSED;
898    }
899
900    ret = encode_index_header(header, &buf.buf, &buf.size);
901    if (ret != COUCHSTORE_SUCCESS) {
902        return ret;
903    }
904
905    ret = write_header(file, &buf, &p);
906    if (ret != COUCHSTORE_SUCCESS) {
907        goto out;
908    }
909
910    cb_assert(p >= 0);
911    *pos = (uint64_t) p;
912
913out:
914    cb_free(buf.buf);
915
916    return ret;
917}
918
919
920static couchstore_error_t view_id_bitmask(const node_pointer *root, bitmap_t *bm)
921{
922    view_id_btree_reduction_t *r = NULL;
923    couchstore_error_t errcode;
924    if (root == NULL) {
925        return COUCHSTORE_SUCCESS;
926    }
927
928    errcode = decode_view_id_btree_reduction(root->reduce_value.buf, &r);
929    if (errcode != COUCHSTORE_SUCCESS) {
930        goto cleanup;
931    }
932
933    union_bitmaps(bm, &r->partitions_bitmap);
934
935cleanup:
936    free_view_id_btree_reduction(r);
937    return errcode;
938}
939
940
941static couchstore_error_t view_bitmask(const node_pointer *root, bitmap_t *bm)
942{
943    if (root == NULL) {
944        return COUCHSTORE_SUCCESS;
945    }
946
947    bitmap_t partitions_bitmap;
948    decode_view_btree_reduction_partitions_bitmap(
949            root->reduce_value.buf, root->reduce_value.size, partitions_bitmap);
950
951    union_bitmaps(bm, &partitions_bitmap);
952
953    return COUCHSTORE_SUCCESS;
954}
955
956
957static couchstore_error_t cleanup_btree(tree_file *file,
958                                        node_pointer *root,
959                                        compare_info *cmp,
960                                        reduce_fn reduce,
961                                        reduce_fn rereduce,
962                                        purge_kv_fn purge_kv,
963                                        purge_kp_fn purge_kp,
964                                        view_purger_ctx_t *purge_ctx,
965                                        view_reducer_ctx_t *red_ctx,
966                                        node_pointer **out_root)
967{
968    couchstore_error_t errcode;
969    couchfile_modify_request rq;
970
971    rq.cmp = *cmp;
972    rq.file = file;
973    rq.actions = NULL;
974    rq.num_actions = 0;
975    rq.reduce = reduce;
976    rq.rereduce = rereduce;
977    rq.compacting = 0;
978    rq.kv_chunk_threshold = VIEW_KV_CHUNK_THRESHOLD;
979    rq.kp_chunk_threshold = VIEW_KP_CHUNK_THRESHOLD;
980    rq.purge_kp = purge_kp;
981    rq.purge_kv = purge_kv;
982    rq.enable_purging = 1;
983    rq.guided_purge_ctx = purge_ctx;
984    rq.user_reduce_ctx = red_ctx;
985
986    *out_root = guided_purge_btree(&rq, root, &errcode);
987
988    return errcode;
989}
990
991static couchstore_error_t cleanup_id_btree(tree_file *file,
992                                           node_pointer *root,
993                                           node_pointer **out_root,
994                                           view_purger_ctx_t *purge_ctx,
995                                           view_error_t *error_info)
996{
997    couchstore_error_t ret;
998    compare_info cmp;
999
1000    cmp.compare = id_btree_cmp;
1001
1002    ret = cleanup_btree(file,
1003                        root,
1004                        &cmp,
1005                        view_id_btree_reduce,
1006                        view_id_btree_rereduce,
1007                        view_id_btree_purge_kv,
1008                        view_id_btree_purge_kp,
1009                        purge_ctx,
1010                        NULL,
1011                        out_root);
1012
1013    return ret;
1014}
1015
1016
1017static couchstore_error_t cleanup_view_btree(tree_file *file,
1018                                             node_pointer *root,
1019                                             const view_btree_info_t *info,
1020                                             node_pointer **out_root,
1021                                             view_purger_ctx_t *purge_ctx,
1022                                             view_error_t *error_info)
1023{
1024    couchstore_error_t ret = COUCHSTORE_SUCCESS;
1025    compare_info cmp;
1026    view_reducer_ctx_t *red_ctx = NULL;
1027    char *error_msg = NULL;
1028
1029    cmp.compare = view_btree_cmp;
1030    red_ctx = make_view_reducer_ctx(info->reducers,
1031                                    info->num_reducers,
1032                                    &error_msg);
1033    if (red_ctx == NULL) {
1034        set_error_info(info, (const char *) error_msg, ret, error_info);
1035        cb_free(error_msg);
1036        return COUCHSTORE_ERROR_REDUCER_FAILURE;
1037    }
1038
1039    ret = cleanup_btree(file,
1040                        root,
1041                        &cmp,
1042                        view_btree_reduce,
1043                        view_btree_rereduce,
1044                        view_btree_purge_kv,
1045                        view_btree_purge_kp,
1046                        purge_ctx,
1047                        red_ctx,
1048                        out_root);
1049
1050    if (ret != COUCHSTORE_SUCCESS) {
1051        const char *error_msg = NULL;
1052        if (red_ctx->error != NULL) {
1053            error_msg = red_ctx->error;
1054        }
1055        set_error_info(info, (const char *) error_msg, ret, error_info);
1056    }
1057
1058    free_view_reducer_ctx(red_ctx);
1059
1060    return ret;
1061}
1062
1063couchstore_error_t couchstore_cleanup_view_group(view_group_info_t *info,
1064                                                 uint64_t *header_pos,
1065                                                 uint64_t *purge_count,
1066                                                 view_error_t *error_info)
1067{
1068    couchstore_error_t ret;
1069    tree_file index_file;
1070    index_header_t *header = NULL;
1071    node_pointer *id_root = NULL;
1072    node_pointer **view_roots = NULL;
1073    view_purger_ctx_t purge_ctx;
1074    bitmap_t bm_cleanup;
1075    int i;
1076
1077    memset(&bm_cleanup, 0, sizeof(bm_cleanup));
1078    error_info->view_name = NULL;
1079    error_info->error_msg = NULL;
1080    index_file.handle = NULL;
1081    index_file.ops = NULL;
1082    index_file.path = NULL;
1083
1084    view_roots = (node_pointer **) cb_calloc(
1085        info->num_btrees, sizeof(node_pointer *));
1086    if (view_roots == NULL) {
1087        return COUCHSTORE_ERROR_ALLOC_FAIL;
1088    }
1089
1090    /* Read info from current index viewgroup file */
1091    ret = open_view_group_file(info->filepath,
1092                               COUCHSTORE_OPEN_FLAG_RDONLY,
1093                               &info->file);
1094    if (ret != COUCHSTORE_SUCCESS) {
1095        goto cleanup;
1096    }
1097
1098    ret = read_view_group_header(info, &header);
1099    if (ret != COUCHSTORE_SUCCESS) {
1100        goto cleanup;
1101    }
1102    cb_assert(info->num_btrees == header->num_views);
1103
1104    /* Setup purger context */
1105    purge_ctx.count = 0;
1106    purge_ctx.cbitmask = header->cleanup_bitmask;
1107
1108    ret = open_view_group_file(info->filepath,
1109                               0,
1110                               &index_file);
1111    if (ret != COUCHSTORE_SUCCESS) {
1112        goto cleanup;
1113    }
1114
1115    index_file.pos = index_file.ops->goto_eof(&index_file.lastError,
1116                                                         index_file.handle);
1117
1118    /* Cleanup id_bree */
1119    ret = cleanup_id_btree(&index_file, header->id_btree_state, &id_root,
1120                                                                &purge_ctx,
1121                                                                error_info);
1122    if (ret != COUCHSTORE_SUCCESS) {
1123        goto cleanup;
1124    }
1125
1126    if (header->id_btree_state != id_root) {
1127        cb_free(header->id_btree_state);
1128    }
1129
1130    header->id_btree_state = id_root;
1131    view_id_bitmask(id_root, &bm_cleanup);
1132    id_root = NULL;
1133
1134    /* Cleanup all view btrees */
1135    for (i = 0; i < info->num_btrees; ++i) {
1136        ret = cleanup_view_btree(&index_file,
1137                                 (node_pointer *) header->view_states[i],
1138                                 &info->view_infos.btree[i],
1139                                 &view_roots[i],
1140                                 &purge_ctx,
1141                                 error_info);
1142
1143        if (ret != COUCHSTORE_SUCCESS) {
1144            goto cleanup;
1145        }
1146
1147        if (header->view_states[i] != view_roots[i]) {
1148            cb_free(header->view_states[i]);
1149        }
1150
1151        header->view_states[i] = view_roots[i];
1152        view_bitmask(view_roots[i], &bm_cleanup);
1153        view_roots[i] = NULL;
1154    }
1155
1156    /* Set resulting cleanup bitmask */
1157    /* TODO: This code can be removed, if we do not plan for cleanup STOP command */
1158    intersect_bitmaps(&bm_cleanup, &purge_ctx.cbitmask);
1159    header->cleanup_bitmask = bm_cleanup;
1160
1161    /* Update header with new btree infos */
1162    ret = write_view_group_header(&index_file, header_pos, header);
1163    if (ret != COUCHSTORE_SUCCESS) {
1164        goto cleanup;
1165    }
1166
1167    *purge_count = purge_ctx.count;
1168    ret = COUCHSTORE_SUCCESS;
1169
1170cleanup:
1171    free_index_header(header);
1172    close_view_group_file(info);
1173    tree_file_close(&index_file);
1174    cb_free(id_root);
1175    if (view_roots != NULL) {
1176        for (i = 0; i < info->num_btrees; ++i) {
1177            cb_free(view_roots[i]);
1178        }
1179        cb_free(view_roots);
1180    }
1181
1182    return ret;
1183}
1184
1185static couchstore_error_t update_btree(const char *source_file,
1186                                       tree_file *dest_file,
1187                                       const node_pointer *root,
1188                                       size_t batch_size,
1189                                       compare_info *cmp,
1190                                       reduce_fn reduce_fun,
1191                                       reduce_fn rereduce_fun,
1192                                       purge_kv_fn purge_kv,
1193                                       purge_kp_fn purge_kp,
1194                                       view_reducer_ctx_t *red_ctx,
1195                                       view_purger_ctx_t *purge_ctx,
1196                                       uint64_t *inserted,
1197                                       uint64_t *removed,
1198                                       node_pointer **out_root)
1199{
1200    couchstore_error_t ret = COUCHSTORE_SUCCESS;
1201    couchfile_modify_request rq;
1202    node_pointer *newroot = (node_pointer *) root;
1203    arena *transient_arena = new_arena(0);
1204    FILE *f = NULL;
1205    couchfile_modify_action *actions = NULL;
1206    sized_buf *keybufs = NULL, *valbufs = NULL;
1207    size_t bufsize = 0;
1208    int last_record = 0;
1209    bitmap_t empty_bm;
1210    int max_actions = MAX_ACTIONS_SIZE /
1211                (sizeof(couchfile_modify_action) + 2 * sizeof(sized_buf));
1212
1213    memset(&empty_bm, 0, sizeof(empty_bm));
1214
1215    if (transient_arena == NULL) {
1216        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1217        goto cleanup;
1218    }
1219
1220
1221    actions = (couchfile_modify_action *) cb_calloc(
1222                                            max_actions,
1223                                            sizeof(couchfile_modify_action));
1224    if (actions == NULL) {
1225        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1226        goto cleanup;
1227    }
1228
1229    keybufs = (sized_buf *) cb_calloc(max_actions, sizeof(sized_buf));
1230    if (keybufs == NULL) {
1231        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1232        goto cleanup;
1233    }
1234
1235    valbufs = (sized_buf *) cb_calloc(max_actions, sizeof(sized_buf));
1236    if (valbufs == NULL) {
1237        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1238        goto cleanup;
1239    }
1240
1241    rq.cmp = *cmp;
1242    rq.file = dest_file;
1243    rq.actions = actions;
1244    rq.num_actions = 0;
1245    rq.reduce = reduce_fun;
1246    rq.rereduce = rereduce_fun;
1247    rq.compacting = 0;
1248    rq.kv_chunk_threshold = VIEW_KV_CHUNK_THRESHOLD;
1249    rq.kp_chunk_threshold = VIEW_KP_CHUNK_THRESHOLD;
1250    rq.purge_kp = purge_kp;
1251    rq.purge_kv = purge_kv;
1252    rq.guided_purge_ctx = purge_ctx;
1253    rq.enable_purging = 1;
1254    rq.user_reduce_ctx = red_ctx;
1255
1256    /* If cleanup bitmask is empty, no need to try purging */
1257    if (is_equal_bitmap(&empty_bm, &purge_ctx->cbitmask)) {
1258        rq.enable_purging = 0;
1259    }
1260
1261    f = fopen(source_file, "rb");
1262    if (f == NULL) {
1263        fprintf(stderr, "Error while opening file: source file %s, errorcode %s\n",
1264                source_file, strerror(errno));
1265        ret = COUCHSTORE_ERROR_OPEN_FILE;
1266        goto cleanup;
1267    }
1268
1269    while (!last_record) {
1270        int read_ret;
1271        uint8_t op;
1272
1273        read_ret = read_record(f, transient_arena, &keybufs[rq.num_actions],
1274                                                   &valbufs[rq.num_actions],
1275                                                   &op);
1276        if (read_ret == 0) {
1277            last_record = 1;
1278            goto flush;
1279        } else if (read_ret < 0) {
1280            ret = (couchstore_error_t) read_ret;
1281            goto cleanup;
1282        }
1283
1284        /* Add action */
1285        actions[rq.num_actions].setType(op);
1286        actions[rq.num_actions].setKey(&keybufs[rq.num_actions]);
1287        actions[rq.num_actions].data = &valbufs[rq.num_actions];
1288
1289        if (inserted && op == ACTION_INSERT) {
1290            (*inserted)++;
1291        } else if (removed && op == ACTION_REMOVE) {
1292            (*removed)++;
1293        }
1294
1295        bufsize += keybufs[rq.num_actions].size +
1296                   valbufs[rq.num_actions].size +
1297                   sizeof(uint8_t);
1298        rq.num_actions++;
1299
1300flush:
1301        if (rq.num_actions && (last_record || bufsize > batch_size ||
1302                               rq.num_actions == max_actions)) {
1303            newroot = modify_btree(&rq, newroot, &ret);
1304            if (ret != COUCHSTORE_SUCCESS) {
1305                goto cleanup;
1306            }
1307
1308            rq.num_actions = 0;
1309            bufsize = 0;
1310            arena_free_all(transient_arena);
1311        }
1312    }
1313
1314    *out_root = newroot;
1315
1316cleanup:
1317    cb_free(actions);
1318    cb_free(keybufs);
1319    cb_free(valbufs);
1320
1321    if (f != NULL) {
1322        fclose(f);
1323    }
1324
1325    if (transient_arena != NULL) {
1326        delete_arena(transient_arena);
1327    }
1328
1329    return ret;
1330}
1331
1332static couchstore_error_t update_id_btree(const char *source_file,
1333                                         tree_file *dest_file,
1334                                         const node_pointer *root,
1335                                         size_t batch_size,
1336                                         view_purger_ctx_t *purge_ctx,
1337                                         uint64_t *inserted,
1338                                         uint64_t *removed,
1339                                         node_pointer **out_root)
1340{
1341    couchstore_error_t ret;
1342    compare_info cmp;
1343
1344    cmp.compare = id_btree_cmp;
1345
1346    ret = update_btree(source_file,
1347                      dest_file,
1348                      root,
1349                      batch_size,
1350                      &cmp,
1351                      view_id_btree_reduce,
1352                      view_id_btree_rereduce,
1353                      view_id_btree_purge_kv,
1354                      view_id_btree_purge_kp,
1355                      NULL,
1356                      purge_ctx,
1357                      inserted,
1358                      removed,
1359                      out_root);
1360
1361    return ret;
1362}
1363
1364
1365static couchstore_error_t update_view_btree(const char *source_file,
1366                                            const view_btree_info_t *info,
1367                                            tree_file *dest_file,
1368                                            const node_pointer *root,
1369                                            size_t batch_size,
1370                                            view_purger_ctx_t *purge_ctx,
1371                                            uint64_t *inserted,
1372                                            uint64_t *removed,
1373                                            node_pointer **out_root,
1374                                            view_error_t *error_info)
1375{
1376    couchstore_error_t ret = COUCHSTORE_SUCCESS;
1377    compare_info cmp;
1378    view_reducer_ctx_t *red_ctx = NULL;
1379    char *error_msg = NULL;
1380
1381    cmp.compare = view_btree_cmp;
1382    red_ctx = make_view_reducer_ctx(info->reducers,
1383                                    info->num_reducers,
1384                                    &error_msg);
1385    if (red_ctx == NULL) {
1386        set_error_info(info, (const char *) error_msg, ret, error_info);
1387        cb_free(error_msg);
1388        return COUCHSTORE_ERROR_REDUCER_FAILURE;
1389    }
1390
1391    ret = update_btree(source_file,
1392                      dest_file,
1393                      root,
1394                      batch_size,
1395                      &cmp,
1396                      view_btree_reduce,
1397                      view_btree_rereduce,
1398                      view_btree_purge_kv,
1399                      view_btree_purge_kp,
1400                      red_ctx,
1401                      purge_ctx,
1402                      inserted,
1403                      removed,
1404                      out_root);
1405
1406    if (ret != COUCHSTORE_SUCCESS) {
1407        set_error_info(info, red_ctx->error, ret, error_info);
1408    }
1409
1410    free_view_reducer_ctx(red_ctx);
1411
1412    return ret;
1413}
1414
1415couchstore_error_t couchstore_update_view_group(view_group_info_t *info,
1416                                               const char *id_records_file,
1417                                               const char *kv_records_files[],
1418                                               size_t batch_size,
1419                                               const sized_buf *header_buf,
1420                                               int is_sorted,
1421                                               const char *tmp_dir,
1422                                               view_group_update_stats_t *stats,
1423                                               sized_buf *header_outbuf,
1424                                               view_error_t *error_info)
1425{
1426    couchstore_error_t ret;
1427    tree_file index_file = {0, NULL, NULL, NULL, {-1}, CRC32};
1428    index_header_t *header = NULL;
1429    node_pointer *id_root = NULL;
1430    node_pointer **view_roots = NULL;
1431    view_purger_ctx_t purge_ctx;
1432    bitmap_t bm_cleanup;
1433    int i;
1434
1435    ret = decode_index_header(header_buf->buf, header_buf->size, &header);
1436    if (ret < 0) {
1437        goto cleanup;
1438    }
1439
1440    memset(&bm_cleanup, 0, sizeof(bm_cleanup));
1441
1442    error_info->view_name = NULL;
1443    error_info->error_msg = NULL;
1444
1445    view_roots = (node_pointer **) cb_calloc(info->num_btrees,
1446                                          sizeof(node_pointer *));
1447    if (view_roots == NULL) {
1448        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1449        goto cleanup;
1450    }
1451
1452    /* Spatial views use the native updater only for sorting the ID b-tree.
1453     * Before starting it, the references to the spatial view index files
1454     * get removed, hence the number of trees in `info` don't match the number
1455     * in the header */
1456    if (info->type != VIEW_INDEX_TYPE_SPATIAL) {
1457        cb_assert(info->num_btrees == header->num_views);
1458    }
1459
1460    /* Setup purger context */
1461    purge_ctx.count = 0;
1462    purge_ctx.cbitmask = header->cleanup_bitmask;
1463
1464    ret = open_view_group_file(info->filepath,
1465                               0,
1466                               &index_file);
1467    if (ret != COUCHSTORE_SUCCESS) {
1468        goto cleanup;
1469    }
1470
1471    /* Move pos to end of file */
1472    index_file.pos = index_file.ops->goto_eof(&index_file.lastError,
1473                                              index_file.handle);
1474
1475    if (!is_sorted) {
1476        ret = (couchstore_error_t) sort_view_ids_ops_file(id_records_file, tmp_dir);
1477        if (ret != COUCHSTORE_SUCCESS) {
1478            char error_msg[1024];
1479            int nw = snprintf(error_msg, sizeof(error_msg),
1480                              "Error sorting records file: %s",
1481                              id_records_file);
1482            if (nw > 0 && size_t(nw) < sizeof(error_msg)) {
1483                error_info->error_msg = cb_strdup(error_msg);
1484            } else {
1485                error_info->error_msg = cb_strdup("Error sorting records file");
1486            }
1487            error_info->idx_type = "MAPREDUCE";
1488            error_info->view_name = (const char *) cb_strdup("id_btree");
1489            goto cleanup;
1490        }
1491    }
1492
1493    ret = update_id_btree(id_records_file, &index_file,
1494                                           header->id_btree_state,
1495                                           batch_size,
1496                                           &purge_ctx,
1497                                           &stats->ids_inserted,
1498                                           &stats->ids_removed,
1499                                           &id_root);
1500    if (ret != COUCHSTORE_SUCCESS) {
1501        goto cleanup;
1502    }
1503
1504
1505    if (header->id_btree_state != id_root) {
1506        cb_free(header->id_btree_state);
1507    }
1508
1509    header->id_btree_state = id_root;
1510    view_id_bitmask(id_root, &bm_cleanup);
1511    id_root = NULL;
1512
1513    for (i = 0; i < info->num_btrees; ++i) {
1514        if (!is_sorted) {
1515            ret = (couchstore_error_t) sort_view_kvs_ops_file(kv_records_files[i], tmp_dir);
1516            if (ret != COUCHSTORE_SUCCESS) {
1517                const char* errmsg = "Error sorting records file";
1518                char error_msg[1024];
1519                int nw = snprintf(error_msg, sizeof(error_msg),
1520                                  "Error sorting records file: %s",
1521                                  kv_records_files[i]);
1522
1523                if (nw > 0 && size_t(nw) < sizeof(error_msg)) {
1524                    errmsg = error_msg;
1525                }
1526                set_error_info(&info->view_infos.btree[i], errmsg, ret,
1527                               error_info);
1528                goto cleanup;
1529            }
1530        }
1531
1532        ret = update_view_btree(kv_records_files[i],
1533                                &info->view_infos.btree[i],
1534                                &index_file,
1535                                header->view_states[i],
1536                                batch_size,
1537                                &purge_ctx,
1538                                &stats->kvs_inserted,
1539                                &stats->kvs_removed,
1540                                &view_roots[i],
1541                                error_info);
1542
1543        if (ret != COUCHSTORE_SUCCESS) {
1544            goto cleanup;
1545        }
1546
1547        if (header->view_states[i] != view_roots[i]) {
1548            cb_free(header->view_states[i]);
1549        }
1550
1551        header->view_states[i] = view_roots[i];
1552        view_bitmask(view_roots[i], &bm_cleanup);
1553        view_roots[i] = NULL;
1554    }
1555
1556    /* Set resulting cleanup bitmask */
1557    intersect_bitmaps(&bm_cleanup, &purge_ctx.cbitmask);
1558    header->cleanup_bitmask = bm_cleanup;
1559    stats->purged = purge_ctx.count;
1560
1561    ret = encode_index_header(header, &header_outbuf->buf, &header_outbuf->size);
1562    if (ret != COUCHSTORE_SUCCESS) {
1563        goto cleanup;
1564    }
1565
1566    ret = COUCHSTORE_SUCCESS;
1567
1568cleanup:
1569    free_index_header(header);
1570    close_view_group_file(info);
1571    tree_file_close(&index_file);
1572    cb_free(id_root);
1573    if (view_roots != NULL) {
1574        for (i = 0; i < info->num_btrees; ++i) {
1575            cb_free(view_roots[i]);
1576        }
1577        cb_free(view_roots);
1578    }
1579
1580    return ret;
1581}
1582
1583/* Add the kv pair to modify result */
1584static couchstore_error_t compact_view_fetchcb(couchfile_lookup_request *rq,
1585                                        const sized_buf *k,
1586                                        const sized_buf *v)
1587{
1588    int ret;
1589    sized_buf *k_c, *v_c;
1590    view_compact_ctx_t *ctx = (view_compact_ctx_t *) rq->callback_ctx;
1591    compactor_stats_t *stats = ctx->stats;
1592
1593    if (k == NULL || v == NULL) {
1594        return COUCHSTORE_ERROR_READ;
1595    }
1596
1597    if (ctx->filter_fun) {
1598        ret = ctx->filter_fun(k, v, ctx->filterbm);
1599        if (ret < 0) {
1600            return (couchstore_error_t) ret;
1601        } else if (ret) {
1602            return COUCHSTORE_SUCCESS;
1603        }
1604    }
1605
1606    k_c = arena_copy_buf(ctx->transient_arena, k);
1607    v_c = arena_copy_buf(ctx->transient_arena, v);
1608    ret = mr_push_item(k_c, v_c, ctx->mr);
1609    if (ret != COUCHSTORE_SUCCESS) {
1610        return static_cast<couchstore_error_t>(ret);
1611    }
1612
1613    if (stats) {
1614        stats->inserted++;
1615        if (stats->update_fun) {
1616            stats->update_fun(stats->freq, stats->inserted);
1617        }
1618    }
1619
1620    if (ctx->mr->count == 0) {
1621        arena_free_all(ctx->transient_arena);
1622    }
1623
1624    return (couchstore_error_t) ret;
1625}
1626
1627static couchstore_error_t compact_btree(tree_file *source,
1628                                 tree_file *target,
1629                                 const node_pointer *root,
1630                                 compare_info *cmp,
1631                                 reduce_fn reduce_fun,
1632                                 reduce_fn rereduce_fun,
1633                                 compact_filter_fn filter_fun,
1634                                 view_reducer_ctx_t *red_ctx,
1635                                 const bitmap_t *filterbm,
1636                                 compactor_stats_t *stats,
1637                                 node_pointer **out_root)
1638{
1639    couchstore_error_t ret = COUCHSTORE_SUCCESS;
1640    arena *transient_arena;
1641    arena *persistent_arena;
1642    couchfile_modify_result *modify_result;
1643    couchfile_lookup_request lookup_rq;
1644    view_compact_ctx_t compact_ctx;
1645    sized_buf nullkey = {NULL, 0};
1646    sized_buf *lowkeys = &nullkey;
1647
1648    if (!root) {
1649        return COUCHSTORE_SUCCESS;
1650    }
1651
1652    transient_arena = new_arena(0);
1653    persistent_arena = new_arena(0);
1654
1655    if (transient_arena == NULL || persistent_arena == NULL) {
1656        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1657        goto cleanup;
1658    }
1659
1660    /* Create new btree on new file */
1661    modify_result = new_btree_modres(persistent_arena,
1662                          transient_arena,
1663                          target,
1664                          cmp,
1665                          reduce_fun,
1666                          rereduce_fun,
1667                          red_ctx,
1668                          VIEW_KV_CHUNK_THRESHOLD + (VIEW_KV_CHUNK_THRESHOLD / 3),
1669                          VIEW_KP_CHUNK_THRESHOLD + (VIEW_KP_CHUNK_THRESHOLD / 3));
1670    if (modify_result == NULL) {
1671        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1672        goto cleanup;
1673    }
1674
1675    compact_ctx.filter_fun = NULL;
1676    compact_ctx.mr = modify_result;
1677    compact_ctx.transient_arena = transient_arena;
1678    compact_ctx.stats = stats;
1679
1680    if (filterbm) {
1681        compact_ctx.filterbm = filterbm;
1682        compact_ctx.filter_fun = filter_fun;
1683    }
1684
1685    lookup_rq.cmp.compare = cmp->compare;
1686    lookup_rq.file = source;
1687    lookup_rq.num_keys = 1;
1688    lookup_rq.keys = &lowkeys;
1689    lookup_rq.callback_ctx = &compact_ctx;
1690    lookup_rq.fetch_callback = compact_view_fetchcb;
1691    lookup_rq.node_callback = NULL;
1692    lookup_rq.fold = 1;
1693
1694    ret = btree_lookup(&lookup_rq, root->pointer);
1695    if (ret != COUCHSTORE_SUCCESS) {
1696        goto cleanup;
1697    }
1698
1699    *out_root = complete_new_btree(modify_result, &ret);
1700
1701cleanup:
1702    if (transient_arena != NULL) {
1703        delete_arena(transient_arena);
1704    }
1705
1706    if (persistent_arena != NULL) {
1707        delete_arena(persistent_arena);
1708    }
1709
1710    return ret;
1711}
1712
1713static couchstore_error_t compact_id_btree(tree_file *source,
1714                                    tree_file *target,
1715                                    const node_pointer *root,
1716                                    const bitmap_t *filterbm,
1717                                    compactor_stats_t *stats,
1718                                    node_pointer **out_root)
1719{
1720    couchstore_error_t ret;
1721    compare_info cmp;
1722
1723    cmp.compare = id_btree_cmp;
1724
1725    ret = compact_btree(source,
1726                        target,
1727                        root,
1728                        &cmp,
1729                        view_id_btree_reduce,
1730                        view_id_btree_rereduce,
1731                        view_id_btree_filter,
1732                        NULL,
1733                        filterbm,
1734                        stats,
1735                        out_root);
1736
1737    return ret;
1738}
1739
1740static couchstore_error_t compact_view_btree(tree_file *source,
1741                                      tree_file *target,
1742                                      const view_btree_info_t *info,
1743                                      const node_pointer *root,
1744                                      const bitmap_t *filterbm,
1745                                      compactor_stats_t *stats,
1746                                      node_pointer **out_root,
1747                                      view_error_t *error_info)
1748{
1749    couchstore_error_t ret = COUCHSTORE_SUCCESS;
1750    compare_info cmp;
1751    view_reducer_ctx_t *red_ctx = NULL;
1752    char *error_msg = NULL;
1753
1754    cmp.compare = view_btree_cmp;
1755    red_ctx = make_view_reducer_ctx(info->reducers,
1756                                    info->num_reducers,
1757                                    &error_msg);
1758    if (red_ctx == NULL) {
1759        set_error_info(info, (const char *) error_msg, ret, error_info);
1760        cb_free(error_msg);
1761        return COUCHSTORE_ERROR_REDUCER_FAILURE;
1762    }
1763
1764    ret = compact_btree(source,
1765                        target,
1766                        root,
1767                        &cmp,
1768                        view_btree_reduce,
1769                        view_btree_rereduce,
1770                        view_btree_filter,
1771                        red_ctx,
1772                        filterbm,
1773                        stats,
1774                        out_root);
1775
1776    if (ret != COUCHSTORE_SUCCESS) {
1777        set_error_info(info, red_ctx->error, ret, error_info);
1778    }
1779
1780    free_view_reducer_ctx(red_ctx);
1781
1782    return ret;
1783}
1784
1785couchstore_error_t couchstore_compact_view_group(view_group_info_t *info,
1786                                                 const char *target_file,
1787                                                 const sized_buf *header_buf,
1788                                                 compactor_stats_t *stats,
1789                                                 sized_buf *header_outbuf,
1790                                                 view_error_t *error_info)
1791{
1792    couchstore_error_t ret;
1793    tree_file index_file;
1794    tree_file compact_file;
1795    index_header_t *header = NULL;
1796    node_pointer *id_root = NULL;
1797    node_pointer **view_roots = NULL;
1798    bitmap_t *filterbm = NULL;
1799    bitmap_t emptybm;
1800    int i;
1801
1802    memset(&emptybm, 0, sizeof(bitmap_t));
1803    error_info->view_name = NULL;
1804    error_info->error_msg = NULL;
1805    index_file.handle = NULL;
1806    index_file.ops = NULL;
1807    index_file.path = NULL;
1808    compact_file.handle = NULL;
1809    compact_file.ops = NULL;
1810    compact_file.path = NULL;
1811
1812    ret = decode_index_header(header_buf->buf, header_buf->size, &header);
1813    if (ret < 0) {
1814        goto cleanup;
1815    }
1816
1817    /* Set filter bitmask if required */
1818    if (!is_equal_bitmap(&emptybm, &header->cleanup_bitmask)) {
1819        filterbm = &header->cleanup_bitmask;
1820    }
1821
1822    view_roots = (node_pointer **) cb_calloc(info->num_btrees,
1823                                          sizeof(node_pointer *));
1824    if (view_roots == NULL) {
1825        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1826        goto cleanup;
1827    }
1828
1829    cb_assert(info->num_btrees == header->num_views);
1830
1831    ret = open_view_group_file(info->filepath,
1832                               COUCHSTORE_OPEN_FLAG_RDONLY,
1833                               &index_file);
1834    if (ret != COUCHSTORE_SUCCESS) {
1835        goto cleanup;
1836    }
1837
1838    /*
1839     * Open target file for compaction
1840     * Expects that caller created the target file
1841     */
1842    ret = open_view_group_file(target_file, 0, &compact_file);
1843    if (ret != COUCHSTORE_SUCCESS) {
1844        goto cleanup;
1845    }
1846
1847    compact_file.pos = compact_file.ops->goto_eof(&compact_file.lastError,
1848                                                  compact_file.handle);
1849    ret = compact_id_btree(&index_file, &compact_file,
1850                                        header->id_btree_state,
1851                                        filterbm,
1852                                        stats,
1853                                        &id_root);
1854    if (ret != COUCHSTORE_SUCCESS) {
1855        goto cleanup;
1856    }
1857
1858    cb_free(header->id_btree_state);
1859    header->id_btree_state = id_root;
1860    id_root = NULL;
1861
1862    for (i = 0; i < info->num_btrees; ++i) {
1863        switch(info->type) {
1864        case VIEW_INDEX_TYPE_MAPREDUCE:
1865            ret = compact_view_btree(&index_file,
1866                                     &compact_file,
1867                                     &info->view_infos.btree[i],
1868                                     header->view_states[i],
1869                                     filterbm,
1870                                     stats,
1871                                     &view_roots[i],
1872                                     error_info);
1873            break;
1874        case VIEW_INDEX_TYPE_SPATIAL:
1875            ret = compact_view_spatial(&index_file,
1876                                       &compact_file,
1877                                       &info->view_infos.spatial[i],
1878                                       header->view_states[i],
1879                                       filterbm,
1880                                       stats,
1881                                       &view_roots[i],
1882                                       error_info);
1883            break;
1884        }
1885        if (ret != COUCHSTORE_SUCCESS) {
1886            goto cleanup;
1887        }
1888
1889        cb_free(header->view_states[i]);
1890        header->view_states[i] = view_roots[i];
1891        view_roots[i] = NULL;
1892    }
1893
1894    header->cleanup_bitmask = emptybm;
1895    ret = encode_index_header(header, &header_outbuf->buf, &header_outbuf->size);
1896    if (ret != COUCHSTORE_SUCCESS) {
1897        goto cleanup;
1898    }
1899
1900    ret = COUCHSTORE_SUCCESS;
1901
1902cleanup:
1903    free_index_header(header);
1904    close_view_group_file(info);
1905    tree_file_close(&index_file);
1906    tree_file_close(&compact_file);
1907    cb_free(id_root);
1908    if (view_roots != NULL) {
1909        for (i = 0; i < info->num_btrees; ++i) {
1910            cb_free(view_roots[i]);
1911        }
1912        cb_free(view_roots);
1913    }
1914
1915    return ret;
1916}
1917
1918
1919/*
1920 * For initial spatial build, feed the spatial builder as soon as
1921 * sorted records are available.
1922 */
1923static file_merger_error_t build_spatial_record_callback(void *buf, void *ctx)
1924{
1925    int ret;
1926    sized_buf *k, *v;
1927    view_file_merge_record_t *rec = (view_file_merge_record_t *) buf;
1928    view_file_merge_ctx_t *merge_ctx = (view_file_merge_ctx_t *) ctx;
1929    view_spatial_builder_ctx_t *build_ctx =
1930            (view_spatial_builder_ctx_t *) merge_ctx->user_ctx;
1931    sized_buf src_k, src_v;
1932
1933    src_k.size = rec->ksize;
1934    src_k.buf = VIEW_RECORD_KEY(rec);
1935    src_v.size = rec->vsize;
1936    src_v.buf = VIEW_RECORD_VAL(rec);
1937
1938    k = arena_copy_buf(build_ctx->transient_arena, &src_k);
1939    v = arena_copy_buf(build_ctx->transient_arena, &src_v);
1940    ret = spatial_push_item(k, v, build_ctx->modify_result);
1941
1942    if (ret != COUCHSTORE_SUCCESS) {
1943        return static_cast<file_merger_error_t>(ret);
1944    }
1945
1946    if (build_ctx->modify_result->count == 0) {
1947        arena_free_all(build_ctx->transient_arena);
1948    }
1949
1950    return static_cast<file_merger_error_t>(ret);
1951}
1952
1953
1954static couchstore_error_t build_view_spatial(const char *source_file,
1955                                             const view_spatial_info_t *info,
1956                                             tree_file *dest_file,
1957                                             const char *tmpdir,
1958                                             node_pointer **out_root,
1959                                             view_error_t *error_info)
1960{
1961    couchstore_error_t ret;
1962    compare_info cmp;
1963
1964    /* cmp.compare is only needed when you read a b-tree node or modify a
1965     * b-tree node (btree_modify:modify_node()). We don't do either in the
1966     * spatial view. */
1967    cmp.compare = NULL;
1968    ret = build_spatial(source_file,
1969                        dest_file,
1970                        &cmp,
1971                        view_spatial_reduce,
1972                        /* Use the reduce function also for the re-reduce */
1973                        view_spatial_reduce,
1974                        info->dimension,
1975                        info->mbb,
1976                        tmpdir,
1977                        sort_spatial_kvs_file,
1978                        out_root);
1979
1980    if (ret != COUCHSTORE_SUCCESS) {
1981        const int buffsize = 64;
1982        char* buf = (char*) cb_malloc(buffsize);
1983        if (buf != NULL) {
1984            int len = snprintf(buf, buffsize, "ret = %d", ret);
1985            if (len > 0 && len < buffsize) {
1986                error_info->error_msg = buf;
1987            } else {
1988                error_info->error_msg = cb_strdup("Failed to build error message");
1989                cb_free(buf);
1990            }
1991        }
1992    }
1993
1994    return ret;
1995}
1996
1997
1998static couchstore_error_t build_spatial(const char *source_file,
1999                                        tree_file *dest_file,
2000                                        compare_info *cmp,
2001                                        reduce_fn reduce_fun,
2002                                        reduce_fn rereduce_fun,
2003                                        const uint16_t dimension,
2004                                        const double *mbb,
2005                                        const char *tmpdir,
2006                                        sort_record_fn sort_fun,
2007                                        node_pointer **out_root)
2008{
2009    couchstore_error_t ret = COUCHSTORE_SUCCESS;
2010    arena *transient_arena = new_arena(0);
2011    arena *persistent_arena = new_arena(0);
2012    couchfile_modify_result *mr;
2013    view_spatial_builder_ctx_t build_ctx;
2014
2015    if (transient_arena == NULL || persistent_arena == NULL) {
2016        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
2017        goto out;
2018    }
2019
2020    mr = new_btree_modres(persistent_arena,
2021                          transient_arena,
2022                          dest_file,
2023                          cmp,
2024                          reduce_fun,
2025                          rereduce_fun,
2026                          NULL,
2027                          /* Normally the nodes are flushed to disk when 2/3 of
2028                           * the threshold is reached. In order to have a
2029                           * higher fill grade, we add 1/3 */
2030                          VIEW_KV_CHUNK_THRESHOLD + (VIEW_KV_CHUNK_THRESHOLD / 3),
2031                          VIEW_KP_CHUNK_THRESHOLD + (VIEW_KP_CHUNK_THRESHOLD / 3));
2032    if (mr == NULL) {
2033        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
2034        goto out;
2035    }
2036
2037    build_ctx.transient_arena = transient_arena;
2038    build_ctx.modify_result = mr;
2039    build_ctx.scale_factor = spatial_scale_factor(mbb, dimension,
2040                                                  ZCODE_MAX_VALUE);
2041
2042    ret = (couchstore_error_t) sort_fun(source_file,
2043                                        tmpdir,
2044                                        build_spatial_record_callback,
2045                                        &build_ctx);
2046    free_spatial_scale_factor(build_ctx.scale_factor);
2047    if (ret != COUCHSTORE_SUCCESS) {
2048        goto out;
2049    }
2050
2051    *out_root = complete_new_spatial(mr, &ret);
2052    if (ret != COUCHSTORE_SUCCESS) {
2053        goto out;
2054    }
2055
2056    /* Don't care about success/failure. Erlang side will eventually delete it. */
2057    remove(source_file);
2058
2059out:
2060    if (transient_arena != NULL) {
2061        delete_arena(transient_arena);
2062    }
2063
2064    return ret;
2065}
2066
2067/* Add the kv pair to modify result
2068 * The difference to the mapreduce views is the call to `spatial_push_item` */
2069static couchstore_error_t compact_spatial_fetchcb(couchfile_lookup_request *rq,
2070                                                  const sized_buf *k,
2071                                                  const sized_buf *v)
2072{
2073    int ret;
2074    sized_buf *k_c, *v_c;
2075    view_compact_ctx_t *ctx = (view_compact_ctx_t *) rq->callback_ctx;
2076    compactor_stats_t *stats = ctx->stats;
2077
2078    if (k == NULL || v == NULL) {
2079        return COUCHSTORE_ERROR_READ;
2080    }
2081
2082    if (ctx->filter_fun) {
2083        ret = ctx->filter_fun(k, v, ctx->filterbm);
2084        if (ret < 0) {
2085            return (couchstore_error_t) ret;
2086        } else if (ret) {
2087            return COUCHSTORE_SUCCESS;
2088        }
2089    }
2090
2091    k_c = arena_copy_buf(ctx->transient_arena, k);
2092    v_c = arena_copy_buf(ctx->transient_arena, v);
2093    ret = spatial_push_item(k_c, v_c, ctx->mr);
2094    if (ret != COUCHSTORE_SUCCESS) {
2095        return static_cast<couchstore_error_t>(ret);
2096    }
2097
2098    if (stats) {
2099        stats->inserted++;
2100        if (stats->update_fun) {
2101            stats->update_fun(stats->freq, stats->inserted);
2102        }
2103    }
2104
2105    if (ctx->mr->count == 0) {
2106        arena_free_all(ctx->transient_arena);
2107    }
2108
2109    return (couchstore_error_t) ret;
2110}
2111
2112static couchstore_error_t compact_view_spatial(tree_file *source,
2113                                               tree_file *target,
2114                                               const view_spatial_info_t *info,
2115                                               const node_pointer *root,
2116                                               const bitmap_t *filterbm,
2117                                               compactor_stats_t *stats,
2118                                               node_pointer **out_root,
2119                                               view_error_t *error_info)
2120{
2121    couchstore_error_t ret;
2122    compare_info cmp;
2123
2124    cmp.compare = view_btree_cmp;
2125    ret = compact_spatial(source,
2126                          target,
2127                          root,
2128                          &cmp,
2129                          view_spatial_reduce,
2130                          /* Use the reduce function also for the re-reduce */
2131                          view_spatial_reduce,
2132                          view_spatial_filter,
2133                          filterbm,
2134                          stats,
2135                          out_root);
2136
2137    return ret;
2138}
2139
2140static couchstore_error_t compact_spatial(tree_file *source,
2141                                          tree_file *target,
2142                                          const node_pointer *root,
2143                                          compare_info *cmp,
2144                                          reduce_fn reduce_fun,
2145                                          reduce_fn rereduce_fun,
2146                                          compact_filter_fn filter_fun,
2147                                          const bitmap_t *filterbm,
2148                                          compactor_stats_t *stats,
2149                                          node_pointer **out_root)
2150{
2151    couchstore_error_t ret = COUCHSTORE_SUCCESS;
2152    arena *transient_arena = new_arena(0);
2153    arena *persistent_arena = new_arena(0);
2154    couchfile_modify_result *modify_result;
2155    couchfile_lookup_request lookup_rq;
2156    view_compact_ctx_t compact_ctx;
2157    sized_buf nullkey = {NULL, 0};
2158    sized_buf *lowkeys = &nullkey;
2159
2160    if (!root) {
2161        return COUCHSTORE_SUCCESS;
2162    }
2163
2164    if (transient_arena == NULL || persistent_arena == NULL) {
2165        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
2166        goto cleanup;
2167    }
2168
2169    /* Create new spatial index on new file */
2170    modify_result = new_btree_modres(persistent_arena,
2171                                     transient_arena,
2172                                     target,
2173                                     cmp,
2174                                     reduce_fun,
2175                                     rereduce_fun,
2176                                     NULL,
2177                                     /* Normally the nodes are flushed to disk
2178                                      * when 2/3 of the threshold is reached.
2179                                      * In order to have a higher fill grade,
2180                                      * we add 1/3 */
2181                                     VIEW_KV_CHUNK_THRESHOLD +
2182                                         (VIEW_KV_CHUNK_THRESHOLD / 3),
2183                                     VIEW_KP_CHUNK_THRESHOLD +
2184                                         (VIEW_KP_CHUNK_THRESHOLD / 3));
2185    if (modify_result == NULL) {
2186        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
2187        goto cleanup;
2188    }
2189
2190    compact_ctx.filter_fun = NULL;
2191    compact_ctx.mr = modify_result;
2192    compact_ctx.transient_arena = transient_arena;
2193    compact_ctx.stats = stats;
2194
2195    if (filterbm) {
2196        compact_ctx.filterbm = filterbm;
2197        compact_ctx.filter_fun = filter_fun;
2198    }
2199
2200    lookup_rq.cmp.compare = cmp->compare;
2201    lookup_rq.file = source;
2202    lookup_rq.num_keys = 1;
2203    lookup_rq.keys = &lowkeys;
2204    lookup_rq.callback_ctx = &compact_ctx;
2205    lookup_rq.fetch_callback = compact_spatial_fetchcb;
2206    lookup_rq.node_callback = NULL;
2207    lookup_rq.fold = 1;
2208
2209    ret = btree_lookup(&lookup_rq, root->pointer);
2210    if (ret != COUCHSTORE_SUCCESS) {
2211        goto cleanup;
2212    }
2213
2214    *out_root = complete_new_spatial(modify_result, &ret);
2215
2216cleanup:
2217    if (transient_arena != NULL) {
2218        delete_arena(transient_arena);
2219    }
2220
2221    if (persistent_arena != NULL) {
2222        delete_arena(persistent_arena);
2223    }
2224
2225    return ret;
2226}
2227