xref: /4.0.0/couchstore/src/views/view_group.c (revision ed519740)
1/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2
3/**
4 * @copyright 2013 Couchbase, Inc.
5 *
6 * @author Filipe Manana  <filipe@couchbase.com>
7 *
8 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
9 * use this file except in compliance with the License. You may obtain a copy of
10 * the License at
11 *
12 *  http://www.apache.org/licenses/LICENSE-2.0
13 *
14 * Unless required by applicable law or agreed to in writing, software
15 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17 * License for the specific language governing permissions and limitations under
18 * the License.
19 **/
20
21#include "config.h"
22#include <assert.h>
23#include <stdlib.h>
24#include <string.h>
25#include "view_group.h"
26#include "reducers.h"
27#include "reductions.h"
28#include "values.h"
29#include "purgers.h"
30#include "util.h"
31#include "file_sorter.h"
32#include "spatial.h"
33#include "../arena.h"
34#include "../couch_btree.h"
35#include "../internal.h"
36#include "../util.h"
37
38#define VIEW_KV_CHUNK_THRESHOLD (7 * 1024)
39#define VIEW_KP_CHUNK_THRESHOLD (6 * 1024)
40#define MAX_HEADER_SIZE         (64 * 1024)
41#define MAX_ACTIONS_SIZE        (2 * 1024 * 1024)
42
43static couchstore_error_t read_btree_info(view_group_info_t *info,
44                                          FILE *in_stream,
45                                          FILE *error_stream);
46
47static couchstore_error_t read_spatial_info(view_group_info_t *info,
48                                            FILE *in_stream,
49                                            FILE *error_stream);
50
51static couchstore_error_t open_view_group_file(const char *path,
52                                               couchstore_open_flags open_flags,
53                                               tree_file *file);
54
55static couchstore_error_t build_btree(const char *source_file,
56                                      tree_file *dest_file,
57                                      compare_info *cmp,
58                                      reduce_fn reduce_fun,
59                                      reduce_fn rereduce_fun,
60                                      const char *tmpdir,
61                                      sort_record_fn sort_fun,
62                                      void *reduce_ctx,
63                                      node_pointer **out_root);
64
65static couchstore_error_t build_id_btree(const char *source_file,
66                                         tree_file *dest_file,
67                                         const char *tmpdir,
68                                         node_pointer **out_root);
69
70static couchstore_error_t build_view_btree(const char *source_file,
71                                           const view_btree_info_t *info,
72                                           tree_file *dest_file,
73                                           const char *tmpdir,
74                                           node_pointer **out_root,
75                                           view_error_t *error_info);
76
77static void close_view_group_file(view_group_info_t *info);
78
79static int read_record(FILE *f, arena *a, sized_buf *k, sized_buf *v,
80                                                        uint8_t *op);
81
82int view_btree_cmp(const sized_buf *key1, const sized_buf *key2);
83
84static couchstore_error_t update_btree(const char *source_file,
85                                       tree_file *dest_file,
86                                       const node_pointer *root,
87                                       size_t batch_size,
88                                       compare_info *cmp,
89                                       reduce_fn reduce_fun,
90                                       reduce_fn rereduce_fun,
91                                       purge_kv_fn purge_kv,
92                                       purge_kp_fn purge_kp,
93                                       view_reducer_ctx_t *red_ctx,
94                                       view_purger_ctx_t *purge_ctx,
95                                       uint64_t *inserted,
96                                       uint64_t *removed,
97                                       node_pointer **out_root);
98
99static couchstore_error_t update_id_btree(const char *source_file,
100                                         tree_file *dest_file,
101                                         const node_pointer *root,
102                                         size_t batch_size,
103                                         view_purger_ctx_t *purge_ctx,
104                                         uint64_t *inserted,
105                                         uint64_t *removed,
106                                         node_pointer **out_root);
107
108static couchstore_error_t update_view_btree(const char *source_file,
109                                            const view_btree_info_t *info,
110                                            tree_file *dest_file,
111                                            const node_pointer *root,
112                                            size_t batch_size,
113                                            view_purger_ctx_t *purge_ctx,
114                                            uint64_t *inserted,
115                                            uint64_t *removed,
116                                            node_pointer **out_root,
117                                            view_error_t *error_info);
118
119static couchstore_error_t compact_view_fetchcb(couchfile_lookup_request *rq,
120                                        const sized_buf *k,
121                                        const sized_buf *v);
122
123static couchstore_error_t compact_btree(tree_file *source,
124                                 tree_file *target,
125                                 const node_pointer *root,
126                                 compare_info *cmp,
127                                 reduce_fn reduce_fun,
128                                 reduce_fn rereduce_fun,
129                                 compact_filter_fn filter_fun,
130                                 view_reducer_ctx_t *red_ctx,
131                                 const bitmap_t *filterbm,
132                                 compactor_stats_t *stats,
133                                 node_pointer **out_root);
134
135static couchstore_error_t compact_id_btree(tree_file *source,
136                                    tree_file *target,
137                                    const node_pointer *root,
138                                    const bitmap_t *filterbm,
139                                    compactor_stats_t *stats,
140                                    node_pointer **out_root);
141
142static couchstore_error_t compact_view_btree(tree_file *source,
143                                      tree_file *target,
144                                      const view_btree_info_t *info,
145                                      const node_pointer *root,
146                                      const bitmap_t *filterbm,
147                                      compactor_stats_t *stats,
148                                      node_pointer **out_root,
149                                      view_error_t *error_info);
150
151/* Some preparations before building the actual spatial index */
152static couchstore_error_t build_view_spatial(const char *source_file,
153                                             const view_spatial_info_t *info,
154                                             tree_file *dest_file,
155                                             const char *tmpdir,
156                                             node_pointer **out_root,
157                                             view_error_t *error_info);
158
159/* Build the actual spatial index */
160static couchstore_error_t build_spatial(const char *source_file,
161                                        tree_file *dest_file,
162                                        compare_info *cmp,
163                                        reduce_fn reduce_fun,
164                                        reduce_fn rereduce_fun,
165                                        const uint16_t dimension,
166                                        const double *mbb,
167                                        const char *tmpdir,
168                                        sort_record_fn sort_fun,
169                                        node_pointer **out_root);
170
171/* Callback for every item that got fetched from the original view */
172static couchstore_error_t compact_spatial_fetchcb(couchfile_lookup_request *rq,
173                                                  const sized_buf *k,
174                                                  const sized_buf *v);
175
176/* Some preparations before compacting the actual spatial index */
177static couchstore_error_t compact_view_spatial(tree_file *source,
178                                               tree_file *target,
179                                               const view_spatial_info_t *info,
180                                               const node_pointer *root,
181                                               const bitmap_t *filterbm,
182                                               compactor_stats_t *stats,
183                                               node_pointer **out_root,
184                                               view_error_t *error_info);
185
186/* Compact the actual spatial index */
187static couchstore_error_t compact_spatial(tree_file *source,
188                                          tree_file *target,
189                                          const node_pointer *root,
190                                          compare_info *cmp,
191                                          reduce_fn reduce_fun,
192                                          reduce_fn rereduce_fun,
193                                          compact_filter_fn filter_fun,
194                                          const bitmap_t *filterbm,
195                                          compactor_stats_t *stats,
196                                          node_pointer **out_root);
197
198LIBCOUCHSTORE_API
199view_group_info_t *couchstore_read_view_group_info(FILE *in_stream,
200                                                   FILE *error_stream)
201{
202    view_group_info_t *info;
203    char buf[4096];
204    char *dup;
205    couchstore_error_t ret;
206    uint64_t type;
207
208    info = (view_group_info_t *) calloc(1, sizeof(*info));
209    if (info == NULL) {
210        fprintf(error_stream, "Memory allocation failure\n");
211        goto out_error;
212    }
213
214    type = couchstore_read_int(in_stream, buf, sizeof(buf), &ret);
215    if (ret != COUCHSTORE_SUCCESS) {
216        fprintf(stderr, "Error reading view file type\n");
217        goto out_error;
218    }
219    switch (type) {
220    case VIEW_INDEX_TYPE_MAPREDUCE:
221        info->type = VIEW_INDEX_TYPE_MAPREDUCE;
222        break;
223    case VIEW_INDEX_TYPE_SPATIAL:
224        info->type = VIEW_INDEX_TYPE_SPATIAL;
225        break;
226    default:
227        fprintf(stderr, "Invalid view file type: %"PRIu64"\n", type);
228        goto out_error;
229    }
230
231    if (couchstore_read_line(in_stream, buf, sizeof(buf)) != buf) {
232        fprintf(stderr, "Error reading source index file path\n");
233        goto out_error;
234    }
235    dup = strdup(buf);
236    if (dup == NULL) {
237        fprintf(error_stream, "Memory allocation failure\n");
238        goto out_error;
239    }
240    info->filepath = (const char *) dup;
241
242    info->header_pos = couchstore_read_int(in_stream, buf,
243                                                      sizeof(buf),
244                                                      &ret);
245    if (ret != COUCHSTORE_SUCCESS) {
246        fprintf(error_stream, "Error reading header position\n");
247        goto out_error;
248    }
249
250    info->num_btrees = couchstore_read_int(in_stream, buf,
251                                                      sizeof(buf),
252                                                      &ret);
253    if (ret != COUCHSTORE_SUCCESS) {
254        fprintf(error_stream, "Error reading number of btrees\n");
255        goto out_error;
256    }
257
258    switch (info->type) {
259    case VIEW_INDEX_TYPE_MAPREDUCE:
260        ret = read_btree_info(info, in_stream, error_stream);
261        break;
262    case VIEW_INDEX_TYPE_SPATIAL:
263        ret = read_spatial_info(info, in_stream, error_stream);
264        break;
265    }
266    if (ret != COUCHSTORE_SUCCESS) {
267        goto out_error;
268    }
269
270    return info;
271
272out_error:
273    couchstore_free_view_group_info(info);
274
275    return NULL;
276}
277
278
279/* Read in the information about the mapreduce view indexes */
280static couchstore_error_t read_btree_info(view_group_info_t *info,
281                                          FILE *in_stream,
282                                          FILE *error_stream)
283{
284    char buf[4096];
285    char *dup;
286    int i, j;
287    int reduce_len;
288    couchstore_error_t ret;
289
290    info->view_infos.btree = (view_btree_info_t *)
291        calloc(info->num_btrees, sizeof(view_btree_info_t));
292    if (info->view_infos.btree == NULL) {
293        fprintf(error_stream, "Memory allocation failure on btree infos\n");
294        info->num_btrees = 0;
295        return COUCHSTORE_ERROR_ALLOC_FAIL;
296    }
297
298    for (i = 0; i < info->num_btrees; ++i) {
299        view_btree_info_t *bti = &info->view_infos.btree[i];
300
301        bti->view_id = i;
302        bti->num_reducers = couchstore_read_int(in_stream,
303                                                buf,
304                                                sizeof(buf),
305                                                &ret);
306
307        if (ret != COUCHSTORE_SUCCESS) {
308            fprintf(error_stream,
309                    "Error reading number of reducers for btree %d\n", i);
310            return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
311        }
312
313        bti->names = (const char **) calloc(bti->num_reducers, sizeof(char *));
314        if (bti->names == NULL) {
315            fprintf(error_stream,
316                    "Memory allocation failure on btree %d reducer names\n",
317                    i);
318            bti->num_reducers = 0;
319            return COUCHSTORE_ERROR_ALLOC_FAIL;
320        }
321
322        bti->reducers = (const char **) calloc(bti->num_reducers, sizeof(char *));
323        if (bti->reducers == NULL) {
324            fprintf(error_stream,
325                    "Memory allocation failure on btree %d reducers\n", i);
326            bti->num_reducers = 0;
327            free(bti->names);
328            return COUCHSTORE_ERROR_ALLOC_FAIL;
329        }
330
331        for (j = 0; j < bti->num_reducers; ++j) {
332            if (couchstore_read_line(in_stream, buf, sizeof(buf)) != buf) {
333                fprintf(error_stream,
334                        "Error reading btree %d view %d name\n", i, j);
335                return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
336            }
337            dup = strdup(buf);
338            if (dup == NULL) {
339                fprintf(error_stream,
340                        "Memory allocation failure on btree %d "
341                        "view %d name\n", i, j);
342                return COUCHSTORE_ERROR_ALLOC_FAIL;
343            }
344            bti->names[j] = (const char *) dup;
345
346            reduce_len = couchstore_read_int(in_stream,
347                                             buf,
348                                             sizeof(buf),
349                                             &ret);
350            if (ret != COUCHSTORE_SUCCESS) {
351                fprintf(error_stream,
352                        "Error reading btree %d view %d "
353                        "reduce function size\n", i, j);
354                return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
355            }
356
357            dup = (char *) malloc(reduce_len + 1);
358            if (dup == NULL) {
359                fprintf(error_stream,
360                        "Memory allocation failure on btree %d "
361                        "view %d reducer\n", i, j);
362                return COUCHSTORE_ERROR_ALLOC_FAIL;
363            }
364
365            if (fread(dup, reduce_len, 1, in_stream) != 1) {
366                fprintf(error_stream,
367                        "Error reading btree %d view %d reducer\n", i, j);
368                free(dup);
369                return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
370            }
371            dup[reduce_len] = '\0';
372            bti->reducers[j] = (const char *) dup;
373        }
374    }
375    return COUCHSTORE_SUCCESS;
376}
377
378
379/* Read in the information about the spatial view indexes */
380static couchstore_error_t read_spatial_info(view_group_info_t *info,
381                                            FILE *in_stream,
382                                            FILE *error_stream)
383{
384    char buf[24];
385    int i;
386    couchstore_error_t ret;
387
388    info->view_infos.spatial = (view_spatial_info_t *)
389        calloc(info->num_btrees, sizeof(view_spatial_info_t));
390    if (info->view_infos.spatial == NULL) {
391        fprintf(error_stream, "Memory allocation failure on spatial infos\n");
392        info->num_btrees = 0;
393        return COUCHSTORE_ERROR_ALLOC_FAIL;
394    }
395
396    for (i = 0; i < info->num_btrees; ++i) {
397        view_spatial_info_t *si = &info->view_infos.spatial[i];
398
399        si->dimension = couchstore_read_int(in_stream, buf, sizeof(buf), &ret);
400
401        if (ret != COUCHSTORE_SUCCESS) {
402            fprintf(error_stream,
403                    "Error reading the dimension of spatial view %d\n", i);
404            return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
405        }
406
407        /* If the dimension is 0, no index exists or hasn't been built yet */
408        if (si->dimension == 0) {
409            continue;
410        }
411
412        si->mbb = (double *) calloc(si->dimension * 2, sizeof(double));
413        if (si->mbb == NULL) {
414            fprintf(error_stream,
415                    "Memory allocation failure on spatial view %d\n", i);
416            si->dimension = 0;
417            return COUCHSTORE_ERROR_ALLOC_FAIL;
418        }
419
420        if (fread(si->mbb, sizeof(double), si->dimension * 2, in_stream) <
421                si->dimension * 2) {
422            fprintf(error_stream, "Error reading mbb of view %d\n", i);
423            return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
424        }
425    }
426    return COUCHSTORE_SUCCESS;
427}
428
429
430LIBCOUCHSTORE_API
431void couchstore_free_view_group_info(view_group_info_t *info)
432{
433    int i, j;
434
435    if (info == NULL)
436        return;
437
438    close_view_group_file(info);
439
440    switch (info->type) {
441    case VIEW_INDEX_TYPE_MAPREDUCE:
442        for (i = 0; i < info->num_btrees; ++i) {
443            view_btree_info_t vi = info->view_infos.btree[i];
444
445            for (j = 0; j < vi.num_reducers; ++j) {
446                free((void *) vi.names[j]);
447                free((void *) vi.reducers[j]);
448            }
449            free(vi.names);
450            free(vi.reducers);
451        }
452        free(info->view_infos.btree);
453        break;
454    case VIEW_INDEX_TYPE_SPATIAL:
455        for (i = 0; i < info->num_btrees; ++i) {
456            free((void *) info->view_infos.spatial[i].mbb);
457        }
458        free(info->view_infos.spatial);
459        break;
460    }
461    free((void *) info->filepath);
462    free(info);
463}
464
465
466static void close_view_group_file(view_group_info_t *info)
467{
468    if (info->file.ops != NULL) {
469        info->file.ops->close(NULL, info->file.handle);
470        info->file.ops->destructor(NULL, info->file.handle);
471        info->file.ops = NULL;
472        info->file.handle = NULL;
473    }
474    free((void *) info->file.path);
475    info->file.path = NULL;
476}
477
478
479static couchstore_error_t open_view_group_file(const char *path,
480                                               couchstore_open_flags open_flags,
481                                               tree_file *file)
482{
483    couchstore_error_t ret;
484    const couch_file_ops *file_ops = couchstore_get_default_file_ops();
485    int flags = 0;
486
487    if ((open_flags & COUCHSTORE_OPEN_FLAG_RDONLY) &&
488        (open_flags & COUCHSTORE_OPEN_FLAG_CREATE)) {
489        return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
490    }
491
492    if (open_flags & COUCHSTORE_OPEN_FLAG_RDONLY) {
493        flags |= O_RDONLY;
494    } else {
495        flags |= O_RDWR;
496    }
497
498    if (open_flags & COUCHSTORE_OPEN_FLAG_CREATE) {
499        flags |= O_CREAT;
500    }
501
502    ret = tree_file_open(file, path, flags, file_ops);
503
504    return ret;
505}
506
507
508LIBCOUCHSTORE_API
509couchstore_error_t couchstore_build_view_group(view_group_info_t *info,
510                                               const char *id_records_file,
511                                               const char *kv_records_files[],
512                                               const char *dst_file,
513                                               const char *tmpdir,
514                                               uint64_t *header_pos,
515                                               view_error_t *error_info)
516{
517    couchstore_error_t ret;
518    tree_file index_file;
519    index_header_t *header = NULL;
520    node_pointer *id_root = NULL;
521    node_pointer **view_roots = NULL;
522    int i;
523
524    error_info->view_name = NULL;
525    error_info->error_msg = NULL;
526    index_file.handle = NULL;
527    index_file.ops = NULL;
528    index_file.path = NULL;
529
530    view_roots = (node_pointer **) calloc(
531        info->num_btrees, sizeof(node_pointer *));
532    if (view_roots == NULL) {
533        return COUCHSTORE_ERROR_ALLOC_FAIL;
534    }
535
536    ret = open_view_group_file(info->filepath,
537                               COUCHSTORE_OPEN_FLAG_RDONLY,
538                               &info->file);
539    if (ret != COUCHSTORE_SUCCESS) {
540        goto out;
541    }
542
543    ret = read_view_group_header(info, &header);
544    if (ret != COUCHSTORE_SUCCESS) {
545        goto out;
546    }
547    assert(info->num_btrees == header->num_views);
548
549    ret = open_view_group_file(dst_file,
550                               COUCHSTORE_OPEN_FLAG_CREATE,
551                               &index_file);
552    if (ret != COUCHSTORE_SUCCESS) {
553        goto out;
554    }
555
556    ret = build_id_btree(id_records_file, &index_file, tmpdir, &id_root);
557    if (ret != COUCHSTORE_SUCCESS) {
558        goto out;
559    }
560
561    free(header->id_btree_state);
562    header->id_btree_state = id_root;
563    id_root = NULL;
564
565    for (i = 0; i < info->num_btrees; ++i) {
566        switch(info->type) {
567        case VIEW_INDEX_TYPE_MAPREDUCE:
568            ret = build_view_btree(kv_records_files[i],
569                                   &info->view_infos.btree[i],
570                                   &index_file,
571                                   tmpdir,
572                                   &view_roots[i],
573                                   error_info);
574            break;
575        case VIEW_INDEX_TYPE_SPATIAL:
576            ret = build_view_spatial(kv_records_files[i],
577                                     &info->view_infos.spatial[i],
578                                     &index_file,
579                                     tmpdir,
580                                     &view_roots[i],
581                                     error_info);
582            break;
583        }
584        if (ret != COUCHSTORE_SUCCESS) {
585            goto out;
586        }
587
588        free(header->view_states[i]);
589        header->view_states[i] = view_roots[i];
590        view_roots[i] = NULL;
591    }
592
593    ret = write_view_group_header(&index_file, header_pos, header);
594    if (ret != COUCHSTORE_SUCCESS) {
595        goto out;
596    }
597
598    ret = COUCHSTORE_SUCCESS;
599
600out:
601    free_index_header(header);
602    close_view_group_file(info);
603    tree_file_close(&index_file);
604    free(id_root);
605    if (view_roots != NULL) {
606        for (i = 0; i < info->num_btrees; ++i) {
607            free(view_roots[i]);
608        }
609        free(view_roots);
610    }
611
612    return ret;
613}
614
615
616/*
617 * Similar to util.c:read_view_record(), but it uses arena allocator, which is
618 * required for the existing semantics/api of btree bottom-up build in
619 * src/btree_modify.cc.
620 */
621static int read_record(FILE *f, arena *a, sized_buf *k, sized_buf *v,
622                                                        uint8_t *op)
623{
624    uint16_t klen;
625    uint8_t oplen = 0;
626    uint32_t vlen, len;
627
628    if (fread(&len, sizeof(len), 1, f) != 1) {
629        if (feof(f)) {
630            return 0;
631        } else {
632            return COUCHSTORE_ERROR_READ;
633        }
634    }
635
636    /* For incremental update, read optype */
637    if (op != NULL) {
638        if (fread(op, sizeof(uint8_t), 1, f) != 1) {
639            return COUCHSTORE_ERROR_READ;
640        }
641
642        oplen = 1;
643    }
644
645    if (fread(&klen, sizeof(klen), 1, f) != 1) {
646        return COUCHSTORE_ERROR_READ;
647    }
648
649    klen = ntohs(klen);
650    vlen = len - sizeof(klen) - klen - oplen;
651
652    k->size = klen;
653    k->buf = (char *) arena_alloc(a, k->size);
654    if (k->buf == NULL) {
655        return COUCHSTORE_ERROR_ALLOC_FAIL;
656    }
657
658    v->size = vlen;
659    /* Handle zero len vals */
660    if (v->size) {
661        v->buf = (char *) arena_alloc(a, v->size);
662        if (v->buf == NULL) {
663            return COUCHSTORE_ERROR_ALLOC_FAIL;
664        }
665    } else {
666        v->buf = NULL;
667    }
668
669    if (fread(k->buf, k->size, 1, f) != 1) {
670        return FILE_MERGER_ERROR_FILE_READ;
671    }
672
673    if (v->size && fread(v->buf, v->size, 1, f) != 1) {
674        return FILE_MERGER_ERROR_FILE_READ;
675    }
676
677    return len;
678}
679
680
681static int id_btree_cmp(const sized_buf *key1, const sized_buf *key2)
682{
683    return view_id_cmp(key1, key2, NULL);
684}
685
686
687int view_btree_cmp(const sized_buf *key1, const sized_buf *key2)
688{
689    return view_key_cmp(key1, key2, NULL);
690}
691
692
693/*
694 * For initial btree build, feed the btree builder as soon as
695 * sorted records are available.
696 */
697static file_merger_error_t build_btree_record_callback(void *buf, void *ctx)
698{
699    int ret;
700    sized_buf *k, *v;
701    view_file_merge_record_t *rec = (view_file_merge_record_t *) buf;
702    view_file_merge_ctx_t *merge_ctx = (view_file_merge_ctx_t *) ctx;
703    view_btree_builder_ctx_t *build_ctx =
704                    (view_btree_builder_ctx_t *) merge_ctx->user_ctx;
705    sized_buf src_k, src_v;
706
707    src_k.size = rec->ksize;
708    src_k.buf = VIEW_RECORD_KEY(rec);
709    src_v.size = rec->vsize;
710    src_v.buf = VIEW_RECORD_VAL(rec);
711
712    k = arena_copy_buf(build_ctx->transient_arena, &src_k);
713    v = arena_copy_buf(build_ctx->transient_arena, &src_v);
714    ret = mr_push_item(k, v, build_ctx->modify_result);
715
716    if (ret != COUCHSTORE_SUCCESS) {
717        return ret;
718    }
719
720    if (build_ctx->modify_result->count == 0) {
721        arena_free_all(build_ctx->transient_arena);
722    }
723
724    return ret;
725}
726
727
728static couchstore_error_t build_btree(const char *source_file,
729                                      tree_file *dest_file,
730                                      compare_info *cmp,
731                                      reduce_fn reduce_fun,
732                                      reduce_fn rereduce_fun,
733                                      const char *tmpdir,
734                                      sort_record_fn sort_fun,
735                                      void *reduce_ctx,
736                                      node_pointer **out_root)
737{
738    couchstore_error_t ret = COUCHSTORE_SUCCESS;
739    arena *transient_arena = new_arena(0);
740    arena *persistent_arena = new_arena(0);
741    couchfile_modify_result *mr;
742    view_btree_builder_ctx_t build_ctx;
743
744    if (transient_arena == NULL || persistent_arena == NULL) {
745        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
746        goto out;
747    }
748
749    mr = new_btree_modres(persistent_arena,
750                          transient_arena,
751                          dest_file,
752                          cmp,
753                          reduce_fun,
754                          rereduce_fun,
755                          reduce_ctx,
756                          VIEW_KV_CHUNK_THRESHOLD + (VIEW_KV_CHUNK_THRESHOLD / 3),
757                          VIEW_KP_CHUNK_THRESHOLD + (VIEW_KP_CHUNK_THRESHOLD / 3));
758    if (mr == NULL) {
759        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
760        goto out;
761    }
762
763    build_ctx.transient_arena = transient_arena;
764    build_ctx.modify_result = mr;
765
766    ret = (couchstore_error_t) sort_fun(source_file,
767                                        tmpdir,
768                                        build_btree_record_callback, &build_ctx);
769    if (ret != COUCHSTORE_SUCCESS) {
770        goto out;
771    }
772
773    *out_root = complete_new_btree(mr, &ret);
774    if (ret != COUCHSTORE_SUCCESS) {
775        goto out;
776    }
777
778    /* Don't care about success/failure. Erlang side will eventually delete it. */
779    remove(source_file);
780
781out:
782    if (transient_arena != NULL) {
783        delete_arena(transient_arena);
784    }
785    if (persistent_arena != NULL) {
786        delete_arena(persistent_arena);
787    }
788
789    return ret;
790}
791
792
793static couchstore_error_t build_id_btree(const char *source_file,
794                                         tree_file *dest_file,
795                                         const char *tmpdir,
796                                         node_pointer **out_root)
797{
798    couchstore_error_t ret;
799    compare_info cmp;
800
801    cmp.compare = id_btree_cmp;
802
803    ret = build_btree(source_file,
804                      dest_file,
805                      &cmp,
806                      view_id_btree_reduce,
807                      view_id_btree_rereduce,
808                      tmpdir,
809                      sort_view_ids_file,
810                      NULL,
811                      out_root);
812
813    return ret;
814}
815
816
817static couchstore_error_t build_view_btree(const char *source_file,
818                                           const view_btree_info_t *info,
819                                           tree_file *dest_file,
820                                           const char *tmpdir,
821                                           node_pointer **out_root,
822                                           view_error_t *error_info)
823{
824    couchstore_error_t ret = COUCHSTORE_SUCCESS;
825    compare_info cmp;
826    view_reducer_ctx_t *red_ctx = NULL;
827    char *error_msg = NULL;
828
829    cmp.compare = view_btree_cmp;
830    red_ctx = make_view_reducer_ctx(info->reducers,
831                                    info->num_reducers,
832                                    &error_msg);
833    if (red_ctx == NULL) {
834        set_error_info(info, (const char *) error_msg, ret, error_info);
835        return COUCHSTORE_ERROR_REDUCER_FAILURE;
836    }
837
838    ret = build_btree(source_file,
839                      dest_file,
840                      &cmp,
841                      view_btree_reduce,
842                      view_btree_rereduce,
843                      tmpdir,
844                      sort_view_kvs_file,
845                      red_ctx,
846                      out_root);
847
848    if (ret != COUCHSTORE_SUCCESS) {
849        set_error_info(info, red_ctx->error, ret, error_info);
850    }
851
852    free_view_reducer_ctx(red_ctx);
853
854    return ret;
855}
856
857
858couchstore_error_t read_view_group_header(view_group_info_t *info,
859                                          index_header_t **header)
860{
861    couchstore_error_t ret;
862    cs_off_t pos = info->header_pos;
863    tree_file *file = &info->file;
864    char *header_buf = NULL;
865    int header_len;
866    char buf;
867
868    if (file->handle == NULL) {
869        return COUCHSTORE_ERROR_FILE_CLOSED;
870    }
871
872    if (info->file.ops->pread(NULL, file->handle, &buf, 1, pos) != 1) {
873        return COUCHSTORE_ERROR_READ;
874    }
875    if (buf == 0) {
876        return COUCHSTORE_ERROR_NO_HEADER;
877    } else if (buf != 1) {
878        return COUCHSTORE_ERROR_CORRUPT;
879    }
880
881    header_len = pread_header(file, pos, &header_buf, MAX_HEADER_SIZE);
882    if (header_len < 0) {
883        return (couchstore_error_t) header_len;
884    }
885
886    ret = decode_index_header(header_buf, (size_t) header_len, header);
887    free(header_buf);
888
889    return ret;
890}
891
892couchstore_error_t write_view_group_header(tree_file *file,
893                                           uint64_t *pos,
894                                           const index_header_t *header)
895{
896    couchstore_error_t ret;
897    sized_buf buf = { NULL, 0 };
898    cs_off_t p;
899
900    if (file->handle == NULL) {
901        return COUCHSTORE_ERROR_FILE_CLOSED;
902    }
903
904    ret = encode_index_header(header, &buf.buf, &buf.size);
905    if (ret != COUCHSTORE_SUCCESS) {
906        return ret;
907    }
908
909    ret = write_header(file, &buf, &p);
910    if (ret != COUCHSTORE_SUCCESS) {
911        goto out;
912    }
913
914    assert(p >= 0);
915    *pos = (uint64_t) p;
916
917out:
918    free(buf.buf);
919
920    return ret;
921}
922
923
924static couchstore_error_t view_id_bitmask(const node_pointer *root, bitmap_t *bm)
925{
926    view_id_btree_reduction_t *r = NULL;
927    couchstore_error_t errcode;
928    if (root == NULL) {
929        return COUCHSTORE_SUCCESS;
930    }
931
932    errcode = decode_view_id_btree_reduction(root->reduce_value.buf, &r);
933    if (errcode != COUCHSTORE_SUCCESS) {
934        goto cleanup;
935    }
936
937    union_bitmaps(bm, &r->partitions_bitmap);
938
939cleanup:
940    free_view_id_btree_reduction(r);
941    return errcode;
942}
943
944
945static couchstore_error_t view_bitmask(const node_pointer *root, bitmap_t *bm)
946{
947    view_btree_reduction_t *r = NULL;
948    couchstore_error_t errcode;
949    if (root == NULL) {
950        return COUCHSTORE_SUCCESS;
951    }
952
953    errcode = decode_view_btree_reduction(root->reduce_value.buf,
954                                          root->reduce_value.size, &r);
955    if (errcode != COUCHSTORE_SUCCESS) {
956        goto cleanup;
957    }
958
959    union_bitmaps(bm, &r->partitions_bitmap);
960
961cleanup:
962    free_view_btree_reduction(r);
963    return errcode;
964}
965
966
967static couchstore_error_t cleanup_btree(tree_file *file,
968                                        node_pointer *root,
969                                        compare_info *cmp,
970                                        reduce_fn reduce,
971                                        reduce_fn rereduce,
972                                        purge_kv_fn purge_kv,
973                                        purge_kp_fn purge_kp,
974                                        view_purger_ctx_t *purge_ctx,
975                                        view_reducer_ctx_t *red_ctx,
976                                        node_pointer **out_root)
977{
978    couchstore_error_t errcode;
979    couchfile_modify_request rq;
980
981    rq.cmp = *cmp;
982    rq.file = file;
983    rq.actions = NULL;
984    rq.num_actions = 0;
985    rq.reduce = reduce;
986    rq.rereduce = rereduce;
987    rq.compacting = 0;
988    rq.kv_chunk_threshold = VIEW_KV_CHUNK_THRESHOLD;
989    rq.kp_chunk_threshold = VIEW_KP_CHUNK_THRESHOLD;
990    rq.purge_kp = purge_kp;
991    rq.purge_kv = purge_kv;
992    rq.enable_purging = 1;
993    rq.guided_purge_ctx = purge_ctx;
994    rq.user_reduce_ctx = red_ctx;
995
996    *out_root = guided_purge_btree(&rq, root, &errcode);
997
998    return errcode;
999}
1000
1001static couchstore_error_t cleanup_id_btree(tree_file *file,
1002                                           node_pointer *root,
1003                                           node_pointer **out_root,
1004                                           view_purger_ctx_t *purge_ctx,
1005                                           view_error_t *error_info)
1006{
1007    couchstore_error_t ret;
1008    compare_info cmp;
1009
1010    cmp.compare = id_btree_cmp;
1011
1012    ret = cleanup_btree(file,
1013                        root,
1014                        &cmp,
1015                        view_id_btree_reduce,
1016                        view_id_btree_rereduce,
1017                        view_id_btree_purge_kv,
1018                        view_id_btree_purge_kp,
1019                        purge_ctx,
1020                        NULL,
1021                        out_root);
1022
1023    return ret;
1024}
1025
1026
1027static couchstore_error_t cleanup_view_btree(tree_file *file,
1028                                             node_pointer *root,
1029                                             const view_btree_info_t *info,
1030                                             node_pointer **out_root,
1031                                             view_purger_ctx_t *purge_ctx,
1032                                             view_error_t *error_info)
1033{
1034    couchstore_error_t ret = COUCHSTORE_SUCCESS;
1035    compare_info cmp;
1036    view_reducer_ctx_t *red_ctx = NULL;
1037    char *error_msg = NULL;
1038
1039    cmp.compare = view_btree_cmp;
1040    red_ctx = make_view_reducer_ctx(info->reducers,
1041                                    info->num_reducers,
1042                                    &error_msg);
1043    if (red_ctx == NULL) {
1044        set_error_info(info, (const char *) error_msg, ret, error_info);
1045        return COUCHSTORE_ERROR_REDUCER_FAILURE;
1046    }
1047
1048    ret = cleanup_btree(file,
1049                        root,
1050                        &cmp,
1051                        view_btree_reduce,
1052                        view_btree_rereduce,
1053                        view_btree_purge_kv,
1054                        view_btree_purge_kp,
1055                        purge_ctx,
1056                        red_ctx,
1057                        out_root);
1058
1059    if (ret != COUCHSTORE_SUCCESS) {
1060        char *error_msg = NULL;
1061
1062        if (red_ctx->error != NULL) {
1063            error_msg = strdup(red_ctx->error);
1064        } else {
1065            error_msg = (char *) malloc(64);
1066            if (error_msg != NULL) {
1067                sprintf(error_msg, "%d", ret);
1068            }
1069        }
1070        set_error_info(info, (const char *) error_msg, ret, error_info);
1071    }
1072
1073    free_view_reducer_ctx(red_ctx);
1074
1075    return ret;
1076}
1077
1078
1079LIBCOUCHSTORE_API
1080couchstore_error_t couchstore_cleanup_view_group(view_group_info_t *info,
1081                                                 uint64_t *header_pos,
1082                                                 uint64_t *purge_count,
1083                                                 view_error_t *error_info)
1084{
1085    couchstore_error_t ret;
1086    tree_file index_file;
1087    index_header_t *header = NULL;
1088    node_pointer *id_root = NULL;
1089    node_pointer **view_roots = NULL;
1090    view_purger_ctx_t purge_ctx;
1091    bitmap_t bm_cleanup;
1092    int i;
1093
1094    memset(&bm_cleanup, 0, sizeof(bm_cleanup));
1095    error_info->view_name = NULL;
1096    error_info->error_msg = NULL;
1097    index_file.handle = NULL;
1098    index_file.ops = NULL;
1099    index_file.path = NULL;
1100
1101    view_roots = (node_pointer **) calloc(
1102        info->num_btrees, sizeof(node_pointer *));
1103    if (view_roots == NULL) {
1104        return COUCHSTORE_ERROR_ALLOC_FAIL;
1105    }
1106
1107    /* Read info from current index viewgroup file */
1108    ret = open_view_group_file(info->filepath,
1109                               COUCHSTORE_OPEN_FLAG_RDONLY,
1110                               &info->file);
1111    if (ret != COUCHSTORE_SUCCESS) {
1112        goto cleanup;
1113    }
1114
1115    ret = read_view_group_header(info, &header);
1116    if (ret != COUCHSTORE_SUCCESS) {
1117        goto cleanup;
1118    }
1119    assert(info->num_btrees == header->num_views);
1120
1121    /* Setup purger context */
1122    purge_ctx.count = 0;
1123    purge_ctx.cbitmask = header->cleanup_bitmask;
1124
1125    ret = open_view_group_file(info->filepath,
1126                               0,
1127                               &index_file);
1128    if (ret != COUCHSTORE_SUCCESS) {
1129        goto cleanup;
1130    }
1131
1132    index_file.pos = index_file.ops->goto_eof(&index_file.lastError,
1133                                                         index_file.handle);
1134
1135    /* Cleanup id_bree */
1136    ret = cleanup_id_btree(&index_file, header->id_btree_state, &id_root,
1137                                                                &purge_ctx,
1138                                                                error_info);
1139    if (ret != COUCHSTORE_SUCCESS) {
1140        goto cleanup;
1141    }
1142
1143    if (header->id_btree_state != id_root) {
1144        free(header->id_btree_state);
1145    }
1146
1147    header->id_btree_state = id_root;
1148    view_id_bitmask(id_root, &bm_cleanup);
1149    id_root = NULL;
1150
1151    /* Cleanup all view btrees */
1152    for (i = 0; i < info->num_btrees; ++i) {
1153        ret = cleanup_view_btree(&index_file,
1154                                 (node_pointer *) header->view_states[i],
1155                                 &info->view_infos.btree[i],
1156                                 &view_roots[i],
1157                                 &purge_ctx,
1158                                 error_info);
1159
1160        if (ret != COUCHSTORE_SUCCESS) {
1161            goto cleanup;
1162        }
1163
1164        if (header->view_states[i] != view_roots[i]) {
1165            free(header->view_states[i]);
1166        }
1167
1168        header->view_states[i] = view_roots[i];
1169        view_bitmask(view_roots[i], &bm_cleanup);
1170        view_roots[i] = NULL;
1171    }
1172
1173    /* Set resulting cleanup bitmask */
1174    /* TODO: This code can be removed, if we do not plan for cleanup STOP command */
1175    intersect_bitmaps(&bm_cleanup, &purge_ctx.cbitmask);
1176    header->cleanup_bitmask = bm_cleanup;
1177
1178    /* Update header with new btree infos */
1179    ret = write_view_group_header(&index_file, header_pos, header);
1180    if (ret != COUCHSTORE_SUCCESS) {
1181        goto cleanup;
1182    }
1183
1184    *purge_count = purge_ctx.count;
1185    ret = COUCHSTORE_SUCCESS;
1186
1187cleanup:
1188    free_index_header(header);
1189    close_view_group_file(info);
1190    tree_file_close(&index_file);
1191    free(id_root);
1192    if (view_roots != NULL) {
1193        for (i = 0; i < info->num_btrees; ++i) {
1194            free(view_roots[i]);
1195        }
1196        free(view_roots);
1197    }
1198
1199    return ret;
1200}
1201
1202static couchstore_error_t update_btree(const char *source_file,
1203                                       tree_file *dest_file,
1204                                       const node_pointer *root,
1205                                       size_t batch_size,
1206                                       compare_info *cmp,
1207                                       reduce_fn reduce_fun,
1208                                       reduce_fn rereduce_fun,
1209                                       purge_kv_fn purge_kv,
1210                                       purge_kp_fn purge_kp,
1211                                       view_reducer_ctx_t *red_ctx,
1212                                       view_purger_ctx_t *purge_ctx,
1213                                       uint64_t *inserted,
1214                                       uint64_t *removed,
1215                                       node_pointer **out_root)
1216{
1217    couchstore_error_t ret = COUCHSTORE_SUCCESS;
1218    couchfile_modify_request rq;
1219    node_pointer *newroot = (node_pointer *) root;
1220    arena *transient_arena = new_arena(0);
1221    FILE *f = NULL;
1222    couchfile_modify_action *actions = NULL;
1223    sized_buf *keybufs = NULL, *valbufs = NULL;
1224    size_t bufsize = 0;
1225    int last_record = 0;
1226    bitmap_t empty_bm;
1227    int max_actions = MAX_ACTIONS_SIZE /
1228                (sizeof(couchfile_modify_action) + 2 * sizeof(sized_buf));
1229
1230    memset(&empty_bm, 0, sizeof(empty_bm));
1231
1232    if (transient_arena == NULL) {
1233        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1234        goto cleanup;
1235    }
1236
1237
1238    actions = (couchfile_modify_action *) calloc(
1239                                            max_actions,
1240                                            sizeof(couchfile_modify_action));
1241    if (actions == NULL) {
1242        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1243        goto cleanup;
1244    }
1245
1246    keybufs = (sized_buf *) calloc(max_actions, sizeof(sized_buf));
1247    if (keybufs == NULL) {
1248        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1249        goto cleanup;
1250    }
1251
1252    valbufs = (sized_buf *) calloc(max_actions, sizeof(sized_buf));
1253    if (valbufs == NULL) {
1254        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1255        goto cleanup;
1256    }
1257
1258    rq.cmp = *cmp;
1259    rq.file = dest_file;
1260    rq.actions = actions;
1261    rq.num_actions = 0;
1262    rq.reduce = reduce_fun;
1263    rq.rereduce = rereduce_fun;
1264    rq.compacting = 0;
1265    rq.kv_chunk_threshold = VIEW_KV_CHUNK_THRESHOLD;
1266    rq.kp_chunk_threshold = VIEW_KP_CHUNK_THRESHOLD;
1267    rq.purge_kp = purge_kp;
1268    rq.purge_kv = purge_kv;
1269    rq.guided_purge_ctx = purge_ctx;
1270    rq.enable_purging = 1;
1271    rq.user_reduce_ctx = red_ctx;
1272
1273    /* If cleanup bitmask is empty, no need to try purging */
1274    if (is_equal_bitmap(&empty_bm, &purge_ctx->cbitmask)) {
1275        rq.enable_purging = 0;
1276    }
1277
1278    f = fopen(source_file, "rb");
1279    if (f == NULL) {
1280        ret = COUCHSTORE_ERROR_OPEN_FILE;
1281        goto cleanup;
1282    }
1283
1284    while (!last_record) {
1285        int read_ret;
1286        uint8_t op;
1287
1288        read_ret = read_record(f, transient_arena, &keybufs[rq.num_actions],
1289                                                   &valbufs[rq.num_actions],
1290                                                   &op);
1291        if (read_ret == 0) {
1292            last_record = 1;
1293            goto flush;
1294        } else if (read_ret < 0) {
1295            ret = (couchstore_error_t) read_ret;
1296            goto cleanup;
1297        }
1298
1299        /* Add action */
1300        actions[rq.num_actions].type = op;
1301        actions[rq.num_actions].key = &keybufs[rq.num_actions];
1302        actions[rq.num_actions].value.data = &valbufs[rq.num_actions];
1303
1304        if (inserted && op == ACTION_INSERT) {
1305            (*inserted)++;
1306        } else if (removed && op == ACTION_REMOVE) {
1307            (*removed)++;
1308        }
1309
1310        bufsize += keybufs[rq.num_actions].size +
1311                   valbufs[rq.num_actions].size +
1312                   sizeof(uint8_t);
1313        rq.num_actions++;
1314
1315flush:
1316        if (rq.num_actions && (last_record || bufsize > batch_size ||
1317                               rq.num_actions == max_actions)) {
1318            newroot = modify_btree(&rq, newroot, &ret);
1319            if (ret != COUCHSTORE_SUCCESS) {
1320                goto cleanup;
1321            }
1322
1323            rq.num_actions = 0;
1324            bufsize = 0;
1325            arena_free_all(transient_arena);
1326        }
1327    }
1328
1329    *out_root = newroot;
1330
1331cleanup:
1332    free(actions);
1333    free(keybufs);
1334    free(valbufs);
1335
1336    if (f != NULL) {
1337        fclose(f);
1338    }
1339
1340    if (transient_arena != NULL) {
1341        delete_arena(transient_arena);
1342    }
1343
1344    return ret;
1345}
1346
1347static couchstore_error_t update_id_btree(const char *source_file,
1348                                         tree_file *dest_file,
1349                                         const node_pointer *root,
1350                                         size_t batch_size,
1351                                         view_purger_ctx_t *purge_ctx,
1352                                         uint64_t *inserted,
1353                                         uint64_t *removed,
1354                                         node_pointer **out_root)
1355{
1356    couchstore_error_t ret;
1357    compare_info cmp;
1358
1359    cmp.compare = id_btree_cmp;
1360
1361    ret = update_btree(source_file,
1362                      dest_file,
1363                      root,
1364                      batch_size,
1365                      &cmp,
1366                      view_id_btree_reduce,
1367                      view_id_btree_rereduce,
1368                      view_id_btree_purge_kv,
1369                      view_id_btree_purge_kp,
1370                      NULL,
1371                      purge_ctx,
1372                      inserted,
1373                      removed,
1374                      out_root);
1375
1376    return ret;
1377}
1378
1379
1380static couchstore_error_t update_view_btree(const char *source_file,
1381                                            const view_btree_info_t *info,
1382                                            tree_file *dest_file,
1383                                            const node_pointer *root,
1384                                            size_t batch_size,
1385                                            view_purger_ctx_t *purge_ctx,
1386                                            uint64_t *inserted,
1387                                            uint64_t *removed,
1388                                            node_pointer **out_root,
1389                                            view_error_t *error_info)
1390{
1391    couchstore_error_t ret = COUCHSTORE_SUCCESS;
1392    compare_info cmp;
1393    view_reducer_ctx_t *red_ctx = NULL;
1394    char *error_msg = NULL;
1395
1396    cmp.compare = view_btree_cmp;
1397    red_ctx = make_view_reducer_ctx(info->reducers,
1398                                    info->num_reducers,
1399                                    &error_msg);
1400    if (red_ctx == NULL) {
1401        set_error_info(info, (const char *) error_msg, ret, error_info);
1402        return COUCHSTORE_ERROR_REDUCER_FAILURE;
1403    }
1404
1405    ret = update_btree(source_file,
1406                      dest_file,
1407                      root,
1408                      batch_size,
1409                      &cmp,
1410                      view_btree_reduce,
1411                      view_btree_rereduce,
1412                      view_btree_purge_kv,
1413                      view_btree_purge_kp,
1414                      red_ctx,
1415                      purge_ctx,
1416                      inserted,
1417                      removed,
1418                      out_root);
1419
1420    if (ret != COUCHSTORE_SUCCESS) {
1421        set_error_info(info, red_ctx->error, ret, error_info);
1422    }
1423
1424    free_view_reducer_ctx(red_ctx);
1425
1426    return ret;
1427}
1428
1429LIBCOUCHSTORE_API
1430couchstore_error_t couchstore_update_view_group(view_group_info_t *info,
1431                                               const char *id_records_file,
1432                                               const char *kv_records_files[],
1433                                               size_t batch_size,
1434                                               const sized_buf *header_buf,
1435                                               int is_sorted,
1436                                               const char *tmp_dir,
1437                                               view_group_update_stats_t *stats,
1438                                               sized_buf *header_outbuf,
1439                                               view_error_t *error_info)
1440{
1441    couchstore_error_t ret;
1442    tree_file index_file;
1443    index_header_t *header = NULL;
1444    node_pointer *id_root = NULL;
1445    node_pointer **view_roots = NULL;
1446    view_purger_ctx_t purge_ctx;
1447    bitmap_t bm_cleanup;
1448    int i;
1449
1450    ret = decode_index_header(header_buf->buf, header_buf->size, &header);
1451    if (ret < 0) {
1452        goto cleanup;
1453    }
1454
1455    memset(&bm_cleanup, 0, sizeof(bm_cleanup));
1456
1457    error_info->view_name = NULL;
1458    error_info->error_msg = NULL;
1459    index_file.handle = NULL;
1460    index_file.ops = NULL;
1461    index_file.path = NULL;
1462
1463    view_roots = (node_pointer **) calloc(info->num_btrees,
1464                                          sizeof(node_pointer *));
1465    if (view_roots == NULL) {
1466        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1467        goto cleanup;
1468    }
1469
1470    /* Spatial views use the native updater only for sorting the ID b-tree.
1471     * Before starting it, the references to the spatial view index files
1472     * get removed, hence the number of trees in `info` don't match the number
1473     * in the header */
1474    if (info->type != VIEW_INDEX_TYPE_SPATIAL) {
1475        assert(info->num_btrees == header->num_views);
1476    }
1477
1478    /* Setup purger context */
1479    purge_ctx.count = 0;
1480    purge_ctx.cbitmask = header->cleanup_bitmask;
1481
1482    ret = open_view_group_file(info->filepath,
1483                               0,
1484                               &index_file);
1485    if (ret != COUCHSTORE_SUCCESS) {
1486        goto cleanup;
1487    }
1488
1489    /* Move pos to end of file */
1490    index_file.pos = index_file.ops->goto_eof(&index_file.lastError,
1491                                              index_file.handle);
1492
1493    if (!is_sorted) {
1494        ret = (couchstore_error_t) sort_view_ids_ops_file(id_records_file, tmp_dir);
1495        if (ret != COUCHSTORE_SUCCESS) {
1496            char buf[1024];
1497            snprintf(buf, sizeof(buf),
1498                    "Error sorting records file: %s", id_records_file);
1499            error_info->error_msg = strdup(buf);
1500            error_info->view_name = (const char *) strdup("id_btree");
1501            goto cleanup;
1502        }
1503    }
1504
1505    ret = update_id_btree(id_records_file, &index_file,
1506                                           header->id_btree_state,
1507                                           batch_size,
1508                                           &purge_ctx,
1509                                           &stats->ids_inserted,
1510                                           &stats->ids_removed,
1511                                           &id_root);
1512    if (ret != COUCHSTORE_SUCCESS) {
1513        goto cleanup;
1514    }
1515
1516
1517    if (header->id_btree_state != id_root) {
1518        free(header->id_btree_state);
1519    }
1520
1521    header->id_btree_state = id_root;
1522    view_id_bitmask(id_root, &bm_cleanup);
1523    id_root = NULL;
1524
1525    for (i = 0; i < info->num_btrees; ++i) {
1526        if (!is_sorted) {
1527            ret = (couchstore_error_t) sort_view_kvs_ops_file(kv_records_files[i], tmp_dir);
1528            if (ret != COUCHSTORE_SUCCESS) {
1529                char buf[1024];
1530                snprintf(buf, sizeof(buf),
1531                        "Error sorting records file: %s", kv_records_files[i]);
1532                set_error_info(&info->view_infos.btree[i], buf, ret, error_info);
1533                goto cleanup;
1534            }
1535        }
1536
1537        ret = update_view_btree(kv_records_files[i],
1538                                &info->view_infos.btree[i],
1539                                &index_file,
1540                                header->view_states[i],
1541                                batch_size,
1542                                &purge_ctx,
1543                                &stats->kvs_inserted,
1544                                &stats->kvs_removed,
1545                                &view_roots[i],
1546                                error_info);
1547
1548        if (ret != COUCHSTORE_SUCCESS) {
1549            goto cleanup;
1550        }
1551
1552        if (header->view_states[i] != view_roots[i]) {
1553            free(header->view_states[i]);
1554        }
1555
1556        header->view_states[i] = view_roots[i];
1557        view_bitmask(view_roots[i], &bm_cleanup);
1558        view_roots[i] = NULL;
1559    }
1560
1561    /* Set resulting cleanup bitmask */
1562    intersect_bitmaps(&bm_cleanup, &purge_ctx.cbitmask);
1563    header->cleanup_bitmask = bm_cleanup;
1564    stats->purged = purge_ctx.count;
1565
1566    ret = encode_index_header(header, &header_outbuf->buf, &header_outbuf->size);
1567    if (ret != COUCHSTORE_SUCCESS) {
1568        goto cleanup;
1569    }
1570
1571    ret = COUCHSTORE_SUCCESS;
1572
1573cleanup:
1574    free_index_header(header);
1575    close_view_group_file(info);
1576    tree_file_close(&index_file);
1577    free(id_root);
1578    if (view_roots != NULL) {
1579        for (i = 0; i < info->num_btrees; ++i) {
1580            free(view_roots[i]);
1581        }
1582        free(view_roots);
1583    }
1584
1585    return ret;
1586}
1587
1588/* Add the kv pair to modify result */
1589static couchstore_error_t compact_view_fetchcb(couchfile_lookup_request *rq,
1590                                        const sized_buf *k,
1591                                        const sized_buf *v)
1592{
1593    int ret;
1594    sized_buf *k_c, *v_c;
1595    view_compact_ctx_t *ctx = (view_compact_ctx_t *) rq->callback_ctx;
1596    compactor_stats_t *stats = ctx->stats;
1597
1598    if (k == NULL || v == NULL) {
1599        return COUCHSTORE_ERROR_READ;
1600    }
1601
1602    if (ctx->filter_fun) {
1603        ret = ctx->filter_fun(k, v, ctx->filterbm);
1604        if (ret < 0) {
1605            return (couchstore_error_t) ret;
1606        } else if (ret) {
1607            return COUCHSTORE_SUCCESS;
1608        }
1609    }
1610
1611    k_c = arena_copy_buf(ctx->transient_arena, k);
1612    v_c = arena_copy_buf(ctx->transient_arena, v);
1613    ret = mr_push_item(k_c, v_c, ctx->mr);
1614    if (ret != COUCHSTORE_SUCCESS) {
1615        return ret;
1616    }
1617
1618    if (stats) {
1619        stats->inserted++;
1620        if (stats->update_fun) {
1621            stats->update_fun(stats->freq, stats->inserted);
1622        }
1623    }
1624
1625    if (ctx->mr->count == 0) {
1626        arena_free_all(ctx->transient_arena);
1627    }
1628
1629    return (couchstore_error_t) ret;
1630}
1631
1632static couchstore_error_t compact_btree(tree_file *source,
1633                                 tree_file *target,
1634                                 const node_pointer *root,
1635                                 compare_info *cmp,
1636                                 reduce_fn reduce_fun,
1637                                 reduce_fn rereduce_fun,
1638                                 compact_filter_fn filter_fun,
1639                                 view_reducer_ctx_t *red_ctx,
1640                                 const bitmap_t *filterbm,
1641                                 compactor_stats_t *stats,
1642                                 node_pointer **out_root)
1643{
1644    couchstore_error_t ret = COUCHSTORE_SUCCESS;
1645    arena *transient_arena;
1646    arena *persistent_arena;
1647    couchfile_modify_result *modify_result;
1648    couchfile_lookup_request lookup_rq;
1649    view_compact_ctx_t compact_ctx;
1650    sized_buf nullkey = {NULL, 0};
1651    sized_buf *lowkeys = &nullkey;
1652
1653    if (!root) {
1654        return COUCHSTORE_SUCCESS;
1655    }
1656
1657    transient_arena = new_arena(0);
1658    persistent_arena = new_arena(0);
1659
1660    if (transient_arena == NULL || persistent_arena == NULL) {
1661        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1662        goto cleanup;
1663    }
1664
1665    /* Create new btree on new file */
1666    modify_result = new_btree_modres(persistent_arena,
1667                          transient_arena,
1668                          target,
1669                          cmp,
1670                          reduce_fun,
1671                          rereduce_fun,
1672                          red_ctx,
1673                          VIEW_KV_CHUNK_THRESHOLD + (VIEW_KV_CHUNK_THRESHOLD / 3),
1674                          VIEW_KP_CHUNK_THRESHOLD + (VIEW_KP_CHUNK_THRESHOLD / 3));
1675    if (modify_result == NULL) {
1676        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1677        goto cleanup;
1678    }
1679
1680    compact_ctx.filter_fun = NULL;
1681    compact_ctx.mr = modify_result;
1682    compact_ctx.transient_arena = transient_arena;
1683    compact_ctx.stats = stats;
1684
1685    if (filterbm) {
1686        compact_ctx.filterbm = filterbm;
1687        compact_ctx.filter_fun = filter_fun;
1688    }
1689
1690    lookup_rq.cmp.compare = cmp->compare;
1691    lookup_rq.file = source;
1692    lookup_rq.num_keys = 1;
1693    lookup_rq.keys = &lowkeys;
1694    lookup_rq.callback_ctx = &compact_ctx;
1695    lookup_rq.fetch_callback = compact_view_fetchcb;
1696    lookup_rq.node_callback = NULL;
1697    lookup_rq.fold = 1;
1698
1699    ret = btree_lookup(&lookup_rq, root->pointer);
1700    if (ret != COUCHSTORE_SUCCESS) {
1701        goto cleanup;
1702    }
1703
1704    *out_root = complete_new_btree(modify_result, &ret);
1705
1706cleanup:
1707    if (transient_arena != NULL) {
1708        delete_arena(transient_arena);
1709    }
1710
1711    if (persistent_arena != NULL) {
1712        delete_arena(persistent_arena);
1713    }
1714
1715    return ret;
1716}
1717
1718static couchstore_error_t compact_id_btree(tree_file *source,
1719                                    tree_file *target,
1720                                    const node_pointer *root,
1721                                    const bitmap_t *filterbm,
1722                                    compactor_stats_t *stats,
1723                                    node_pointer **out_root)
1724{
1725    couchstore_error_t ret;
1726    compare_info cmp;
1727
1728    cmp.compare = id_btree_cmp;
1729
1730    ret = compact_btree(source,
1731                        target,
1732                        root,
1733                        &cmp,
1734                        view_id_btree_reduce,
1735                        view_id_btree_rereduce,
1736                        view_id_btree_filter,
1737                        NULL,
1738                        filterbm,
1739                        stats,
1740                        out_root);
1741
1742    return ret;
1743}
1744
1745static couchstore_error_t compact_view_btree(tree_file *source,
1746                                      tree_file *target,
1747                                      const view_btree_info_t *info,
1748                                      const node_pointer *root,
1749                                      const bitmap_t *filterbm,
1750                                      compactor_stats_t *stats,
1751                                      node_pointer **out_root,
1752                                      view_error_t *error_info)
1753{
1754    couchstore_error_t ret = COUCHSTORE_SUCCESS;
1755    compare_info cmp;
1756    view_reducer_ctx_t *red_ctx = NULL;
1757    char *error_msg = NULL;
1758
1759    cmp.compare = view_btree_cmp;
1760    red_ctx = make_view_reducer_ctx(info->reducers,
1761                                    info->num_reducers,
1762                                    &error_msg);
1763    if (red_ctx == NULL) {
1764        set_error_info(info, (const char *) error_msg, ret, error_info);
1765        return COUCHSTORE_ERROR_REDUCER_FAILURE;
1766    }
1767
1768    ret = compact_btree(source,
1769                        target,
1770                        root,
1771                        &cmp,
1772                        view_btree_reduce,
1773                        view_btree_rereduce,
1774                        view_btree_filter,
1775                        red_ctx,
1776                        filterbm,
1777                        stats,
1778                        out_root);
1779
1780    if (ret != COUCHSTORE_SUCCESS) {
1781        set_error_info(info, red_ctx->error, ret, error_info);
1782    }
1783
1784    free_view_reducer_ctx(red_ctx);
1785
1786    return ret;
1787}
1788
1789LIBCOUCHSTORE_API
1790couchstore_error_t couchstore_compact_view_group(view_group_info_t *info,
1791                                                 const char *target_file,
1792                                                 const sized_buf *header_buf,
1793                                                 compactor_stats_t *stats,
1794                                                 sized_buf *header_outbuf,
1795                                                 view_error_t *error_info)
1796{
1797    couchstore_error_t ret;
1798    tree_file index_file;
1799    tree_file compact_file;
1800    index_header_t *header = NULL;
1801    node_pointer *id_root = NULL;
1802    node_pointer **view_roots = NULL;
1803    bitmap_t *filterbm = NULL;
1804    bitmap_t emptybm;
1805    int i;
1806
1807    memset(&emptybm, 0, sizeof(bitmap_t));
1808    error_info->view_name = NULL;
1809    error_info->error_msg = NULL;
1810    index_file.handle = NULL;
1811    index_file.ops = NULL;
1812    index_file.path = NULL;
1813    compact_file.handle = NULL;
1814    compact_file.ops = NULL;
1815    compact_file.path = NULL;
1816
1817    ret = decode_index_header(header_buf->buf, header_buf->size, &header);
1818    if (ret < 0) {
1819        goto cleanup;
1820    }
1821
1822    /* Set filter bitmask if required */
1823    if (!is_equal_bitmap(&emptybm, &header->cleanup_bitmask)) {
1824        filterbm = &header->cleanup_bitmask;
1825    }
1826
1827    view_roots = (node_pointer **) calloc(info->num_btrees,
1828                                          sizeof(node_pointer *));
1829    if (view_roots == NULL) {
1830        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1831        goto cleanup;
1832    }
1833
1834    assert(info->num_btrees == header->num_views);
1835
1836    ret = open_view_group_file(info->filepath,
1837                               COUCHSTORE_OPEN_FLAG_RDONLY,
1838                               &index_file);
1839    if (ret != COUCHSTORE_SUCCESS) {
1840        goto cleanup;
1841    }
1842
1843    /*
1844     * Open target file for compaction
1845     * Expects that caller created the target file
1846     */
1847    ret = open_view_group_file(target_file, 0, &compact_file);
1848    if (ret != COUCHSTORE_SUCCESS) {
1849        goto cleanup;
1850    }
1851
1852    compact_file.pos = compact_file.ops->goto_eof(&compact_file.lastError,
1853                                                  compact_file.handle);
1854    ret = compact_id_btree(&index_file, &compact_file,
1855                                        header->id_btree_state,
1856                                        filterbm,
1857                                        stats,
1858                                        &id_root);
1859    if (ret != COUCHSTORE_SUCCESS) {
1860        goto cleanup;
1861    }
1862
1863    free(header->id_btree_state);
1864    header->id_btree_state = id_root;
1865    id_root = NULL;
1866
1867    for (i = 0; i < info->num_btrees; ++i) {
1868        switch(info->type) {
1869        case VIEW_INDEX_TYPE_MAPREDUCE:
1870            ret = compact_view_btree(&index_file,
1871                                     &compact_file,
1872                                     &info->view_infos.btree[i],
1873                                     header->view_states[i],
1874                                     filterbm,
1875                                     stats,
1876                                     &view_roots[i],
1877                                     error_info);
1878            break;
1879        case VIEW_INDEX_TYPE_SPATIAL:
1880            ret = compact_view_spatial(&index_file,
1881                                       &compact_file,
1882                                       &info->view_infos.spatial[i],
1883                                       header->view_states[i],
1884                                       filterbm,
1885                                       stats,
1886                                       &view_roots[i],
1887                                       error_info);
1888            break;
1889        }
1890        if (ret != COUCHSTORE_SUCCESS) {
1891            goto cleanup;
1892        }
1893
1894        free(header->view_states[i]);
1895        header->view_states[i] = view_roots[i];
1896        view_roots[i] = NULL;
1897    }
1898
1899    header->cleanup_bitmask = emptybm;
1900    ret = encode_index_header(header, &header_outbuf->buf, &header_outbuf->size);
1901    if (ret != COUCHSTORE_SUCCESS) {
1902        goto cleanup;
1903    }
1904
1905    ret = COUCHSTORE_SUCCESS;
1906
1907cleanup:
1908    free_index_header(header);
1909    close_view_group_file(info);
1910    tree_file_close(&index_file);
1911    tree_file_close(&compact_file);
1912    free(id_root);
1913    if (view_roots != NULL) {
1914        for (i = 0; i < info->num_btrees; ++i) {
1915            free(view_roots[i]);
1916        }
1917        free(view_roots);
1918    }
1919
1920    return ret;
1921}
1922
1923
1924/*
1925 * For initial spatial build, feed the spatial builder as soon as
1926 * sorted records are available.
1927 */
1928static file_merger_error_t build_spatial_record_callback(void *buf, void *ctx)
1929{
1930    int ret;
1931    sized_buf *k, *v;
1932    view_file_merge_record_t *rec = (view_file_merge_record_t *) buf;
1933    view_file_merge_ctx_t *merge_ctx = (view_file_merge_ctx_t *) ctx;
1934    view_spatial_builder_ctx_t *build_ctx =
1935            (view_spatial_builder_ctx_t *) merge_ctx->user_ctx;
1936    sized_buf src_k, src_v;
1937
1938    src_k.size = rec->ksize;
1939    src_k.buf = VIEW_RECORD_KEY(rec);
1940    src_v.size = rec->vsize;
1941    src_v.buf = VIEW_RECORD_VAL(rec);
1942
1943    k = arena_copy_buf(build_ctx->transient_arena, &src_k);
1944    v = arena_copy_buf(build_ctx->transient_arena, &src_v);
1945    ret = spatial_push_item(k, v, build_ctx->modify_result);
1946
1947    if (ret != COUCHSTORE_SUCCESS) {
1948        return ret;
1949    }
1950
1951    if (build_ctx->modify_result->count == 0) {
1952        arena_free_all(build_ctx->transient_arena);
1953    }
1954
1955    return ret;
1956}
1957
1958
1959static couchstore_error_t build_view_spatial(const char *source_file,
1960                                             const view_spatial_info_t *info,
1961                                             tree_file *dest_file,
1962                                             const char *tmpdir,
1963                                             node_pointer **out_root,
1964                                             view_error_t *error_info)
1965{
1966    couchstore_error_t ret;
1967    compare_info cmp;
1968    char buf[64];
1969    size_t len = 0;
1970
1971    /* cmp.compare is only needed when you read a b-tree node or modify a
1972     * b-tree node (btree_modify:modify_node()). We don't do either in the
1973     * spatial view. */
1974    cmp.compare = NULL;
1975    ret = build_spatial(source_file,
1976                        dest_file,
1977                        &cmp,
1978                        view_spatial_reduce,
1979                        /* Use the reduce function also for the re-reduce */
1980                        view_spatial_reduce,
1981                        info->dimension,
1982                        info->mbb,
1983                        tmpdir,
1984                        sort_spatial_kvs_file,
1985                        out_root);
1986
1987    if (ret != COUCHSTORE_SUCCESS) {
1988        len = snprintf(buf, sizeof(buf), "ret = %d", ret);
1989        buf[len] = '\0';
1990        if (len) {
1991            error_info->error_msg = (const char *) strdup(buf);
1992        }
1993    }
1994
1995    return ret;
1996}
1997
1998
1999static couchstore_error_t build_spatial(const char *source_file,
2000                                        tree_file *dest_file,
2001                                        compare_info *cmp,
2002                                        reduce_fn reduce_fun,
2003                                        reduce_fn rereduce_fun,
2004                                        const uint16_t dimension,
2005                                        const double *mbb,
2006                                        const char *tmpdir,
2007                                        sort_record_fn sort_fun,
2008                                        node_pointer **out_root)
2009{
2010    couchstore_error_t ret = COUCHSTORE_SUCCESS;
2011    arena *transient_arena = new_arena(0);
2012    arena *persistent_arena = new_arena(0);
2013    couchfile_modify_result *mr;
2014    view_spatial_builder_ctx_t build_ctx;
2015
2016    if (transient_arena == NULL || persistent_arena == NULL) {
2017        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
2018        goto out;
2019    }
2020
2021    mr = new_btree_modres(persistent_arena,
2022                          transient_arena,
2023                          dest_file,
2024                          cmp,
2025                          reduce_fun,
2026                          rereduce_fun,
2027                          NULL,
2028                          /* Normally the nodes are flushed to disk when 2/3 of
2029                           * the threshold is reached. In order to have a
2030                           * higher fill grade, we add 1/3 */
2031                          VIEW_KV_CHUNK_THRESHOLD + (VIEW_KV_CHUNK_THRESHOLD / 3),
2032                          VIEW_KP_CHUNK_THRESHOLD + (VIEW_KP_CHUNK_THRESHOLD / 3));
2033    if (mr == NULL) {
2034        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
2035        goto out;
2036    }
2037
2038    build_ctx.transient_arena = transient_arena;
2039    build_ctx.modify_result = mr;
2040    build_ctx.scale_factor = spatial_scale_factor(mbb, dimension,
2041                                                  ZCODE_MAX_VALUE);
2042
2043    ret = (couchstore_error_t) sort_fun(source_file,
2044                                        tmpdir,
2045                                        build_spatial_record_callback,
2046                                        &build_ctx);
2047    free_spatial_scale_factor(build_ctx.scale_factor);
2048    if (ret != COUCHSTORE_SUCCESS) {
2049        goto out;
2050    }
2051
2052    *out_root = complete_new_spatial(mr, &ret);
2053    if (ret != COUCHSTORE_SUCCESS) {
2054        goto out;
2055    }
2056
2057    /* Don't care about success/failure. Erlang side will eventually delete it. */
2058    remove(source_file);
2059
2060out:
2061    if (transient_arena != NULL) {
2062        delete_arena(transient_arena);
2063    }
2064
2065    return ret;
2066}
2067
2068/* Add the kv pair to modify result
2069 * The difference to the mapreduce views is the call to `spatial_push_item` */
2070static couchstore_error_t compact_spatial_fetchcb(couchfile_lookup_request *rq,
2071                                                  const sized_buf *k,
2072                                                  const sized_buf *v)
2073{
2074    int ret;
2075    sized_buf *k_c, *v_c;
2076    view_compact_ctx_t *ctx = (view_compact_ctx_t *) rq->callback_ctx;
2077    compactor_stats_t *stats = ctx->stats;
2078
2079    if (k == NULL || v == NULL) {
2080        return COUCHSTORE_ERROR_READ;
2081    }
2082
2083    if (ctx->filter_fun) {
2084        ret = ctx->filter_fun(k, v, ctx->filterbm);
2085        if (ret < 0) {
2086            return (couchstore_error_t) ret;
2087        } else if (ret) {
2088            return COUCHSTORE_SUCCESS;
2089        }
2090    }
2091
2092    k_c = arena_copy_buf(ctx->transient_arena, k);
2093    v_c = arena_copy_buf(ctx->transient_arena, v);
2094    ret = spatial_push_item(k_c, v_c, ctx->mr);
2095    if (ret != COUCHSTORE_SUCCESS) {
2096        return ret;
2097    }
2098
2099    if (stats) {
2100        stats->inserted++;
2101        if (stats->update_fun) {
2102            stats->update_fun(stats->freq, stats->inserted);
2103        }
2104    }
2105
2106    if (ctx->mr->count == 0) {
2107        arena_free_all(ctx->transient_arena);
2108    }
2109
2110    return (couchstore_error_t) ret;
2111}
2112
2113static couchstore_error_t compact_view_spatial(tree_file *source,
2114                                               tree_file *target,
2115                                               const view_spatial_info_t *info,
2116                                               const node_pointer *root,
2117                                               const bitmap_t *filterbm,
2118                                               compactor_stats_t *stats,
2119                                               node_pointer **out_root,
2120                                               view_error_t *error_info)
2121{
2122    couchstore_error_t ret;
2123    compare_info cmp;
2124
2125    cmp.compare = view_btree_cmp;
2126    ret = compact_spatial(source,
2127                          target,
2128                          root,
2129                          &cmp,
2130                          view_spatial_reduce,
2131                          /* Use the reduce function also for the re-reduce */
2132                          view_spatial_reduce,
2133                          view_spatial_filter,
2134                          filterbm,
2135                          stats,
2136                          out_root);
2137
2138    return ret;
2139}
2140
2141static couchstore_error_t compact_spatial(tree_file *source,
2142                                          tree_file *target,
2143                                          const node_pointer *root,
2144                                          compare_info *cmp,
2145                                          reduce_fn reduce_fun,
2146                                          reduce_fn rereduce_fun,
2147                                          compact_filter_fn filter_fun,
2148                                          const bitmap_t *filterbm,
2149                                          compactor_stats_t *stats,
2150                                          node_pointer **out_root)
2151{
2152    couchstore_error_t ret = COUCHSTORE_SUCCESS;
2153    arena *transient_arena = new_arena(0);
2154    arena *persistent_arena = new_arena(0);
2155    couchfile_modify_result *modify_result;
2156    couchfile_lookup_request lookup_rq;
2157    view_compact_ctx_t compact_ctx;
2158    sized_buf nullkey = {NULL, 0};
2159    sized_buf *lowkeys = &nullkey;
2160
2161    if (!root) {
2162        return COUCHSTORE_SUCCESS;
2163    }
2164
2165    if (transient_arena == NULL || persistent_arena == NULL) {
2166        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
2167        goto cleanup;
2168    }
2169
2170    /* Create new spatial index on new file */
2171    modify_result = new_btree_modres(persistent_arena,
2172                                     transient_arena,
2173                                     target,
2174                                     cmp,
2175                                     reduce_fun,
2176                                     rereduce_fun,
2177                                     NULL,
2178                                     /* Normally the nodes are flushed to disk
2179                                      * when 2/3 of the threshold is reached.
2180                                      * In order to have a higher fill grade,
2181                                      * we add 1/3 */
2182                                     VIEW_KV_CHUNK_THRESHOLD +
2183                                         (VIEW_KV_CHUNK_THRESHOLD / 3),
2184                                     VIEW_KP_CHUNK_THRESHOLD +
2185                                         (VIEW_KP_CHUNK_THRESHOLD / 3));
2186    if (modify_result == NULL) {
2187        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
2188        goto cleanup;
2189    }
2190
2191    compact_ctx.filter_fun = NULL;
2192    compact_ctx.mr = modify_result;
2193    compact_ctx.transient_arena = transient_arena;
2194    compact_ctx.stats = stats;
2195
2196    if (filterbm) {
2197        compact_ctx.filterbm = filterbm;
2198        compact_ctx.filter_fun = filter_fun;
2199    }
2200
2201    lookup_rq.cmp.compare = cmp->compare;
2202    lookup_rq.file = source;
2203    lookup_rq.num_keys = 1;
2204    lookup_rq.keys = &lowkeys;
2205    lookup_rq.callback_ctx = &compact_ctx;
2206    lookup_rq.fetch_callback = compact_spatial_fetchcb;
2207    lookup_rq.node_callback = NULL;
2208    lookup_rq.fold = 1;
2209
2210    ret = btree_lookup(&lookup_rq, root->pointer);
2211    if (ret != COUCHSTORE_SUCCESS) {
2212        goto cleanup;
2213    }
2214
2215    *out_root = complete_new_spatial(modify_result, &ret);
2216
2217cleanup:
2218    if (transient_arena != NULL) {
2219        delete_arena(transient_arena);
2220    }
2221
2222    if (persistent_arena != NULL) {
2223        delete_arena(persistent_arena);
2224    }
2225
2226    return ret;
2227}
2228