xref: /6.6.0/couchstore/src/views/view_group.cc (revision cf120ada)
1/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2
3/**
4 * @copyright 2013 Couchbase, Inc.
5 *
6 * @author Filipe Manana  <filipe@couchbase.com>
7 *
8 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
9 * use this file except in compliance with the License. You may obtain a copy of
10 * the License at
11 *
12 *  http://www.apache.org/licenses/LICENSE-2.0
13 *
14 * Unless required by applicable law or agreed to in writing, software
15 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17 * License for the specific language governing permissions and limitations under
18 * the License.
19 **/
20
21#include "couchstore_config.h"
22#include <errno.h>
23#include <stdlib.h>
24#include <string.h>
25
26#include "view_group.h"
27#include "reducers.h"
28#include "reductions.h"
29#include "values.h"
30#include "purgers.h"
31#include "util.h"
32#include "file_sorter.h"
33#include "spatial.h"
34#include "../arena.h"
35#include "../couch_btree.h"
36#include "../internal.h"
37#include "../util.h"
38#include <platform/cb_malloc.h>
39#include <platform/cbassert.h>
40
41#define VIEW_KV_CHUNK_THRESHOLD (7 * 1024)
42#define VIEW_KP_CHUNK_THRESHOLD (6 * 1024)
43#define MAX_ACTIONS_SIZE        (2 * 1024 * 1024)
44
45static couchstore_error_t read_btree_info(view_group_info_t *info,
46                                          FILE *in_stream,
47                                          FILE *error_stream);
48
49static couchstore_error_t read_spatial_info(view_group_info_t *info,
50                                            FILE *in_stream,
51                                            FILE *error_stream);
52
53static couchstore_error_t build_btree(const char *source_file,
54                                      tree_file *dest_file,
55                                      compare_info *cmp,
56                                      reduce_fn reduce_fun,
57                                      reduce_fn rereduce_fun,
58                                      const char *tmpdir,
59                                      sort_record_fn sort_fun,
60                                      void *reduce_ctx,
61                                      node_pointer **out_root);
62
63static couchstore_error_t build_id_btree(const char *source_file,
64                                         tree_file *dest_file,
65                                         const char *tmpdir,
66                                         node_pointer **out_root);
67
68static couchstore_error_t build_view_btree(const char *source_file,
69                                           const view_btree_info_t *info,
70                                           tree_file *dest_file,
71                                           const char *tmpdir,
72                                           node_pointer **out_root,
73                                           view_error_t *error_info);
74
75static void close_view_group_file(view_group_info_t *info);
76
77static int read_record(FILE *f, arena *a, sized_buf *k, sized_buf *v,
78                                                        uint8_t *op);
79
80extern "C" {
81    int view_btree_cmp(const sized_buf *key1, const sized_buf *key2);
82}
83static couchstore_error_t update_btree(const char *source_file,
84                                       tree_file *dest_file,
85                                       const node_pointer *root,
86                                       size_t batch_size,
87                                       compare_info *cmp,
88                                       reduce_fn reduce_fun,
89                                       reduce_fn rereduce_fun,
90                                       purge_kv_fn purge_kv,
91                                       purge_kp_fn purge_kp,
92                                       view_reducer_ctx_t *red_ctx,
93                                       view_purger_ctx_t *purge_ctx,
94                                       uint64_t *inserted,
95                                       uint64_t *removed,
96                                       node_pointer **out_root);
97
98static couchstore_error_t update_id_btree(const char *source_file,
99                                         tree_file *dest_file,
100                                         const node_pointer *root,
101                                         size_t batch_size,
102                                         view_purger_ctx_t *purge_ctx,
103                                         uint64_t *inserted,
104                                         uint64_t *removed,
105                                         node_pointer **out_root);
106
107static couchstore_error_t update_view_btree(const char *source_file,
108                                            const view_btree_info_t *info,
109                                            tree_file *dest_file,
110                                            const node_pointer *root,
111                                            size_t batch_size,
112                                            view_purger_ctx_t *purge_ctx,
113                                            uint64_t *inserted,
114                                            uint64_t *removed,
115                                            node_pointer **out_root,
116                                            view_error_t *error_info);
117
118static couchstore_error_t compact_view_fetchcb(couchfile_lookup_request *rq,
119                                        const sized_buf *k,
120                                        const sized_buf *v);
121
122static couchstore_error_t compact_btree(tree_file *source,
123                                 tree_file *target,
124                                 const node_pointer *root,
125                                 compare_info *cmp,
126                                 reduce_fn reduce_fun,
127                                 reduce_fn rereduce_fun,
128                                 compact_filter_fn filter_fun,
129                                 view_reducer_ctx_t *red_ctx,
130                                 const bitmap_t *filterbm,
131                                 compactor_stats_t *stats,
132                                 node_pointer **out_root);
133
134static couchstore_error_t compact_id_btree(tree_file *source,
135                                    tree_file *target,
136                                    const node_pointer *root,
137                                    const bitmap_t *filterbm,
138                                    compactor_stats_t *stats,
139                                    node_pointer **out_root);
140
141static couchstore_error_t compact_view_btree(tree_file *source,
142                                      tree_file *target,
143                                      const view_btree_info_t *info,
144                                      const node_pointer *root,
145                                      const bitmap_t *filterbm,
146                                      compactor_stats_t *stats,
147                                      node_pointer **out_root,
148                                      view_error_t *error_info);
149
150/* Some preparations before building the actual spatial index */
151static couchstore_error_t build_view_spatial(const char *source_file,
152                                             const view_spatial_info_t *info,
153                                             tree_file *dest_file,
154                                             const char *tmpdir,
155                                             node_pointer **out_root,
156                                             view_error_t *error_info);
157
158/* Build the actual spatial index */
159static couchstore_error_t build_spatial(const char *source_file,
160                                        tree_file *dest_file,
161                                        compare_info *cmp,
162                                        reduce_fn reduce_fun,
163                                        reduce_fn rereduce_fun,
164                                        const uint16_t dimension,
165                                        const double *mbb,
166                                        const char *tmpdir,
167                                        sort_record_fn sort_fun,
168                                        node_pointer **out_root);
169
170/* Callback for every item that got fetched from the original view */
171static couchstore_error_t compact_spatial_fetchcb(couchfile_lookup_request *rq,
172                                                  const sized_buf *k,
173                                                  const sized_buf *v);
174
175/* Some preparations before compacting the actual spatial index */
176static couchstore_error_t compact_view_spatial(tree_file *source,
177                                               tree_file *target,
178                                               const view_spatial_info_t *info,
179                                               const node_pointer *root,
180                                               const bitmap_t *filterbm,
181                                               compactor_stats_t *stats,
182                                               node_pointer **out_root,
183                                               view_error_t *error_info);
184
185/* Compact the actual spatial index */
186static couchstore_error_t compact_spatial(tree_file *source,
187                                          tree_file *target,
188                                          const node_pointer *root,
189                                          compare_info *cmp,
190                                          reduce_fn reduce_fun,
191                                          reduce_fn rereduce_fun,
192                                          compact_filter_fn filter_fun,
193                                          const bitmap_t *filterbm,
194                                          compactor_stats_t *stats,
195                                          node_pointer **out_root);
196
197view_group_info_t *couchstore_read_view_group_info(FILE *in_stream,
198                                                   FILE *error_stream)
199{
200    view_group_info_t *info;
201    char buf[4096];
202    char *dup;
203    couchstore_error_t ret;
204    uint64_t type;
205
206    info = (view_group_info_t *) cb_calloc(1, sizeof(*info));
207    if (info == NULL) {
208        fprintf(error_stream, "Memory allocation failure\n");
209        goto out_error;
210    }
211
212    type = couchstore_read_int(in_stream, buf, sizeof(buf), &ret);
213    if (ret != COUCHSTORE_SUCCESS) {
214        fprintf(stderr, "Error reading view file type\n");
215        goto out_error;
216    }
217    switch (type) {
218    case VIEW_INDEX_TYPE_MAPREDUCE:
219        info->type = VIEW_INDEX_TYPE_MAPREDUCE;
220        break;
221    case VIEW_INDEX_TYPE_SPATIAL:
222        info->type = VIEW_INDEX_TYPE_SPATIAL;
223        break;
224    default:
225        fprintf(stderr, "Invalid view file type: %" PRIu64 "\n", type);
226        goto out_error;
227    }
228
229    if (couchstore_read_line(in_stream, buf, sizeof(buf)) != buf) {
230        fprintf(stderr, "Error reading source index file path\n");
231        goto out_error;
232    }
233    dup = cb_strdup(buf);
234    if (dup == NULL) {
235        fprintf(error_stream, "Memory allocation failure\n");
236        goto out_error;
237    }
238    info->filepath = (const char *) dup;
239
240    info->header_pos = couchstore_read_int(in_stream, buf,
241                                                      sizeof(buf),
242                                                      &ret);
243    if (ret != COUCHSTORE_SUCCESS) {
244        fprintf(error_stream, "Error reading header position\n");
245        goto out_error;
246    }
247
248    info->num_btrees = couchstore_read_int(in_stream, buf,
249                                                      sizeof(buf),
250                                                      &ret);
251    if (ret != COUCHSTORE_SUCCESS) {
252        fprintf(error_stream, "Error reading number of btrees\n");
253        goto out_error;
254    }
255
256    switch (info->type) {
257    case VIEW_INDEX_TYPE_MAPREDUCE:
258        ret = read_btree_info(info, in_stream, error_stream);
259        break;
260    case VIEW_INDEX_TYPE_SPATIAL:
261        ret = read_spatial_info(info, in_stream, error_stream);
262        break;
263    }
264    if (ret != COUCHSTORE_SUCCESS) {
265        goto out_error;
266    }
267
268    return info;
269
270out_error:
271    couchstore_free_view_group_info(info);
272
273    return NULL;
274}
275
276
277/* Read in the information about the mapreduce view indexes */
278static couchstore_error_t read_btree_info(view_group_info_t *info,
279                                          FILE *in_stream,
280                                          FILE *error_stream)
281{
282    char buf[4096];
283    char *dup;
284    int i, j;
285    int reduce_len;
286    couchstore_error_t ret;
287
288    info->view_infos.btree = (view_btree_info_t *)
289        cb_calloc(info->num_btrees, sizeof(view_btree_info_t));
290    if (info->view_infos.btree == NULL) {
291        fprintf(error_stream, "Memory allocation failure on btree infos\n");
292        info->num_btrees = 0;
293        return COUCHSTORE_ERROR_ALLOC_FAIL;
294    }
295
296    for (i = 0; i < info->num_btrees; ++i) {
297        view_btree_info_t *bti = &info->view_infos.btree[i];
298
299        bti->view_id = i;
300        bti->num_reducers = couchstore_read_int(in_stream,
301                                                buf,
302                                                sizeof(buf),
303                                                &ret);
304
305        if (ret != COUCHSTORE_SUCCESS) {
306            fprintf(error_stream,
307                    "Error reading number of reducers for btree %d\n", i);
308            return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
309        }
310
311        bti->names = (const char **) cb_calloc(bti->num_reducers, sizeof(char *));
312        if (bti->names == NULL) {
313            fprintf(error_stream,
314                    "Memory allocation failure on btree %d reducer names\n",
315                    i);
316            bti->num_reducers = 0;
317            return COUCHSTORE_ERROR_ALLOC_FAIL;
318        }
319
320        bti->reducers = (const char **) cb_calloc(bti->num_reducers, sizeof(char *));
321        if (bti->reducers == NULL) {
322            fprintf(error_stream,
323                    "Memory allocation failure on btree %d reducers\n", i);
324            bti->num_reducers = 0;
325            cb_free(bti->names);
326            return COUCHSTORE_ERROR_ALLOC_FAIL;
327        }
328
329        for (j = 0; j < bti->num_reducers; ++j) {
330            if (couchstore_read_line(in_stream, buf, sizeof(buf)) != buf) {
331                fprintf(error_stream,
332                        "Error reading btree %d view %d name\n", i, j);
333                return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
334            }
335            dup = cb_strdup(buf);
336            if (dup == NULL) {
337                fprintf(error_stream,
338                        "Memory allocation failure on btree %d "
339                        "view %d name\n", i, j);
340                return COUCHSTORE_ERROR_ALLOC_FAIL;
341            }
342            bti->names[j] = (const char *) dup;
343
344            reduce_len = couchstore_read_int(in_stream,
345                                             buf,
346                                             sizeof(buf),
347                                             &ret);
348            if (ret != COUCHSTORE_SUCCESS) {
349                fprintf(error_stream,
350                        "Error reading btree %d view %d "
351                        "reduce function size\n", i, j);
352                return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
353            }
354
355            dup = (char *) cb_malloc(reduce_len + 1);
356            if (dup == NULL) {
357                fprintf(error_stream,
358                        "Memory allocation failure on btree %d "
359                        "view %d reducer\n", i, j);
360                return COUCHSTORE_ERROR_ALLOC_FAIL;
361            }
362
363            if (fread(dup, reduce_len, 1, in_stream) != 1) {
364                fprintf(error_stream,
365                        "Error reading btree %d view %d reducer\n", i, j);
366                cb_free(dup);
367                return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
368            }
369            dup[reduce_len] = '\0';
370            bti->reducers[j] = (const char *) dup;
371        }
372    }
373    return COUCHSTORE_SUCCESS;
374}
375
376
377/* Read in the information about the spatial view indexes */
378static couchstore_error_t read_spatial_info(view_group_info_t *info,
379                                            FILE *in_stream,
380                                            FILE *error_stream)
381{
382    char buf[24];
383    int i;
384    couchstore_error_t ret;
385
386    info->view_infos.spatial = (view_spatial_info_t *)
387        cb_calloc(info->num_btrees, sizeof(view_spatial_info_t));
388    if (info->view_infos.spatial == NULL) {
389        fprintf(error_stream, "Memory allocation failure on spatial infos\n");
390        info->num_btrees = 0;
391        return COUCHSTORE_ERROR_ALLOC_FAIL;
392    }
393
394    for (i = 0; i < info->num_btrees; ++i) {
395        view_spatial_info_t *si = &info->view_infos.spatial[i];
396
397        si->dimension = couchstore_read_int(in_stream, buf, sizeof(buf), &ret);
398
399        if (ret != COUCHSTORE_SUCCESS) {
400            fprintf(error_stream,
401                    "Error reading the dimension of spatial view %d\n", i);
402            return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
403        }
404
405        /* If the dimension is 0, no index exists or hasn't been built yet */
406        if (si->dimension == 0) {
407            continue;
408        }
409
410        si->mbb = (double *) cb_calloc(si->dimension * 2, sizeof(double));
411        if (si->mbb == NULL) {
412            fprintf(error_stream,
413                    "Memory allocation failure on spatial view %d\n", i);
414            si->dimension = 0;
415            return COUCHSTORE_ERROR_ALLOC_FAIL;
416        }
417
418        if (fread(si->mbb, sizeof(double), si->dimension * 2, in_stream) <
419                si->dimension * 2) {
420            fprintf(error_stream, "Error reading mbb of view %d\n", i);
421            return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
422        }
423    }
424    return COUCHSTORE_SUCCESS;
425}
426
427void couchstore_free_view_group_info(view_group_info_t *info)
428{
429    int i, j;
430
431    if (info == NULL)
432        return;
433
434    close_view_group_file(info);
435
436    switch (info->type) {
437    case VIEW_INDEX_TYPE_MAPREDUCE:
438        for (i = 0; i < info->num_btrees; ++i) {
439            view_btree_info_t vi = info->view_infos.btree[i];
440
441            for (j = 0; j < vi.num_reducers; ++j) {
442                cb_free((void *) vi.names[j]);
443                cb_free((void *) vi.reducers[j]);
444            }
445            cb_free(vi.names);
446            cb_free(vi.reducers);
447        }
448        cb_free(info->view_infos.btree);
449        break;
450    case VIEW_INDEX_TYPE_SPATIAL:
451        for (i = 0; i < info->num_btrees; ++i) {
452            cb_free((void *) info->view_infos.spatial[i].mbb);
453        }
454        cb_free(info->view_infos.spatial);
455        break;
456    }
457    cb_free((void *) info->filepath);
458    cb_free(info);
459}
460
461
462static void close_view_group_file(view_group_info_t *info)
463{
464    if (info->file.ops != NULL) {
465        info->file.ops->close(&info->file.lastError, info->file.handle);
466        info->file.ops->destructor(info->file.handle);
467        info->file.ops = NULL;
468        info->file.handle = NULL;
469    }
470    cb_free((void *) info->file.path);
471    info->file.path = NULL;
472}
473
474
475couchstore_error_t open_view_group_file(const char *path,
476                                               couchstore_open_flags open_flags,
477                                               tree_file *file)
478{
479    couchstore_error_t ret;
480    FileOpsInterface* file_ops = couchstore_get_default_file_ops();
481    int flags = 0;
482
483    if ((open_flags & COUCHSTORE_OPEN_FLAG_RDONLY) &&
484        (open_flags & COUCHSTORE_OPEN_FLAG_CREATE)) {
485        return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
486    }
487
488    if (open_flags & COUCHSTORE_OPEN_FLAG_RDONLY) {
489        flags |= O_RDONLY;
490    } else {
491        flags |= O_RDWR;
492    }
493
494    if (open_flags & COUCHSTORE_OPEN_FLAG_CREATE) {
495        flags |= O_CREAT;
496    }
497
498    ret = tree_file_open(file, path, flags, CRC32, file_ops,
499                         tree_file_options() /* Enable buffered IO by default. */);
500
501    return ret;
502}
503
504couchstore_error_t couchstore_build_view_group(view_group_info_t *info,
505                                               const char *id_records_file,
506                                               const char *kv_records_files[],
507                                               const char *dst_file,
508                                               const char *tmpdir,
509                                               uint64_t *header_pos,
510                                               view_error_t *error_info)
511{
512    couchstore_error_t ret;
513    tree_file index_file;
514    index_header_t *header = NULL;
515    node_pointer *id_root = NULL;
516    node_pointer **view_roots = NULL;
517    int i;
518
519    error_info->view_name = NULL;
520    error_info->error_msg = NULL;
521    index_file.handle = NULL;
522    index_file.ops = NULL;
523    index_file.path = NULL;
524
525    view_roots = (node_pointer **) cb_calloc(
526        info->num_btrees, sizeof(node_pointer *));
527    if (view_roots == NULL) {
528        return COUCHSTORE_ERROR_ALLOC_FAIL;
529    }
530
531    ret = open_view_group_file(info->filepath,
532                               COUCHSTORE_OPEN_FLAG_RDONLY,
533                               &info->file);
534    if (ret != COUCHSTORE_SUCCESS) {
535        goto out;
536    }
537
538    ret = read_view_group_header(info, &header);
539    if (ret != COUCHSTORE_SUCCESS) {
540        goto out;
541    }
542    cb_assert(info->num_btrees == header->num_views);
543
544    ret = open_view_group_file(dst_file,
545                               COUCHSTORE_OPEN_FLAG_CREATE,
546                               &index_file);
547    if (ret != COUCHSTORE_SUCCESS) {
548        goto out;
549    }
550
551    ret = build_id_btree(id_records_file, &index_file, tmpdir, &id_root);
552    if (ret != COUCHSTORE_SUCCESS) {
553        goto out;
554    }
555
556    cb_free(header->id_btree_state);
557    header->id_btree_state = id_root;
558    id_root = NULL;
559
560    for (i = 0; i < info->num_btrees; ++i) {
561        switch(info->type) {
562        case VIEW_INDEX_TYPE_MAPREDUCE:
563            ret = build_view_btree(kv_records_files[i],
564                                   &info->view_infos.btree[i],
565                                   &index_file,
566                                   tmpdir,
567                                   &view_roots[i],
568                                   error_info);
569            break;
570        case VIEW_INDEX_TYPE_SPATIAL:
571            ret = build_view_spatial(kv_records_files[i],
572                                     &info->view_infos.spatial[i],
573                                     &index_file,
574                                     tmpdir,
575                                     &view_roots[i],
576                                     error_info);
577            break;
578        }
579        if (ret != COUCHSTORE_SUCCESS) {
580            goto out;
581        }
582
583        cb_free(header->view_states[i]);
584        header->view_states[i] = view_roots[i];
585        view_roots[i] = NULL;
586    }
587
588    ret = write_view_group_header(&index_file, header_pos, header);
589    if (ret != COUCHSTORE_SUCCESS) {
590        goto out;
591    }
592
593    ret = COUCHSTORE_SUCCESS;
594
595out:
596    free_index_header(header);
597    close_view_group_file(info);
598    tree_file_close(&index_file);
599    cb_free(id_root);
600    if (view_roots != NULL) {
601        for (i = 0; i < info->num_btrees; ++i) {
602            cb_free(view_roots[i]);
603        }
604        cb_free(view_roots);
605    }
606
607    return ret;
608}
609
610
611/*
612 * Similar to util.c:read_view_record(), but it uses arena allocator, which is
613 * required for the existing semantics/api of btree bottom-up build in
614 * src/btree_modify.cc.
615 */
616static int read_record(FILE *f, arena *a, sized_buf *k, sized_buf *v,
617                                                        uint8_t *op)
618{
619    uint16_t klen;
620    uint8_t oplen = 0;
621    uint32_t vlen, len;
622
623    if (fread(&len, sizeof(len), 1, f) != 1) {
624        if (feof(f)) {
625            return 0;
626        } else {
627            return COUCHSTORE_ERROR_READ;
628        }
629    }
630
631    /* For incremental update, read optype */
632    if (op != NULL) {
633        if (fread(op, sizeof(uint8_t), 1, f) != 1) {
634            return COUCHSTORE_ERROR_READ;
635        }
636
637        oplen = 1;
638    }
639
640    if (fread(&klen, sizeof(klen), 1, f) != 1) {
641        return COUCHSTORE_ERROR_READ;
642    }
643
644    klen = ntohs(klen);
645    vlen = len - sizeof(klen) - klen - oplen;
646
647    k->size = klen;
648    k->buf = (char *) arena_alloc(a, k->size);
649    if (k->buf == NULL) {
650        return COUCHSTORE_ERROR_ALLOC_FAIL;
651    }
652
653    v->size = vlen;
654    /* Handle zero len vals */
655    if (v->size) {
656        v->buf = (char *) arena_alloc(a, v->size);
657        if (v->buf == NULL) {
658            return COUCHSTORE_ERROR_ALLOC_FAIL;
659        }
660    } else {
661        v->buf = NULL;
662    }
663
664    if (fread(k->buf, k->size, 1, f) != 1) {
665        return FILE_MERGER_ERROR_FILE_READ;
666    }
667
668    if (v->size && fread(v->buf, v->size, 1, f) != 1) {
669        return FILE_MERGER_ERROR_FILE_READ;
670    }
671
672    return len;
673}
674
675
676static int id_btree_cmp(const sized_buf *key1, const sized_buf *key2)
677{
678    return view_id_cmp(key1, key2, NULL);
679}
680
681
682int view_btree_cmp(const sized_buf *key1, const sized_buf *key2)
683{
684    return view_key_cmp(key1, key2, NULL);
685}
686
687
688/*
689 * For initial btree build, feed the btree builder as soon as
690 * sorted records are available.
691 */
692static file_merger_error_t build_btree_record_callback(void *buf, void *ctx)
693{
694    int ret;
695    sized_buf *k, *v;
696    view_file_merge_record_t *rec = (view_file_merge_record_t *) buf;
697    view_file_merge_ctx_t *merge_ctx = (view_file_merge_ctx_t *) ctx;
698    view_btree_builder_ctx_t *build_ctx =
699                    (view_btree_builder_ctx_t *) merge_ctx->user_ctx;
700    sized_buf src_k, src_v;
701
702    src_k.size = rec->ksize;
703    src_k.buf = VIEW_RECORD_KEY(rec);
704    src_v.size = rec->vsize;
705    src_v.buf = VIEW_RECORD_VAL(rec);
706
707    k = arena_copy_buf(build_ctx->transient_arena, &src_k);
708    v = arena_copy_buf(build_ctx->transient_arena, &src_v);
709    ret = mr_push_item(k, v, build_ctx->modify_result);
710
711    if (ret != COUCHSTORE_SUCCESS) {
712        return static_cast<file_merger_error_t>(ret);
713    }
714
715    if (build_ctx->modify_result->count == 0) {
716        arena_free_all(build_ctx->transient_arena);
717    }
718
719    return static_cast<file_merger_error_t>(ret);
720}
721
722
723static couchstore_error_t build_btree(const char *source_file,
724                                      tree_file *dest_file,
725                                      compare_info *cmp,
726                                      reduce_fn reduce_fun,
727                                      reduce_fn rereduce_fun,
728                                      const char *tmpdir,
729                                      sort_record_fn sort_fun,
730                                      void *reduce_ctx,
731                                      node_pointer **out_root)
732{
733    couchstore_error_t ret = COUCHSTORE_SUCCESS;
734    arena *transient_arena = new_arena(0);
735    arena *persistent_arena = new_arena(0);
736    couchfile_modify_result *mr;
737    view_btree_builder_ctx_t build_ctx;
738
739    if (transient_arena == NULL || persistent_arena == NULL) {
740        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
741        goto out;
742    }
743
744    mr = new_btree_modres(persistent_arena,
745                          transient_arena,
746                          dest_file,
747                          cmp,
748                          reduce_fun,
749                          rereduce_fun,
750                          reduce_ctx,
751                          VIEW_KV_CHUNK_THRESHOLD + (VIEW_KV_CHUNK_THRESHOLD / 3),
752                          VIEW_KP_CHUNK_THRESHOLD + (VIEW_KP_CHUNK_THRESHOLD / 3));
753    if (mr == NULL) {
754        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
755        goto out;
756    }
757
758    build_ctx.transient_arena = transient_arena;
759    build_ctx.modify_result = mr;
760
761    ret = (couchstore_error_t) sort_fun(source_file,
762                                        tmpdir,
763                                        build_btree_record_callback, &build_ctx);
764    if (ret != COUCHSTORE_SUCCESS) {
765        goto out;
766    }
767
768    *out_root = complete_new_btree(mr, &ret);
769    if (ret != COUCHSTORE_SUCCESS) {
770        goto out;
771    }
772
773    /* Don't care about success/failure. Erlang side will eventually delete it. */
774    remove(source_file);
775
776out:
777    if (transient_arena != NULL) {
778        delete_arena(transient_arena);
779    }
780    if (persistent_arena != NULL) {
781        delete_arena(persistent_arena);
782    }
783
784    return ret;
785}
786
787
788static couchstore_error_t build_id_btree(const char *source_file,
789                                         tree_file *dest_file,
790                                         const char *tmpdir,
791                                         node_pointer **out_root)
792{
793    couchstore_error_t ret;
794    compare_info cmp;
795
796    cmp.compare = id_btree_cmp;
797
798    ret = build_btree(source_file,
799                      dest_file,
800                      &cmp,
801                      view_id_btree_reduce,
802                      view_id_btree_rereduce,
803                      tmpdir,
804                      sort_view_ids_file,
805                      NULL,
806                      out_root);
807
808    return ret;
809}
810
811
812static couchstore_error_t build_view_btree(const char *source_file,
813                                           const view_btree_info_t *info,
814                                           tree_file *dest_file,
815                                           const char *tmpdir,
816                                           node_pointer **out_root,
817                                           view_error_t *error_info)
818{
819    couchstore_error_t ret = COUCHSTORE_SUCCESS;
820    compare_info cmp;
821    view_reducer_ctx_t *red_ctx = NULL;
822    char *error_msg = NULL;
823
824    cmp.compare = view_btree_cmp;
825    red_ctx = make_view_reducer_ctx(info->reducers,
826                                    info->num_reducers,
827                                    &error_msg);
828    if (red_ctx == NULL) {
829        set_error_info(info, (const char *) error_msg, ret, error_info);
830        cb_free(error_msg);
831        return COUCHSTORE_ERROR_REDUCER_FAILURE;
832    }
833
834    ret = build_btree(source_file,
835                      dest_file,
836                      &cmp,
837                      view_btree_reduce,
838                      view_btree_rereduce,
839                      tmpdir,
840                      sort_view_kvs_file,
841                      red_ctx,
842                      out_root);
843
844    if (ret != COUCHSTORE_SUCCESS) {
845        set_error_info(info, red_ctx->error, ret, error_info);
846    }
847
848    free_view_reducer_ctx(red_ctx);
849
850    return ret;
851}
852
853
854couchstore_error_t read_view_group_header(view_group_info_t *info,
855                                          index_header_t **header)
856{
857    couchstore_error_t ret;
858    cs_off_t pos = info->header_pos;
859    tree_file *file = &info->file;
860    char *header_buf = NULL;
861    int header_len;
862    char buf;
863
864    if (file->handle == NULL) {
865        return COUCHSTORE_ERROR_FILE_CLOSED;
866    }
867
868    if (info->file.ops->pread(NULL, file->handle, &buf, 1, pos) != 1) {
869        return COUCHSTORE_ERROR_READ;
870    }
871    if (buf == 0) {
872        return COUCHSTORE_ERROR_NO_HEADER;
873    } else if (buf != 1) {
874        return COUCHSTORE_ERROR_CORRUPT;
875    }
876
877    header_len = pread_header(file, pos, &header_buf, MAX_VIEW_HEADER_SIZE);
878    if (header_len < 0) {
879        return (couchstore_error_t) header_len;
880    }
881
882    ret = decode_index_header(header_buf, (size_t) header_len, header);
883    cb_free(header_buf);
884
885    return ret;
886}
887
888couchstore_error_t write_view_group_header(tree_file *file,
889                                           uint64_t *pos,
890                                           const index_header_t *header)
891{
892    couchstore_error_t ret;
893    sized_buf buf = { NULL, 0 };
894    cs_off_t p;
895
896    if (file->handle == NULL) {
897        return COUCHSTORE_ERROR_FILE_CLOSED;
898    }
899
900    ret = encode_index_header(header, &buf.buf, &buf.size);
901    if (ret != COUCHSTORE_SUCCESS) {
902        return ret;
903    }
904
905    ret = write_header(file, &buf, &p);
906    if (ret != COUCHSTORE_SUCCESS) {
907        goto out;
908    }
909
910    cb_assert(p >= 0);
911    *pos = (uint64_t) p;
912
913out:
914    cb_free(buf.buf);
915
916    return ret;
917}
918
919
920static couchstore_error_t view_id_bitmask(const node_pointer *root, bitmap_t *bm)
921{
922    view_id_btree_reduction_t *r = NULL;
923    couchstore_error_t errcode;
924    if (root == NULL) {
925        return COUCHSTORE_SUCCESS;
926    }
927
928    errcode = decode_view_id_btree_reduction(root->reduce_value.buf, &r);
929    if (errcode != COUCHSTORE_SUCCESS) {
930        goto cleanup;
931    }
932
933    union_bitmaps(bm, &r->partitions_bitmap);
934
935cleanup:
936    free_view_id_btree_reduction(r);
937    return errcode;
938}
939
940
941static couchstore_error_t view_bitmask(const node_pointer *root, bitmap_t *bm)
942{
943    view_btree_reduction_t *r = NULL;
944    couchstore_error_t errcode;
945    if (root == NULL) {
946        return COUCHSTORE_SUCCESS;
947    }
948
949    errcode = decode_view_btree_reduction(root->reduce_value.buf,
950                                          root->reduce_value.size, &r);
951    if (errcode != COUCHSTORE_SUCCESS) {
952        goto cleanup;
953    }
954
955    union_bitmaps(bm, &r->partitions_bitmap);
956
957cleanup:
958    free_view_btree_reduction(r);
959    return errcode;
960}
961
962
963static couchstore_error_t cleanup_btree(tree_file *file,
964                                        node_pointer *root,
965                                        compare_info *cmp,
966                                        reduce_fn reduce,
967                                        reduce_fn rereduce,
968                                        purge_kv_fn purge_kv,
969                                        purge_kp_fn purge_kp,
970                                        view_purger_ctx_t *purge_ctx,
971                                        view_reducer_ctx_t *red_ctx,
972                                        node_pointer **out_root)
973{
974    couchstore_error_t errcode;
975    couchfile_modify_request rq;
976
977    rq.cmp = *cmp;
978    rq.file = file;
979    rq.actions = NULL;
980    rq.num_actions = 0;
981    rq.reduce = reduce;
982    rq.rereduce = rereduce;
983    rq.compacting = 0;
984    rq.kv_chunk_threshold = VIEW_KV_CHUNK_THRESHOLD;
985    rq.kp_chunk_threshold = VIEW_KP_CHUNK_THRESHOLD;
986    rq.purge_kp = purge_kp;
987    rq.purge_kv = purge_kv;
988    rq.enable_purging = 1;
989    rq.guided_purge_ctx = purge_ctx;
990    rq.user_reduce_ctx = red_ctx;
991
992    *out_root = guided_purge_btree(&rq, root, &errcode);
993
994    return errcode;
995}
996
997static couchstore_error_t cleanup_id_btree(tree_file *file,
998                                           node_pointer *root,
999                                           node_pointer **out_root,
1000                                           view_purger_ctx_t *purge_ctx,
1001                                           view_error_t *error_info)
1002{
1003    couchstore_error_t ret;
1004    compare_info cmp;
1005
1006    cmp.compare = id_btree_cmp;
1007
1008    ret = cleanup_btree(file,
1009                        root,
1010                        &cmp,
1011                        view_id_btree_reduce,
1012                        view_id_btree_rereduce,
1013                        view_id_btree_purge_kv,
1014                        view_id_btree_purge_kp,
1015                        purge_ctx,
1016                        NULL,
1017                        out_root);
1018
1019    return ret;
1020}
1021
1022
1023static couchstore_error_t cleanup_view_btree(tree_file *file,
1024                                             node_pointer *root,
1025                                             const view_btree_info_t *info,
1026                                             node_pointer **out_root,
1027                                             view_purger_ctx_t *purge_ctx,
1028                                             view_error_t *error_info)
1029{
1030    couchstore_error_t ret = COUCHSTORE_SUCCESS;
1031    compare_info cmp;
1032    view_reducer_ctx_t *red_ctx = NULL;
1033    char *error_msg = NULL;
1034
1035    cmp.compare = view_btree_cmp;
1036    red_ctx = make_view_reducer_ctx(info->reducers,
1037                                    info->num_reducers,
1038                                    &error_msg);
1039    if (red_ctx == NULL) {
1040        set_error_info(info, (const char *) error_msg, ret, error_info);
1041        cb_free(error_msg);
1042        return COUCHSTORE_ERROR_REDUCER_FAILURE;
1043    }
1044
1045    ret = cleanup_btree(file,
1046                        root,
1047                        &cmp,
1048                        view_btree_reduce,
1049                        view_btree_rereduce,
1050                        view_btree_purge_kv,
1051                        view_btree_purge_kp,
1052                        purge_ctx,
1053                        red_ctx,
1054                        out_root);
1055
1056    if (ret != COUCHSTORE_SUCCESS) {
1057        const char *error_msg = NULL;
1058        if (red_ctx->error != NULL) {
1059            error_msg = red_ctx->error;
1060        }
1061        set_error_info(info, (const char *) error_msg, ret, error_info);
1062    }
1063
1064    free_view_reducer_ctx(red_ctx);
1065
1066    return ret;
1067}
1068
1069couchstore_error_t couchstore_cleanup_view_group(view_group_info_t *info,
1070                                                 uint64_t *header_pos,
1071                                                 uint64_t *purge_count,
1072                                                 view_error_t *error_info)
1073{
1074    couchstore_error_t ret;
1075    tree_file index_file;
1076    index_header_t *header = NULL;
1077    node_pointer *id_root = NULL;
1078    node_pointer **view_roots = NULL;
1079    view_purger_ctx_t purge_ctx;
1080    bitmap_t bm_cleanup;
1081    int i;
1082
1083    memset(&bm_cleanup, 0, sizeof(bm_cleanup));
1084    error_info->view_name = NULL;
1085    error_info->error_msg = NULL;
1086    index_file.handle = NULL;
1087    index_file.ops = NULL;
1088    index_file.path = NULL;
1089
1090    view_roots = (node_pointer **) cb_calloc(
1091        info->num_btrees, sizeof(node_pointer *));
1092    if (view_roots == NULL) {
1093        return COUCHSTORE_ERROR_ALLOC_FAIL;
1094    }
1095
1096    /* Read info from current index viewgroup file */
1097    ret = open_view_group_file(info->filepath,
1098                               COUCHSTORE_OPEN_FLAG_RDONLY,
1099                               &info->file);
1100    if (ret != COUCHSTORE_SUCCESS) {
1101        goto cleanup;
1102    }
1103
1104    ret = read_view_group_header(info, &header);
1105    if (ret != COUCHSTORE_SUCCESS) {
1106        goto cleanup;
1107    }
1108    cb_assert(info->num_btrees == header->num_views);
1109
1110    /* Setup purger context */
1111    purge_ctx.count = 0;
1112    purge_ctx.cbitmask = header->cleanup_bitmask;
1113
1114    ret = open_view_group_file(info->filepath,
1115                               0,
1116                               &index_file);
1117    if (ret != COUCHSTORE_SUCCESS) {
1118        goto cleanup;
1119    }
1120
1121    index_file.pos = index_file.ops->goto_eof(&index_file.lastError,
1122                                                         index_file.handle);
1123
1124    /* Cleanup id_bree */
1125    ret = cleanup_id_btree(&index_file, header->id_btree_state, &id_root,
1126                                                                &purge_ctx,
1127                                                                error_info);
1128    if (ret != COUCHSTORE_SUCCESS) {
1129        goto cleanup;
1130    }
1131
1132    if (header->id_btree_state != id_root) {
1133        cb_free(header->id_btree_state);
1134    }
1135
1136    header->id_btree_state = id_root;
1137    view_id_bitmask(id_root, &bm_cleanup);
1138    id_root = NULL;
1139
1140    /* Cleanup all view btrees */
1141    for (i = 0; i < info->num_btrees; ++i) {
1142        ret = cleanup_view_btree(&index_file,
1143                                 (node_pointer *) header->view_states[i],
1144                                 &info->view_infos.btree[i],
1145                                 &view_roots[i],
1146                                 &purge_ctx,
1147                                 error_info);
1148
1149        if (ret != COUCHSTORE_SUCCESS) {
1150            goto cleanup;
1151        }
1152
1153        if (header->view_states[i] != view_roots[i]) {
1154            cb_free(header->view_states[i]);
1155        }
1156
1157        header->view_states[i] = view_roots[i];
1158        view_bitmask(view_roots[i], &bm_cleanup);
1159        view_roots[i] = NULL;
1160    }
1161
1162    /* Set resulting cleanup bitmask */
1163    /* TODO: This code can be removed, if we do not plan for cleanup STOP command */
1164    intersect_bitmaps(&bm_cleanup, &purge_ctx.cbitmask);
1165    header->cleanup_bitmask = bm_cleanup;
1166
1167    /* Update header with new btree infos */
1168    ret = write_view_group_header(&index_file, header_pos, header);
1169    if (ret != COUCHSTORE_SUCCESS) {
1170        goto cleanup;
1171    }
1172
1173    *purge_count = purge_ctx.count;
1174    ret = COUCHSTORE_SUCCESS;
1175
1176cleanup:
1177    free_index_header(header);
1178    close_view_group_file(info);
1179    tree_file_close(&index_file);
1180    cb_free(id_root);
1181    if (view_roots != NULL) {
1182        for (i = 0; i < info->num_btrees; ++i) {
1183            cb_free(view_roots[i]);
1184        }
1185        cb_free(view_roots);
1186    }
1187
1188    return ret;
1189}
1190
1191static couchstore_error_t update_btree(const char *source_file,
1192                                       tree_file *dest_file,
1193                                       const node_pointer *root,
1194                                       size_t batch_size,
1195                                       compare_info *cmp,
1196                                       reduce_fn reduce_fun,
1197                                       reduce_fn rereduce_fun,
1198                                       purge_kv_fn purge_kv,
1199                                       purge_kp_fn purge_kp,
1200                                       view_reducer_ctx_t *red_ctx,
1201                                       view_purger_ctx_t *purge_ctx,
1202                                       uint64_t *inserted,
1203                                       uint64_t *removed,
1204                                       node_pointer **out_root)
1205{
1206    couchstore_error_t ret = COUCHSTORE_SUCCESS;
1207    couchfile_modify_request rq;
1208    node_pointer *newroot = (node_pointer *) root;
1209    arena *transient_arena = new_arena(0);
1210    FILE *f = NULL;
1211    couchfile_modify_action *actions = NULL;
1212    sized_buf *keybufs = NULL, *valbufs = NULL;
1213    size_t bufsize = 0;
1214    int last_record = 0;
1215    bitmap_t empty_bm;
1216    int max_actions = MAX_ACTIONS_SIZE /
1217                (sizeof(couchfile_modify_action) + 2 * sizeof(sized_buf));
1218
1219    memset(&empty_bm, 0, sizeof(empty_bm));
1220
1221    if (transient_arena == NULL) {
1222        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1223        goto cleanup;
1224    }
1225
1226
1227    actions = (couchfile_modify_action *) cb_calloc(
1228                                            max_actions,
1229                                            sizeof(couchfile_modify_action));
1230    if (actions == NULL) {
1231        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1232        goto cleanup;
1233    }
1234
1235    keybufs = (sized_buf *) cb_calloc(max_actions, sizeof(sized_buf));
1236    if (keybufs == NULL) {
1237        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1238        goto cleanup;
1239    }
1240
1241    valbufs = (sized_buf *) cb_calloc(max_actions, sizeof(sized_buf));
1242    if (valbufs == NULL) {
1243        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1244        goto cleanup;
1245    }
1246
1247    rq.cmp = *cmp;
1248    rq.file = dest_file;
1249    rq.actions = actions;
1250    rq.num_actions = 0;
1251    rq.reduce = reduce_fun;
1252    rq.rereduce = rereduce_fun;
1253    rq.compacting = 0;
1254    rq.kv_chunk_threshold = VIEW_KV_CHUNK_THRESHOLD;
1255    rq.kp_chunk_threshold = VIEW_KP_CHUNK_THRESHOLD;
1256    rq.purge_kp = purge_kp;
1257    rq.purge_kv = purge_kv;
1258    rq.guided_purge_ctx = purge_ctx;
1259    rq.enable_purging = 1;
1260    rq.user_reduce_ctx = red_ctx;
1261
1262    /* If cleanup bitmask is empty, no need to try purging */
1263    if (is_equal_bitmap(&empty_bm, &purge_ctx->cbitmask)) {
1264        rq.enable_purging = 0;
1265    }
1266
1267    f = fopen(source_file, "rb");
1268    if (f == NULL) {
1269        fprintf(stderr, "Error while opening file: source file %s, errorcode %s\n",
1270                source_file, strerror(errno));
1271        ret = COUCHSTORE_ERROR_OPEN_FILE;
1272        goto cleanup;
1273    }
1274
1275    while (!last_record) {
1276        int read_ret;
1277        uint8_t op;
1278
1279        read_ret = read_record(f, transient_arena, &keybufs[rq.num_actions],
1280                                                   &valbufs[rq.num_actions],
1281                                                   &op);
1282        if (read_ret == 0) {
1283            last_record = 1;
1284            goto flush;
1285        } else if (read_ret < 0) {
1286            ret = (couchstore_error_t) read_ret;
1287            goto cleanup;
1288        }
1289
1290        /* Add action */
1291        actions[rq.num_actions].type = op;
1292        actions[rq.num_actions].key = &keybufs[rq.num_actions];
1293        actions[rq.num_actions].value.data = &valbufs[rq.num_actions];
1294
1295        if (inserted && op == ACTION_INSERT) {
1296            (*inserted)++;
1297        } else if (removed && op == ACTION_REMOVE) {
1298            (*removed)++;
1299        }
1300
1301        bufsize += keybufs[rq.num_actions].size +
1302                   valbufs[rq.num_actions].size +
1303                   sizeof(uint8_t);
1304        rq.num_actions++;
1305
1306flush:
1307        if (rq.num_actions && (last_record || bufsize > batch_size ||
1308                               rq.num_actions == max_actions)) {
1309            newroot = modify_btree(&rq, newroot, &ret);
1310            if (ret != COUCHSTORE_SUCCESS) {
1311                goto cleanup;
1312            }
1313
1314            rq.num_actions = 0;
1315            bufsize = 0;
1316            arena_free_all(transient_arena);
1317        }
1318    }
1319
1320    *out_root = newroot;
1321
1322cleanup:
1323    cb_free(actions);
1324    cb_free(keybufs);
1325    cb_free(valbufs);
1326
1327    if (f != NULL) {
1328        fclose(f);
1329    }
1330
1331    if (transient_arena != NULL) {
1332        delete_arena(transient_arena);
1333    }
1334
1335    return ret;
1336}
1337
1338static couchstore_error_t update_id_btree(const char *source_file,
1339                                         tree_file *dest_file,
1340                                         const node_pointer *root,
1341                                         size_t batch_size,
1342                                         view_purger_ctx_t *purge_ctx,
1343                                         uint64_t *inserted,
1344                                         uint64_t *removed,
1345                                         node_pointer **out_root)
1346{
1347    couchstore_error_t ret;
1348    compare_info cmp;
1349
1350    cmp.compare = id_btree_cmp;
1351
1352    ret = update_btree(source_file,
1353                      dest_file,
1354                      root,
1355                      batch_size,
1356                      &cmp,
1357                      view_id_btree_reduce,
1358                      view_id_btree_rereduce,
1359                      view_id_btree_purge_kv,
1360                      view_id_btree_purge_kp,
1361                      NULL,
1362                      purge_ctx,
1363                      inserted,
1364                      removed,
1365                      out_root);
1366
1367    return ret;
1368}
1369
1370
1371static couchstore_error_t update_view_btree(const char *source_file,
1372                                            const view_btree_info_t *info,
1373                                            tree_file *dest_file,
1374                                            const node_pointer *root,
1375                                            size_t batch_size,
1376                                            view_purger_ctx_t *purge_ctx,
1377                                            uint64_t *inserted,
1378                                            uint64_t *removed,
1379                                            node_pointer **out_root,
1380                                            view_error_t *error_info)
1381{
1382    couchstore_error_t ret = COUCHSTORE_SUCCESS;
1383    compare_info cmp;
1384    view_reducer_ctx_t *red_ctx = NULL;
1385    char *error_msg = NULL;
1386
1387    cmp.compare = view_btree_cmp;
1388    red_ctx = make_view_reducer_ctx(info->reducers,
1389                                    info->num_reducers,
1390                                    &error_msg);
1391    if (red_ctx == NULL) {
1392        set_error_info(info, (const char *) error_msg, ret, error_info);
1393        cb_free(error_msg);
1394        return COUCHSTORE_ERROR_REDUCER_FAILURE;
1395    }
1396
1397    ret = update_btree(source_file,
1398                      dest_file,
1399                      root,
1400                      batch_size,
1401                      &cmp,
1402                      view_btree_reduce,
1403                      view_btree_rereduce,
1404                      view_btree_purge_kv,
1405                      view_btree_purge_kp,
1406                      red_ctx,
1407                      purge_ctx,
1408                      inserted,
1409                      removed,
1410                      out_root);
1411
1412    if (ret != COUCHSTORE_SUCCESS) {
1413        set_error_info(info, red_ctx->error, ret, error_info);
1414    }
1415
1416    free_view_reducer_ctx(red_ctx);
1417
1418    return ret;
1419}
1420
1421couchstore_error_t couchstore_update_view_group(view_group_info_t *info,
1422                                               const char *id_records_file,
1423                                               const char *kv_records_files[],
1424                                               size_t batch_size,
1425                                               const sized_buf *header_buf,
1426                                               int is_sorted,
1427                                               const char *tmp_dir,
1428                                               view_group_update_stats_t *stats,
1429                                               sized_buf *header_outbuf,
1430                                               view_error_t *error_info)
1431{
1432    couchstore_error_t ret;
1433    tree_file index_file = {0, NULL, NULL, NULL, {-1}, CRC32};
1434    index_header_t *header = NULL;
1435    node_pointer *id_root = NULL;
1436    node_pointer **view_roots = NULL;
1437    view_purger_ctx_t purge_ctx;
1438    bitmap_t bm_cleanup;
1439    int i;
1440
1441    ret = decode_index_header(header_buf->buf, header_buf->size, &header);
1442    if (ret < 0) {
1443        goto cleanup;
1444    }
1445
1446    memset(&bm_cleanup, 0, sizeof(bm_cleanup));
1447
1448    error_info->view_name = NULL;
1449    error_info->error_msg = NULL;
1450
1451    view_roots = (node_pointer **) cb_calloc(info->num_btrees,
1452                                          sizeof(node_pointer *));
1453    if (view_roots == NULL) {
1454        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1455        goto cleanup;
1456    }
1457
1458    /* Spatial views use the native updater only for sorting the ID b-tree.
1459     * Before starting it, the references to the spatial view index files
1460     * get removed, hence the number of trees in `info` don't match the number
1461     * in the header */
1462    if (info->type != VIEW_INDEX_TYPE_SPATIAL) {
1463        cb_assert(info->num_btrees == header->num_views);
1464    }
1465
1466    /* Setup purger context */
1467    purge_ctx.count = 0;
1468    purge_ctx.cbitmask = header->cleanup_bitmask;
1469
1470    ret = open_view_group_file(info->filepath,
1471                               0,
1472                               &index_file);
1473    if (ret != COUCHSTORE_SUCCESS) {
1474        goto cleanup;
1475    }
1476
1477    /* Move pos to end of file */
1478    index_file.pos = index_file.ops->goto_eof(&index_file.lastError,
1479                                              index_file.handle);
1480
1481    if (!is_sorted) {
1482        ret = (couchstore_error_t) sort_view_ids_ops_file(id_records_file, tmp_dir);
1483        if (ret != COUCHSTORE_SUCCESS) {
1484            char error_msg[1024];
1485            int nw = snprintf(error_msg, sizeof(error_msg),
1486                              "Error sorting records file: %s",
1487                              id_records_file);
1488            if (nw > 0 && size_t(nw) < sizeof(error_msg)) {
1489                error_info->error_msg = cb_strdup(error_msg);
1490            } else {
1491                error_info->error_msg = cb_strdup("Error sorting records file");
1492            }
1493            error_info->idx_type = "MAPREDUCE";
1494            error_info->view_name = (const char *) cb_strdup("id_btree");
1495            goto cleanup;
1496        }
1497    }
1498
1499    ret = update_id_btree(id_records_file, &index_file,
1500                                           header->id_btree_state,
1501                                           batch_size,
1502                                           &purge_ctx,
1503                                           &stats->ids_inserted,
1504                                           &stats->ids_removed,
1505                                           &id_root);
1506    if (ret != COUCHSTORE_SUCCESS) {
1507        goto cleanup;
1508    }
1509
1510
1511    if (header->id_btree_state != id_root) {
1512        cb_free(header->id_btree_state);
1513    }
1514
1515    header->id_btree_state = id_root;
1516    view_id_bitmask(id_root, &bm_cleanup);
1517    id_root = NULL;
1518
1519    for (i = 0; i < info->num_btrees; ++i) {
1520        if (!is_sorted) {
1521            ret = (couchstore_error_t) sort_view_kvs_ops_file(kv_records_files[i], tmp_dir);
1522            if (ret != COUCHSTORE_SUCCESS) {
1523                const char* errmsg = "Error sorting records file";
1524                char error_msg[1024];
1525                int nw = snprintf(error_msg, sizeof(error_msg),
1526                                  "Error sorting records file: %s",
1527                                  kv_records_files[i]);
1528
1529                if (nw > 0 && size_t(nw) < sizeof(error_msg)) {
1530                    errmsg = error_msg;
1531                }
1532                set_error_info(&info->view_infos.btree[i], errmsg, ret,
1533                               error_info);
1534                goto cleanup;
1535            }
1536        }
1537
1538        ret = update_view_btree(kv_records_files[i],
1539                                &info->view_infos.btree[i],
1540                                &index_file,
1541                                header->view_states[i],
1542                                batch_size,
1543                                &purge_ctx,
1544                                &stats->kvs_inserted,
1545                                &stats->kvs_removed,
1546                                &view_roots[i],
1547                                error_info);
1548
1549        if (ret != COUCHSTORE_SUCCESS) {
1550            goto cleanup;
1551        }
1552
1553        if (header->view_states[i] != view_roots[i]) {
1554            cb_free(header->view_states[i]);
1555        }
1556
1557        header->view_states[i] = view_roots[i];
1558        view_bitmask(view_roots[i], &bm_cleanup);
1559        view_roots[i] = NULL;
1560    }
1561
1562    /* Set resulting cleanup bitmask */
1563    intersect_bitmaps(&bm_cleanup, &purge_ctx.cbitmask);
1564    header->cleanup_bitmask = bm_cleanup;
1565    stats->purged = purge_ctx.count;
1566
1567    ret = encode_index_header(header, &header_outbuf->buf, &header_outbuf->size);
1568    if (ret != COUCHSTORE_SUCCESS) {
1569        goto cleanup;
1570    }
1571
1572    ret = COUCHSTORE_SUCCESS;
1573
1574cleanup:
1575    free_index_header(header);
1576    close_view_group_file(info);
1577    tree_file_close(&index_file);
1578    cb_free(id_root);
1579    if (view_roots != NULL) {
1580        for (i = 0; i < info->num_btrees; ++i) {
1581            cb_free(view_roots[i]);
1582        }
1583        cb_free(view_roots);
1584    }
1585
1586    return ret;
1587}
1588
1589/* Add the kv pair to modify result */
1590static couchstore_error_t compact_view_fetchcb(couchfile_lookup_request *rq,
1591                                        const sized_buf *k,
1592                                        const sized_buf *v)
1593{
1594    int ret;
1595    sized_buf *k_c, *v_c;
1596    view_compact_ctx_t *ctx = (view_compact_ctx_t *) rq->callback_ctx;
1597    compactor_stats_t *stats = ctx->stats;
1598
1599    if (k == NULL || v == NULL) {
1600        return COUCHSTORE_ERROR_READ;
1601    }
1602
1603    if (ctx->filter_fun) {
1604        ret = ctx->filter_fun(k, v, ctx->filterbm);
1605        if (ret < 0) {
1606            return (couchstore_error_t) ret;
1607        } else if (ret) {
1608            return COUCHSTORE_SUCCESS;
1609        }
1610    }
1611
1612    k_c = arena_copy_buf(ctx->transient_arena, k);
1613    v_c = arena_copy_buf(ctx->transient_arena, v);
1614    ret = mr_push_item(k_c, v_c, ctx->mr);
1615    if (ret != COUCHSTORE_SUCCESS) {
1616        return static_cast<couchstore_error_t>(ret);
1617    }
1618
1619    if (stats) {
1620        stats->inserted++;
1621        if (stats->update_fun) {
1622            stats->update_fun(stats->freq, stats->inserted);
1623        }
1624    }
1625
1626    if (ctx->mr->count == 0) {
1627        arena_free_all(ctx->transient_arena);
1628    }
1629
1630    return (couchstore_error_t) ret;
1631}
1632
1633static couchstore_error_t compact_btree(tree_file *source,
1634                                 tree_file *target,
1635                                 const node_pointer *root,
1636                                 compare_info *cmp,
1637                                 reduce_fn reduce_fun,
1638                                 reduce_fn rereduce_fun,
1639                                 compact_filter_fn filter_fun,
1640                                 view_reducer_ctx_t *red_ctx,
1641                                 const bitmap_t *filterbm,
1642                                 compactor_stats_t *stats,
1643                                 node_pointer **out_root)
1644{
1645    couchstore_error_t ret = COUCHSTORE_SUCCESS;
1646    arena *transient_arena;
1647    arena *persistent_arena;
1648    couchfile_modify_result *modify_result;
1649    couchfile_lookup_request lookup_rq;
1650    view_compact_ctx_t compact_ctx;
1651    sized_buf nullkey = {NULL, 0};
1652    sized_buf *lowkeys = &nullkey;
1653
1654    if (!root) {
1655        return COUCHSTORE_SUCCESS;
1656    }
1657
1658    transient_arena = new_arena(0);
1659    persistent_arena = new_arena(0);
1660
1661    if (transient_arena == NULL || persistent_arena == NULL) {
1662        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1663        goto cleanup;
1664    }
1665
1666    /* Create new btree on new file */
1667    modify_result = new_btree_modres(persistent_arena,
1668                          transient_arena,
1669                          target,
1670                          cmp,
1671                          reduce_fun,
1672                          rereduce_fun,
1673                          red_ctx,
1674                          VIEW_KV_CHUNK_THRESHOLD + (VIEW_KV_CHUNK_THRESHOLD / 3),
1675                          VIEW_KP_CHUNK_THRESHOLD + (VIEW_KP_CHUNK_THRESHOLD / 3));
1676    if (modify_result == NULL) {
1677        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1678        goto cleanup;
1679    }
1680
1681    compact_ctx.filter_fun = NULL;
1682    compact_ctx.mr = modify_result;
1683    compact_ctx.transient_arena = transient_arena;
1684    compact_ctx.stats = stats;
1685
1686    if (filterbm) {
1687        compact_ctx.filterbm = filterbm;
1688        compact_ctx.filter_fun = filter_fun;
1689    }
1690
1691    lookup_rq.cmp.compare = cmp->compare;
1692    lookup_rq.file = source;
1693    lookup_rq.num_keys = 1;
1694    lookup_rq.keys = &lowkeys;
1695    lookup_rq.callback_ctx = &compact_ctx;
1696    lookup_rq.fetch_callback = compact_view_fetchcb;
1697    lookup_rq.node_callback = NULL;
1698    lookup_rq.fold = 1;
1699
1700    ret = btree_lookup(&lookup_rq, root->pointer);
1701    if (ret != COUCHSTORE_SUCCESS) {
1702        goto cleanup;
1703    }
1704
1705    *out_root = complete_new_btree(modify_result, &ret);
1706
1707cleanup:
1708    if (transient_arena != NULL) {
1709        delete_arena(transient_arena);
1710    }
1711
1712    if (persistent_arena != NULL) {
1713        delete_arena(persistent_arena);
1714    }
1715
1716    return ret;
1717}
1718
1719static couchstore_error_t compact_id_btree(tree_file *source,
1720                                    tree_file *target,
1721                                    const node_pointer *root,
1722                                    const bitmap_t *filterbm,
1723                                    compactor_stats_t *stats,
1724                                    node_pointer **out_root)
1725{
1726    couchstore_error_t ret;
1727    compare_info cmp;
1728
1729    cmp.compare = id_btree_cmp;
1730
1731    ret = compact_btree(source,
1732                        target,
1733                        root,
1734                        &cmp,
1735                        view_id_btree_reduce,
1736                        view_id_btree_rereduce,
1737                        view_id_btree_filter,
1738                        NULL,
1739                        filterbm,
1740                        stats,
1741                        out_root);
1742
1743    return ret;
1744}
1745
1746static couchstore_error_t compact_view_btree(tree_file *source,
1747                                      tree_file *target,
1748                                      const view_btree_info_t *info,
1749                                      const node_pointer *root,
1750                                      const bitmap_t *filterbm,
1751                                      compactor_stats_t *stats,
1752                                      node_pointer **out_root,
1753                                      view_error_t *error_info)
1754{
1755    couchstore_error_t ret = COUCHSTORE_SUCCESS;
1756    compare_info cmp;
1757    view_reducer_ctx_t *red_ctx = NULL;
1758    char *error_msg = NULL;
1759
1760    cmp.compare = view_btree_cmp;
1761    red_ctx = make_view_reducer_ctx(info->reducers,
1762                                    info->num_reducers,
1763                                    &error_msg);
1764    if (red_ctx == NULL) {
1765        set_error_info(info, (const char *) error_msg, ret, error_info);
1766        cb_free(error_msg);
1767        return COUCHSTORE_ERROR_REDUCER_FAILURE;
1768    }
1769
1770    ret = compact_btree(source,
1771                        target,
1772                        root,
1773                        &cmp,
1774                        view_btree_reduce,
1775                        view_btree_rereduce,
1776                        view_btree_filter,
1777                        red_ctx,
1778                        filterbm,
1779                        stats,
1780                        out_root);
1781
1782    if (ret != COUCHSTORE_SUCCESS) {
1783        set_error_info(info, red_ctx->error, ret, error_info);
1784    }
1785
1786    free_view_reducer_ctx(red_ctx);
1787
1788    return ret;
1789}
1790
1791couchstore_error_t couchstore_compact_view_group(view_group_info_t *info,
1792                                                 const char *target_file,
1793                                                 const sized_buf *header_buf,
1794                                                 compactor_stats_t *stats,
1795                                                 sized_buf *header_outbuf,
1796                                                 view_error_t *error_info)
1797{
1798    couchstore_error_t ret;
1799    tree_file index_file;
1800    tree_file compact_file;
1801    index_header_t *header = NULL;
1802    node_pointer *id_root = NULL;
1803    node_pointer **view_roots = NULL;
1804    bitmap_t *filterbm = NULL;
1805    bitmap_t emptybm;
1806    int i;
1807
1808    memset(&emptybm, 0, sizeof(bitmap_t));
1809    error_info->view_name = NULL;
1810    error_info->error_msg = NULL;
1811    index_file.handle = NULL;
1812    index_file.ops = NULL;
1813    index_file.path = NULL;
1814    compact_file.handle = NULL;
1815    compact_file.ops = NULL;
1816    compact_file.path = NULL;
1817
1818    ret = decode_index_header(header_buf->buf, header_buf->size, &header);
1819    if (ret < 0) {
1820        goto cleanup;
1821    }
1822
1823    /* Set filter bitmask if required */
1824    if (!is_equal_bitmap(&emptybm, &header->cleanup_bitmask)) {
1825        filterbm = &header->cleanup_bitmask;
1826    }
1827
1828    view_roots = (node_pointer **) cb_calloc(info->num_btrees,
1829                                          sizeof(node_pointer *));
1830    if (view_roots == NULL) {
1831        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1832        goto cleanup;
1833    }
1834
1835    cb_assert(info->num_btrees == header->num_views);
1836
1837    ret = open_view_group_file(info->filepath,
1838                               COUCHSTORE_OPEN_FLAG_RDONLY,
1839                               &index_file);
1840    if (ret != COUCHSTORE_SUCCESS) {
1841        goto cleanup;
1842    }
1843
1844    /*
1845     * Open target file for compaction
1846     * Expects that caller created the target file
1847     */
1848    ret = open_view_group_file(target_file, 0, &compact_file);
1849    if (ret != COUCHSTORE_SUCCESS) {
1850        goto cleanup;
1851    }
1852
1853    compact_file.pos = compact_file.ops->goto_eof(&compact_file.lastError,
1854                                                  compact_file.handle);
1855    ret = compact_id_btree(&index_file, &compact_file,
1856                                        header->id_btree_state,
1857                                        filterbm,
1858                                        stats,
1859                                        &id_root);
1860    if (ret != COUCHSTORE_SUCCESS) {
1861        goto cleanup;
1862    }
1863
1864    cb_free(header->id_btree_state);
1865    header->id_btree_state = id_root;
1866    id_root = NULL;
1867
1868    for (i = 0; i < info->num_btrees; ++i) {
1869        switch(info->type) {
1870        case VIEW_INDEX_TYPE_MAPREDUCE:
1871            ret = compact_view_btree(&index_file,
1872                                     &compact_file,
1873                                     &info->view_infos.btree[i],
1874                                     header->view_states[i],
1875                                     filterbm,
1876                                     stats,
1877                                     &view_roots[i],
1878                                     error_info);
1879            break;
1880        case VIEW_INDEX_TYPE_SPATIAL:
1881            ret = compact_view_spatial(&index_file,
1882                                       &compact_file,
1883                                       &info->view_infos.spatial[i],
1884                                       header->view_states[i],
1885                                       filterbm,
1886                                       stats,
1887                                       &view_roots[i],
1888                                       error_info);
1889            break;
1890        }
1891        if (ret != COUCHSTORE_SUCCESS) {
1892            goto cleanup;
1893        }
1894
1895        cb_free(header->view_states[i]);
1896        header->view_states[i] = view_roots[i];
1897        view_roots[i] = NULL;
1898    }
1899
1900    header->cleanup_bitmask = emptybm;
1901    ret = encode_index_header(header, &header_outbuf->buf, &header_outbuf->size);
1902    if (ret != COUCHSTORE_SUCCESS) {
1903        goto cleanup;
1904    }
1905
1906    ret = COUCHSTORE_SUCCESS;
1907
1908cleanup:
1909    free_index_header(header);
1910    close_view_group_file(info);
1911    tree_file_close(&index_file);
1912    tree_file_close(&compact_file);
1913    cb_free(id_root);
1914    if (view_roots != NULL) {
1915        for (i = 0; i < info->num_btrees; ++i) {
1916            cb_free(view_roots[i]);
1917        }
1918        cb_free(view_roots);
1919    }
1920
1921    return ret;
1922}
1923
1924
1925/*
1926 * For initial spatial build, feed the spatial builder as soon as
1927 * sorted records are available.
1928 */
1929static file_merger_error_t build_spatial_record_callback(void *buf, void *ctx)
1930{
1931    int ret;
1932    sized_buf *k, *v;
1933    view_file_merge_record_t *rec = (view_file_merge_record_t *) buf;
1934    view_file_merge_ctx_t *merge_ctx = (view_file_merge_ctx_t *) ctx;
1935    view_spatial_builder_ctx_t *build_ctx =
1936            (view_spatial_builder_ctx_t *) merge_ctx->user_ctx;
1937    sized_buf src_k, src_v;
1938
1939    src_k.size = rec->ksize;
1940    src_k.buf = VIEW_RECORD_KEY(rec);
1941    src_v.size = rec->vsize;
1942    src_v.buf = VIEW_RECORD_VAL(rec);
1943
1944    k = arena_copy_buf(build_ctx->transient_arena, &src_k);
1945    v = arena_copy_buf(build_ctx->transient_arena, &src_v);
1946    ret = spatial_push_item(k, v, build_ctx->modify_result);
1947
1948    if (ret != COUCHSTORE_SUCCESS) {
1949        return static_cast<file_merger_error_t>(ret);
1950    }
1951
1952    if (build_ctx->modify_result->count == 0) {
1953        arena_free_all(build_ctx->transient_arena);
1954    }
1955
1956    return static_cast<file_merger_error_t>(ret);
1957}
1958
1959
1960static couchstore_error_t build_view_spatial(const char *source_file,
1961                                             const view_spatial_info_t *info,
1962                                             tree_file *dest_file,
1963                                             const char *tmpdir,
1964                                             node_pointer **out_root,
1965                                             view_error_t *error_info)
1966{
1967    couchstore_error_t ret;
1968    compare_info cmp;
1969
1970    /* cmp.compare is only needed when you read a b-tree node or modify a
1971     * b-tree node (btree_modify:modify_node()). We don't do either in the
1972     * spatial view. */
1973    cmp.compare = NULL;
1974    ret = build_spatial(source_file,
1975                        dest_file,
1976                        &cmp,
1977                        view_spatial_reduce,
1978                        /* Use the reduce function also for the re-reduce */
1979                        view_spatial_reduce,
1980                        info->dimension,
1981                        info->mbb,
1982                        tmpdir,
1983                        sort_spatial_kvs_file,
1984                        out_root);
1985
1986    if (ret != COUCHSTORE_SUCCESS) {
1987        const int buffsize = 64;
1988        char* buf = (char*) cb_malloc(buffsize);
1989        if (buf != NULL) {
1990            int len = snprintf(buf, buffsize, "ret = %d", ret);
1991            if (len > 0 && len < buffsize) {
1992                error_info->error_msg = buf;
1993            } else {
1994                error_info->error_msg = cb_strdup("Failed to build error message");
1995                cb_free(buf);
1996            }
1997        }
1998    }
1999
2000    return ret;
2001}
2002
2003
2004static couchstore_error_t build_spatial(const char *source_file,
2005                                        tree_file *dest_file,
2006                                        compare_info *cmp,
2007                                        reduce_fn reduce_fun,
2008                                        reduce_fn rereduce_fun,
2009                                        const uint16_t dimension,
2010                                        const double *mbb,
2011                                        const char *tmpdir,
2012                                        sort_record_fn sort_fun,
2013                                        node_pointer **out_root)
2014{
2015    couchstore_error_t ret = COUCHSTORE_SUCCESS;
2016    arena *transient_arena = new_arena(0);
2017    arena *persistent_arena = new_arena(0);
2018    couchfile_modify_result *mr;
2019    view_spatial_builder_ctx_t build_ctx;
2020
2021    if (transient_arena == NULL || persistent_arena == NULL) {
2022        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
2023        goto out;
2024    }
2025
2026    mr = new_btree_modres(persistent_arena,
2027                          transient_arena,
2028                          dest_file,
2029                          cmp,
2030                          reduce_fun,
2031                          rereduce_fun,
2032                          NULL,
2033                          /* Normally the nodes are flushed to disk when 2/3 of
2034                           * the threshold is reached. In order to have a
2035                           * higher fill grade, we add 1/3 */
2036                          VIEW_KV_CHUNK_THRESHOLD + (VIEW_KV_CHUNK_THRESHOLD / 3),
2037                          VIEW_KP_CHUNK_THRESHOLD + (VIEW_KP_CHUNK_THRESHOLD / 3));
2038    if (mr == NULL) {
2039        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
2040        goto out;
2041    }
2042
2043    build_ctx.transient_arena = transient_arena;
2044    build_ctx.modify_result = mr;
2045    build_ctx.scale_factor = spatial_scale_factor(mbb, dimension,
2046                                                  ZCODE_MAX_VALUE);
2047
2048    ret = (couchstore_error_t) sort_fun(source_file,
2049                                        tmpdir,
2050                                        build_spatial_record_callback,
2051                                        &build_ctx);
2052    free_spatial_scale_factor(build_ctx.scale_factor);
2053    if (ret != COUCHSTORE_SUCCESS) {
2054        goto out;
2055    }
2056
2057    *out_root = complete_new_spatial(mr, &ret);
2058    if (ret != COUCHSTORE_SUCCESS) {
2059        goto out;
2060    }
2061
2062    /* Don't care about success/failure. Erlang side will eventually delete it. */
2063    remove(source_file);
2064
2065out:
2066    if (transient_arena != NULL) {
2067        delete_arena(transient_arena);
2068    }
2069
2070    return ret;
2071}
2072
2073/* Add the kv pair to modify result
2074 * The difference to the mapreduce views is the call to `spatial_push_item` */
2075static couchstore_error_t compact_spatial_fetchcb(couchfile_lookup_request *rq,
2076                                                  const sized_buf *k,
2077                                                  const sized_buf *v)
2078{
2079    int ret;
2080    sized_buf *k_c, *v_c;
2081    view_compact_ctx_t *ctx = (view_compact_ctx_t *) rq->callback_ctx;
2082    compactor_stats_t *stats = ctx->stats;
2083
2084    if (k == NULL || v == NULL) {
2085        return COUCHSTORE_ERROR_READ;
2086    }
2087
2088    if (ctx->filter_fun) {
2089        ret = ctx->filter_fun(k, v, ctx->filterbm);
2090        if (ret < 0) {
2091            return (couchstore_error_t) ret;
2092        } else if (ret) {
2093            return COUCHSTORE_SUCCESS;
2094        }
2095    }
2096
2097    k_c = arena_copy_buf(ctx->transient_arena, k);
2098    v_c = arena_copy_buf(ctx->transient_arena, v);
2099    ret = spatial_push_item(k_c, v_c, ctx->mr);
2100    if (ret != COUCHSTORE_SUCCESS) {
2101        return static_cast<couchstore_error_t>(ret);
2102    }
2103
2104    if (stats) {
2105        stats->inserted++;
2106        if (stats->update_fun) {
2107            stats->update_fun(stats->freq, stats->inserted);
2108        }
2109    }
2110
2111    if (ctx->mr->count == 0) {
2112        arena_free_all(ctx->transient_arena);
2113    }
2114
2115    return (couchstore_error_t) ret;
2116}
2117
2118static couchstore_error_t compact_view_spatial(tree_file *source,
2119                                               tree_file *target,
2120                                               const view_spatial_info_t *info,
2121                                               const node_pointer *root,
2122                                               const bitmap_t *filterbm,
2123                                               compactor_stats_t *stats,
2124                                               node_pointer **out_root,
2125                                               view_error_t *error_info)
2126{
2127    couchstore_error_t ret;
2128    compare_info cmp;
2129
2130    cmp.compare = view_btree_cmp;
2131    ret = compact_spatial(source,
2132                          target,
2133                          root,
2134                          &cmp,
2135                          view_spatial_reduce,
2136                          /* Use the reduce function also for the re-reduce */
2137                          view_spatial_reduce,
2138                          view_spatial_filter,
2139                          filterbm,
2140                          stats,
2141                          out_root);
2142
2143    return ret;
2144}
2145
2146static couchstore_error_t compact_spatial(tree_file *source,
2147                                          tree_file *target,
2148                                          const node_pointer *root,
2149                                          compare_info *cmp,
2150                                          reduce_fn reduce_fun,
2151                                          reduce_fn rereduce_fun,
2152                                          compact_filter_fn filter_fun,
2153                                          const bitmap_t *filterbm,
2154                                          compactor_stats_t *stats,
2155                                          node_pointer **out_root)
2156{
2157    couchstore_error_t ret = COUCHSTORE_SUCCESS;
2158    arena *transient_arena = new_arena(0);
2159    arena *persistent_arena = new_arena(0);
2160    couchfile_modify_result *modify_result;
2161    couchfile_lookup_request lookup_rq;
2162    view_compact_ctx_t compact_ctx;
2163    sized_buf nullkey = {NULL, 0};
2164    sized_buf *lowkeys = &nullkey;
2165
2166    if (!root) {
2167        return COUCHSTORE_SUCCESS;
2168    }
2169
2170    if (transient_arena == NULL || persistent_arena == NULL) {
2171        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
2172        goto cleanup;
2173    }
2174
2175    /* Create new spatial index on new file */
2176    modify_result = new_btree_modres(persistent_arena,
2177                                     transient_arena,
2178                                     target,
2179                                     cmp,
2180                                     reduce_fun,
2181                                     rereduce_fun,
2182                                     NULL,
2183                                     /* Normally the nodes are flushed to disk
2184                                      * when 2/3 of the threshold is reached.
2185                                      * In order to have a higher fill grade,
2186                                      * we add 1/3 */
2187                                     VIEW_KV_CHUNK_THRESHOLD +
2188                                         (VIEW_KV_CHUNK_THRESHOLD / 3),
2189                                     VIEW_KP_CHUNK_THRESHOLD +
2190                                         (VIEW_KP_CHUNK_THRESHOLD / 3));
2191    if (modify_result == NULL) {
2192        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
2193        goto cleanup;
2194    }
2195
2196    compact_ctx.filter_fun = NULL;
2197    compact_ctx.mr = modify_result;
2198    compact_ctx.transient_arena = transient_arena;
2199    compact_ctx.stats = stats;
2200
2201    if (filterbm) {
2202        compact_ctx.filterbm = filterbm;
2203        compact_ctx.filter_fun = filter_fun;
2204    }
2205
2206    lookup_rq.cmp.compare = cmp->compare;
2207    lookup_rq.file = source;
2208    lookup_rq.num_keys = 1;
2209    lookup_rq.keys = &lowkeys;
2210    lookup_rq.callback_ctx = &compact_ctx;
2211    lookup_rq.fetch_callback = compact_spatial_fetchcb;
2212    lookup_rq.node_callback = NULL;
2213    lookup_rq.fold = 1;
2214
2215    ret = btree_lookup(&lookup_rq, root->pointer);
2216    if (ret != COUCHSTORE_SUCCESS) {
2217        goto cleanup;
2218    }
2219
2220    *out_root = complete_new_spatial(modify_result, &ret);
2221
2222cleanup:
2223    if (transient_arena != NULL) {
2224        delete_arena(transient_arena);
2225    }
2226
2227    if (persistent_arena != NULL) {
2228        delete_arena(persistent_arena);
2229    }
2230
2231    return ret;
2232}
2233