xref: /3.0.2-MP2/couchstore/src/views/view_group.c (revision b0ed321f)
1/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2
3/**
4 * @copyright 2013 Couchbase, Inc.
5 *
6 * @author Filipe Manana  <filipe@couchbase.com>
7 *
8 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
9 * use this file except in compliance with the License. You may obtain a copy of
10 * the License at
11 *
12 *  http://www.apache.org/licenses/LICENSE-2.0
13 *
14 * Unless required by applicable law or agreed to in writing, software
15 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17 * License for the specific language governing permissions and limitations under
18 * the License.
19 **/
20
21#include "config.h"
22#include <assert.h>
23#include <stdlib.h>
24#include <string.h>
25#include "view_group.h"
26#include "reducers.h"
27#include "reductions.h"
28#include "values.h"
29#include "purgers.h"
30#include "util.h"
31#include "file_sorter.h"
32#include "../arena.h"
33#include "../couch_btree.h"
34#include "../internal.h"
35#include "../util.h"
36
37#define VIEW_KV_CHUNK_THRESHOLD (7 * 1024)
38#define VIEW_KP_CHUNK_THRESHOLD (6 * 1024)
39#define MAX_HEADER_SIZE         (64 * 1024)
40#define MAX_ACTIONS_SIZE        (2 * 1024 * 1024)
41
42static couchstore_error_t read_btree_info(view_group_info_t *info,
43                                          FILE *in_stream,
44                                          FILE *error_stream);
45
46static couchstore_error_t open_view_group_file(const char *path,
47                                               couchstore_open_flags open_flags,
48                                               tree_file *file);
49
50static couchstore_error_t build_btree(const char *source_file,
51                                      tree_file *dest_file,
52                                      compare_info *cmp,
53                                      reduce_fn reduce_fun,
54                                      reduce_fn rereduce_fun,
55                                      const char *tmpdir,
56                                      sort_record_fn sort_fun,
57                                      void *reduce_ctx,
58                                      node_pointer **out_root);
59
60static couchstore_error_t build_id_btree(const char *source_file,
61                                         tree_file *dest_file,
62                                         const char *tmpdir,
63                                         node_pointer **out_root);
64
65static couchstore_error_t build_view_btree(const char *source_file,
66                                           const view_btree_info_t *info,
67                                           tree_file *dest_file,
68                                           const char *tmpdir,
69                                           node_pointer **out_root,
70                                           view_error_t *error_info);
71
72static void close_view_group_file(view_group_info_t *info);
73
74static int read_record(FILE *f, arena *a, sized_buf *k, sized_buf *v,
75                                                        uint8_t *op);
76
77static couchstore_error_t update_btree(const char *source_file,
78                                       tree_file *dest_file,
79                                       const node_pointer *root,
80                                       size_t batch_size,
81                                       compare_info *cmp,
82                                       reduce_fn reduce_fun,
83                                       reduce_fn rereduce_fun,
84                                       purge_kv_fn purge_kv,
85                                       purge_kp_fn purge_kp,
86                                       view_reducer_ctx_t *red_ctx,
87                                       view_purger_ctx_t *purge_ctx,
88                                       uint64_t *inserted,
89                                       uint64_t *removed,
90                                       node_pointer **out_root);
91
92static couchstore_error_t update_id_btree(const char *source_file,
93                                         tree_file *dest_file,
94                                         const node_pointer *root,
95                                         size_t batch_size,
96                                         view_purger_ctx_t *purge_ctx,
97                                         uint64_t *inserted,
98                                         uint64_t *removed,
99                                         node_pointer **out_root);
100
101static couchstore_error_t update_view_btree(const char *source_file,
102                                            const view_btree_info_t *info,
103                                            tree_file *dest_file,
104                                            const node_pointer *root,
105                                            size_t batch_size,
106                                            view_purger_ctx_t *purge_ctx,
107                                            uint64_t *inserted,
108                                            uint64_t *removed,
109                                            node_pointer **out_root,
110                                            view_error_t *error_info);
111
112static couchstore_error_t compact_view_fetchcb(couchfile_lookup_request *rq,
113                                        const sized_buf *k,
114                                        const sized_buf *v);
115
116static couchstore_error_t compact_btree(tree_file *source,
117                                 tree_file *target,
118                                 const node_pointer *root,
119                                 compare_info *cmp,
120                                 reduce_fn reduce_fun,
121                                 reduce_fn rereduce_fun,
122                                 compact_filter_fn filter_fun,
123                                 view_reducer_ctx_t *red_ctx,
124                                 const bitmap_t *filterbm,
125                                 compactor_stats_t *stats,
126                                 node_pointer **out_root);
127
128static couchstore_error_t compact_id_btree(tree_file *source,
129                                    tree_file *target,
130                                    const node_pointer *root,
131                                    const bitmap_t *filterbm,
132                                    compactor_stats_t *stats,
133                                    node_pointer **out_root);
134
135static couchstore_error_t compact_view_btree(tree_file *source,
136                                      tree_file *target,
137                                      const view_btree_info_t *info,
138                                      const node_pointer *root,
139                                      const bitmap_t *filterbm,
140                                      compactor_stats_t *stats,
141                                      node_pointer **out_root,
142                                      view_error_t *error_info);
143
144LIBCOUCHSTORE_API
145view_group_info_t *couchstore_read_view_group_info(FILE *in_stream,
146                                                   FILE *error_stream)
147{
148    view_group_info_t *info;
149    char buf[4096];
150    char *dup;
151    couchstore_error_t ret;
152
153    info = (view_group_info_t *) calloc(1, sizeof(*info));
154    if (info == NULL) {
155        fprintf(error_stream, "Memory allocation failure\n");
156        goto out_error;
157    }
158
159    if (couchstore_read_line(in_stream, buf, sizeof(buf)) != buf) {
160        fprintf(stderr, "Error reading source index file path\n");
161        goto out_error;
162    }
163    dup = strdup(buf);
164    if (dup == NULL) {
165        fprintf(error_stream, "Memory allocation failure\n");
166        goto out_error;
167    }
168    info->filepath = (const char *) dup;
169
170    info->header_pos = couchstore_read_int(in_stream, buf,
171                                                      sizeof(buf),
172                                                      &ret);
173    if (ret != COUCHSTORE_SUCCESS) {
174        fprintf(error_stream, "Error reading header position\n");
175        goto out_error;
176    }
177
178    info->num_btrees = couchstore_read_int(in_stream, buf,
179                                                      sizeof(buf),
180                                                      &ret);
181    if (ret != COUCHSTORE_SUCCESS) {
182        fprintf(error_stream, "Error reading number of btrees\n");
183        goto out_error;
184    }
185
186    ret = read_btree_info(info, in_stream, error_stream);
187    if (ret != COUCHSTORE_SUCCESS) {
188        goto out_error;
189    }
190
191    return info;
192
193out_error:
194    couchstore_free_view_group_info(info);
195
196    return NULL;
197}
198
199
200/* Read in the information about the mapreduce view indexes */
201static couchstore_error_t read_btree_info(view_group_info_t *info,
202                                          FILE *in_stream,
203                                          FILE *error_stream)
204{
205    char buf[4096];
206    char *dup;
207    int i, j;
208    int reduce_len;
209    couchstore_error_t ret;
210
211    info->btree_infos = (view_btree_info_t *)
212        calloc(info->num_btrees, sizeof(view_btree_info_t));
213    if (info->btree_infos == NULL) {
214        fprintf(error_stream, "Memory allocation failure on btree infos\n");
215        info->num_btrees = 0;
216        return COUCHSTORE_ERROR_ALLOC_FAIL;
217    }
218
219    for (i = 0; i < info->num_btrees; ++i) {
220        view_btree_info_t *bti = &info->btree_infos[i];
221
222        bti->view_id = i;
223        bti->num_reducers = couchstore_read_int(in_stream,
224                                                buf,
225                                                sizeof(buf),
226                                                &ret);
227
228        if (ret != COUCHSTORE_SUCCESS) {
229            fprintf(error_stream,
230                    "Error reading number of reducers for btree %d\n", i);
231            return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
232        }
233
234        bti->names = (const char **) calloc(bti->num_reducers, sizeof(char *));
235        if (bti->names == NULL) {
236            fprintf(error_stream,
237                    "Memory allocation failure on btree %d reducer names\n",
238                    i);
239            bti->num_reducers = 0;
240            return COUCHSTORE_ERROR_ALLOC_FAIL;
241        }
242
243        bti->reducers = (const char **) calloc(bti->num_reducers, sizeof(char *));
244        if (bti->reducers == NULL) {
245            fprintf(error_stream,
246                    "Memory allocation failure on btree %d reducers\n", i);
247            bti->num_reducers = 0;
248            free(bti->names);
249            return COUCHSTORE_ERROR_ALLOC_FAIL;
250        }
251
252        for (j = 0; j < bti->num_reducers; ++j) {
253            if (couchstore_read_line(in_stream, buf, sizeof(buf)) != buf) {
254                fprintf(error_stream,
255                        "Error reading btree %d view %d name\n", i, j);
256                return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
257            }
258            dup = strdup(buf);
259            if (dup == NULL) {
260                fprintf(error_stream,
261                        "Memory allocation failure on btree %d "
262                        "view %d name\n", i, j);
263                return COUCHSTORE_ERROR_ALLOC_FAIL;
264            }
265            bti->names[j] = (const char *) dup;
266
267            reduce_len = couchstore_read_int(in_stream,
268                                             buf,
269                                             sizeof(buf),
270                                             &ret);
271            if (ret != COUCHSTORE_SUCCESS) {
272                fprintf(error_stream,
273                        "Error reading btree %d view %d "
274                        "reduce function size\n", i, j);
275                return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
276            }
277
278            dup = (char *) malloc(reduce_len + 1);
279            if (dup == NULL) {
280                fprintf(error_stream,
281                        "Memory allocation failure on btree %d "
282                        "view %d reducer\n", i, j);
283                return COUCHSTORE_ERROR_ALLOC_FAIL;
284            }
285
286            if (fread(dup, reduce_len, 1, in_stream) != 1) {
287                fprintf(error_stream,
288                        "Error reading btree %d view %d reducer\n", i, j);
289                free(dup);
290                return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
291            }
292            dup[reduce_len] = '\0';
293            bti->reducers[j] = (const char *) dup;
294        }
295    }
296    return COUCHSTORE_SUCCESS;
297}
298
299
300LIBCOUCHSTORE_API
301void couchstore_free_view_group_info(view_group_info_t *info)
302{
303    int i, j;
304
305    if (info == NULL)
306        return;
307
308    close_view_group_file(info);
309
310    for (i = 0; i < info->num_btrees; ++i) {
311        view_btree_info_t vi = info->btree_infos[i];
312
313        for (j = 0; j < vi.num_reducers; ++j) {
314            free((void *) vi.names[j]);
315            free((void *) vi.reducers[j]);
316        }
317        free(vi.names);
318        free(vi.reducers);
319    }
320    free(info->btree_infos);
321    free((void *) info->filepath);
322    free(info);
323}
324
325
326static void close_view_group_file(view_group_info_t *info)
327{
328    if (info->file.ops != NULL) {
329        info->file.ops->close(NULL, info->file.handle);
330        info->file.ops->destructor(NULL, info->file.handle);
331        info->file.ops = NULL;
332        info->file.handle = NULL;
333    }
334    free((void *) info->file.path);
335    info->file.path = NULL;
336}
337
338
339static couchstore_error_t open_view_group_file(const char *path,
340                                               couchstore_open_flags open_flags,
341                                               tree_file *file)
342{
343    couchstore_error_t ret;
344    const couch_file_ops *file_ops = couchstore_get_default_file_ops();
345    int flags = 0;
346
347    if ((open_flags & COUCHSTORE_OPEN_FLAG_RDONLY) &&
348        (open_flags & COUCHSTORE_OPEN_FLAG_CREATE)) {
349        return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
350    }
351
352    if (open_flags & COUCHSTORE_OPEN_FLAG_RDONLY) {
353        flags |= O_RDONLY;
354    } else {
355        flags |= O_RDWR;
356    }
357
358    if (open_flags & COUCHSTORE_OPEN_FLAG_CREATE) {
359        flags |= O_CREAT;
360    }
361
362    ret = tree_file_open(file, path, flags, file_ops);
363
364    return ret;
365}
366
367
368LIBCOUCHSTORE_API
369couchstore_error_t couchstore_build_view_group(view_group_info_t *info,
370                                               const char *id_records_file,
371                                               const char *kv_records_files[],
372                                               const char *dst_file,
373                                               const char *tmpdir,
374                                               uint64_t *header_pos,
375                                               view_error_t *error_info)
376{
377    couchstore_error_t ret;
378    tree_file index_file;
379    index_header_t *header = NULL;
380    node_pointer *id_root = NULL;
381    node_pointer **view_roots = NULL;
382    int i;
383
384    error_info->view_name = NULL;
385    error_info->error_msg = NULL;
386    index_file.handle = NULL;
387    index_file.ops = NULL;
388    index_file.path = NULL;
389
390    view_roots = (node_pointer **) calloc(
391        info->num_btrees, sizeof(node_pointer *));
392    if (view_roots == NULL) {
393        return COUCHSTORE_ERROR_ALLOC_FAIL;
394    }
395
396    ret = open_view_group_file(info->filepath,
397                               COUCHSTORE_OPEN_FLAG_RDONLY,
398                               &info->file);
399    if (ret != COUCHSTORE_SUCCESS) {
400        goto out;
401    }
402
403    ret = read_view_group_header(info, &header);
404    if (ret != COUCHSTORE_SUCCESS) {
405        goto out;
406    }
407    assert(info->num_btrees == header->num_views);
408
409    ret = open_view_group_file(dst_file,
410                               COUCHSTORE_OPEN_FLAG_CREATE,
411                               &index_file);
412    if (ret != COUCHSTORE_SUCCESS) {
413        goto out;
414    }
415
416    ret = build_id_btree(id_records_file, &index_file, tmpdir, &id_root);
417    if (ret != COUCHSTORE_SUCCESS) {
418        goto out;
419    }
420
421    free(header->id_btree_state);
422    header->id_btree_state = id_root;
423    id_root = NULL;
424
425    for (i = 0; i < info->num_btrees; ++i) {
426        ret = build_view_btree(kv_records_files[i],
427                               &info->btree_infos[i],
428                               &index_file,
429                               tmpdir,
430                               &view_roots[i],
431                               error_info);
432        if (ret != COUCHSTORE_SUCCESS) {
433            goto out;
434        }
435
436        free(header->view_states[i]);
437        header->view_states[i] = view_roots[i];
438        view_roots[i] = NULL;
439    }
440
441    ret = write_view_group_header(&index_file, header_pos, header);
442    if (ret != COUCHSTORE_SUCCESS) {
443        goto out;
444    }
445
446    ret = COUCHSTORE_SUCCESS;
447
448out:
449    free_index_header(header);
450    close_view_group_file(info);
451    tree_file_close(&index_file);
452    free(id_root);
453    if (view_roots != NULL) {
454        for (i = 0; i < info->num_btrees; ++i) {
455            free(view_roots[i]);
456        }
457        free(view_roots);
458    }
459
460    return ret;
461}
462
463
464/*
465 * Similar to util.c:read_view_record(), but it uses arena allocator, which is
466 * required for the existing semantics/api of btree bottom-up build in
467 * src/btree_modify.cc.
468 */
469static int read_record(FILE *f, arena *a, sized_buf *k, sized_buf *v,
470                                                        uint8_t *op)
471{
472    uint16_t klen;
473    uint8_t oplen = 0;
474    uint32_t vlen, len;
475
476    if (fread(&len, sizeof(len), 1, f) != 1) {
477        if (feof(f)) {
478            return 0;
479        } else {
480            return COUCHSTORE_ERROR_READ;
481        }
482    }
483
484    /* For incremental update, read optype */
485    if (op != NULL) {
486        if (fread(op, sizeof(uint8_t), 1, f) != 1) {
487            return COUCHSTORE_ERROR_READ;
488        }
489
490        oplen = 1;
491    }
492
493    if (fread(&klen, sizeof(klen), 1, f) != 1) {
494        return COUCHSTORE_ERROR_READ;
495    }
496
497    klen = ntohs(klen);
498    vlen = len - sizeof(klen) - klen - oplen;
499
500    k->size = klen;
501    k->buf = (char *) arena_alloc(a, k->size);
502    if (k->buf == NULL) {
503        return COUCHSTORE_ERROR_ALLOC_FAIL;
504    }
505
506    v->size = vlen;
507    /* Handle zero len vals */
508    if (v->size) {
509        v->buf = (char *) arena_alloc(a, v->size);
510        if (v->buf == NULL) {
511            return COUCHSTORE_ERROR_ALLOC_FAIL;
512        }
513    } else {
514        v->buf = NULL;
515    }
516
517    if (fread(k->buf, k->size, 1, f) != 1) {
518        return FILE_MERGER_ERROR_FILE_READ;
519    }
520
521    if (v->size && fread(v->buf, v->size, 1, f) != 1) {
522        return FILE_MERGER_ERROR_FILE_READ;
523    }
524
525    return len;
526}
527
528
529static int id_btree_cmp(const sized_buf *key1, const sized_buf *key2)
530{
531    return view_id_cmp(key1, key2, NULL);
532}
533
534
535static int view_btree_cmp(const sized_buf *key1, const sized_buf *key2)
536{
537    return view_key_cmp(key1, key2, NULL);
538}
539
540
541/*
542 * For initial btree build, feed the btree builder as soon as
543 * sorted records are available.
544 */
545static file_merger_error_t build_btree_record_callback(void *buf, void *ctx)
546{
547    int ret;
548    sized_buf *k, *v;
549    view_file_merge_record_t *rec = (view_file_merge_record_t *) buf;
550    view_file_merge_ctx_t *merge_ctx = (view_file_merge_ctx_t *) ctx;
551    view_btree_builder_ctx_t *build_ctx =
552                    (view_btree_builder_ctx_t *) merge_ctx->user_ctx;
553    sized_buf src_k, src_v;
554
555    src_k.size = rec->ksize;
556    src_k.buf = VIEW_RECORD_KEY(rec);
557    src_v.size = rec->vsize;
558    src_v.buf = VIEW_RECORD_VAL(rec);
559
560    k = arena_copy_buf(build_ctx->transient_arena, &src_k);
561    v = arena_copy_buf(build_ctx->transient_arena, &src_v);
562    ret = mr_push_item(k, v, build_ctx->modify_result);
563
564    if (ret != COUCHSTORE_SUCCESS) {
565        return ret;
566    }
567
568    if (build_ctx->modify_result->count == 0) {
569        arena_free_all(build_ctx->transient_arena);
570    }
571
572    return ret;
573}
574
575
576static couchstore_error_t build_btree(const char *source_file,
577                                      tree_file *dest_file,
578                                      compare_info *cmp,
579                                      reduce_fn reduce_fun,
580                                      reduce_fn rereduce_fun,
581                                      const char *tmpdir,
582                                      sort_record_fn sort_fun,
583                                      void *reduce_ctx,
584                                      node_pointer **out_root)
585{
586    couchstore_error_t ret = COUCHSTORE_SUCCESS;
587    arena *transient_arena = new_arena(0);
588    arena *persistent_arena = new_arena(0);
589    couchfile_modify_result *mr;
590    view_btree_builder_ctx_t build_ctx;
591
592    if (transient_arena == NULL || persistent_arena == NULL) {
593        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
594        goto out;
595    }
596
597    mr = new_btree_modres(persistent_arena,
598                          transient_arena,
599                          dest_file,
600                          cmp,
601                          reduce_fun,
602                          rereduce_fun,
603                          reduce_ctx,
604                          VIEW_KV_CHUNK_THRESHOLD + (VIEW_KV_CHUNK_THRESHOLD / 3),
605                          VIEW_KP_CHUNK_THRESHOLD + (VIEW_KP_CHUNK_THRESHOLD / 3));
606    if (mr == NULL) {
607        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
608        goto out;
609    }
610
611    build_ctx.transient_arena = transient_arena;
612    build_ctx.modify_result = mr;
613
614    ret = (couchstore_error_t) sort_fun(source_file,
615                                        tmpdir,
616                                        build_btree_record_callback, &build_ctx);
617    if (ret != COUCHSTORE_SUCCESS) {
618        goto out;
619    }
620
621    *out_root = complete_new_btree(mr, &ret);
622    if (ret != COUCHSTORE_SUCCESS) {
623        goto out;
624    }
625
626    /* Don't care about success/failure. Erlang side will eventually delete it. */
627    remove(source_file);
628
629out:
630    if (transient_arena != NULL) {
631        delete_arena(transient_arena);
632    }
633    if (persistent_arena != NULL) {
634        delete_arena(persistent_arena);
635    }
636
637    return ret;
638}
639
640
641static couchstore_error_t build_id_btree(const char *source_file,
642                                         tree_file *dest_file,
643                                         const char *tmpdir,
644                                         node_pointer **out_root)
645{
646    couchstore_error_t ret;
647    compare_info cmp;
648
649    cmp.compare = id_btree_cmp;
650
651    ret = build_btree(source_file,
652                      dest_file,
653                      &cmp,
654                      view_id_btree_reduce,
655                      view_id_btree_rereduce,
656                      tmpdir,
657                      sort_view_ids_file,
658                      NULL,
659                      out_root);
660
661    return ret;
662}
663
664
665static couchstore_error_t build_view_btree(const char *source_file,
666                                           const view_btree_info_t *info,
667                                           tree_file *dest_file,
668                                           const char *tmpdir,
669                                           node_pointer **out_root,
670                                           view_error_t *error_info)
671{
672    couchstore_error_t ret;
673    compare_info cmp;
674    view_reducer_ctx_t *red_ctx = NULL;
675    char *error_msg = NULL;
676
677    cmp.compare = view_btree_cmp;
678    red_ctx = make_view_reducer_ctx(info->reducers,
679                                    info->num_reducers,
680                                    &error_msg);
681    if (red_ctx == NULL) {
682        error_info->error_msg = (const char *) error_msg;
683        error_info->view_name = (const char *) strdup(info->names[0]);
684        return COUCHSTORE_ERROR_REDUCER_FAILURE;
685    }
686
687    ret = build_btree(source_file,
688                      dest_file,
689                      &cmp,
690                      view_btree_reduce,
691                      view_btree_rereduce,
692                      tmpdir,
693                      sort_view_kvs_file,
694                      red_ctx,
695                      out_root);
696
697    if (ret != COUCHSTORE_SUCCESS) {
698        set_error_info(info, red_ctx->error, ret, error_info);
699    }
700
701    free_view_reducer_ctx(red_ctx);
702
703    return ret;
704}
705
706
707couchstore_error_t read_view_group_header(view_group_info_t *info,
708                                          index_header_t **header)
709{
710    couchstore_error_t ret;
711    cs_off_t pos = info->header_pos;
712    tree_file *file = &info->file;
713    char *header_buf = NULL;
714    int header_len;
715    char buf;
716
717    if (file->handle == NULL) {
718        return COUCHSTORE_ERROR_FILE_CLOSED;
719    }
720
721    if (info->file.ops->pread(NULL, file->handle, &buf, 1, pos) != 1) {
722        return COUCHSTORE_ERROR_READ;
723    }
724    if (buf == 0) {
725        return COUCHSTORE_ERROR_NO_HEADER;
726    } else if (buf != 1) {
727        return COUCHSTORE_ERROR_CORRUPT;
728    }
729
730    header_len = pread_header(file, pos, &header_buf, MAX_HEADER_SIZE);
731    if (header_len < 0) {
732        return (couchstore_error_t) header_len;
733    }
734
735    ret = decode_index_header(header_buf, (size_t) header_len, header);
736    free(header_buf);
737
738    return ret;
739}
740
741couchstore_error_t write_view_group_header(tree_file *file,
742                                           uint64_t *pos,
743                                           const index_header_t *header)
744{
745    couchstore_error_t ret;
746    sized_buf buf = { NULL, 0 };
747    cs_off_t p;
748
749    if (file->handle == NULL) {
750        return COUCHSTORE_ERROR_FILE_CLOSED;
751    }
752
753    ret = encode_index_header(header, &buf.buf, &buf.size);
754    if (ret != COUCHSTORE_SUCCESS) {
755        return ret;
756    }
757
758    ret = write_header(file, &buf, &p);
759    if (ret != COUCHSTORE_SUCCESS) {
760        goto out;
761    }
762
763    assert(p >= 0);
764    *pos = (uint64_t) p;
765
766out:
767    free(buf.buf);
768
769    return ret;
770}
771
772
773static couchstore_error_t view_id_bitmask(const node_pointer *root, bitmap_t *bm)
774{
775    view_id_btree_reduction_t *r = NULL;
776    couchstore_error_t errcode;
777    if (root == NULL) {
778        return COUCHSTORE_SUCCESS;
779    }
780
781    errcode = decode_view_id_btree_reduction(root->reduce_value.buf, &r);
782    if (errcode != COUCHSTORE_SUCCESS) {
783        goto cleanup;
784    }
785
786    union_bitmaps(bm, &r->partitions_bitmap);
787
788cleanup:
789    free_view_id_btree_reduction(r);
790    return errcode;
791}
792
793
794static couchstore_error_t view_bitmask(const node_pointer *root, bitmap_t *bm)
795{
796    view_btree_reduction_t *r = NULL;
797    couchstore_error_t errcode;
798    if (root == NULL) {
799        return COUCHSTORE_SUCCESS;
800    }
801
802    errcode = decode_view_btree_reduction(root->reduce_value.buf,
803                                          root->reduce_value.size, &r);
804    if (errcode != COUCHSTORE_SUCCESS) {
805        goto cleanup;
806    }
807
808    union_bitmaps(bm, &r->partitions_bitmap);
809
810cleanup:
811    free_view_btree_reduction(r);
812    return errcode;
813}
814
815
816static couchstore_error_t cleanup_btree(tree_file *file,
817                                        node_pointer *root,
818                                        compare_info *cmp,
819                                        reduce_fn reduce,
820                                        reduce_fn rereduce,
821                                        purge_kv_fn purge_kv,
822                                        purge_kp_fn purge_kp,
823                                        view_purger_ctx_t *purge_ctx,
824                                        view_reducer_ctx_t *red_ctx,
825                                        node_pointer **out_root)
826{
827    couchstore_error_t errcode;
828    couchfile_modify_request rq;
829
830    rq.cmp = *cmp;
831    rq.file = file;
832    rq.actions = NULL;
833    rq.num_actions = 0;
834    rq.reduce = reduce;
835    rq.rereduce = rereduce;
836    rq.compacting = 0;
837    rq.kv_chunk_threshold = VIEW_KV_CHUNK_THRESHOLD;
838    rq.kp_chunk_threshold = VIEW_KP_CHUNK_THRESHOLD;
839    rq.purge_kp = purge_kp;
840    rq.purge_kv = purge_kv;
841    rq.enable_purging = 1;
842    rq.guided_purge_ctx = purge_ctx;
843    rq.user_reduce_ctx = red_ctx;
844
845    *out_root = guided_purge_btree(&rq, root, &errcode);
846
847    return errcode;
848}
849
850static couchstore_error_t cleanup_id_btree(tree_file *file,
851                                           node_pointer *root,
852                                           node_pointer **out_root,
853                                           view_purger_ctx_t *purge_ctx,
854                                           view_error_t *error_info)
855{
856    couchstore_error_t ret;
857    compare_info cmp;
858
859    cmp.compare = id_btree_cmp;
860
861    ret = cleanup_btree(file,
862                        root,
863                        &cmp,
864                        view_id_btree_reduce,
865                        view_id_btree_rereduce,
866                        view_id_btree_purge_kv,
867                        view_id_btree_purge_kp,
868                        purge_ctx,
869                        NULL,
870                        out_root);
871
872    return ret;
873}
874
875
876static couchstore_error_t cleanup_view_btree(tree_file *file,
877                                             node_pointer *root,
878                                             const view_btree_info_t *info,
879                                             node_pointer **out_root,
880                                             view_purger_ctx_t *purge_ctx,
881                                             view_error_t *error_info)
882{
883    couchstore_error_t ret;
884    compare_info cmp;
885    view_reducer_ctx_t *red_ctx = NULL;
886    char *error_msg = NULL;
887
888    cmp.compare = view_btree_cmp;
889    red_ctx = make_view_reducer_ctx(info->reducers,
890                                    info->num_reducers,
891                                    &error_msg);
892    if (red_ctx == NULL) {
893        error_info->error_msg = (const char *) error_msg;
894        error_info->view_name = (const char *) strdup(info->names[0]);
895        return COUCHSTORE_ERROR_REDUCER_FAILURE;
896    }
897
898    ret = cleanup_btree(file,
899                        root,
900                        &cmp,
901                        view_btree_reduce,
902                        view_btree_rereduce,
903                        view_btree_purge_kv,
904                        view_btree_purge_kp,
905                        purge_ctx,
906                        red_ctx,
907                        out_root);
908
909    if (ret != COUCHSTORE_SUCCESS) {
910        char *error_msg = NULL;
911
912        if (red_ctx->error != NULL) {
913            error_msg = strdup(red_ctx->error);
914        } else {
915            error_msg = (char *) malloc(64);
916            if (error_msg != NULL) {
917                sprintf(error_msg, "%d", ret);
918            }
919        }
920        error_info->error_msg = (const char *) error_msg;
921        error_info->view_name = (const char *) strdup(info->names[0]);
922    }
923
924    free_view_reducer_ctx(red_ctx);
925
926    return ret;
927}
928
929
930LIBCOUCHSTORE_API
931couchstore_error_t couchstore_cleanup_view_group(view_group_info_t *info,
932                                                 uint64_t *header_pos,
933                                                 uint64_t *purge_count,
934                                                 view_error_t *error_info)
935{
936    couchstore_error_t ret;
937    tree_file index_file;
938    index_header_t *header = NULL;
939    node_pointer *id_root = NULL;
940    node_pointer **view_roots = NULL;
941    view_purger_ctx_t purge_ctx;
942    bitmap_t bm_cleanup;
943    int i;
944
945    memset(&bm_cleanup, 0, sizeof(bm_cleanup));
946    error_info->view_name = NULL;
947    error_info->error_msg = NULL;
948    index_file.handle = NULL;
949    index_file.ops = NULL;
950    index_file.path = NULL;
951
952    view_roots = (node_pointer **) calloc(
953        info->num_btrees, sizeof(node_pointer *));
954    if (view_roots == NULL) {
955        return COUCHSTORE_ERROR_ALLOC_FAIL;
956    }
957
958    /* Read info from current index viewgroup file */
959    ret = open_view_group_file(info->filepath,
960                               COUCHSTORE_OPEN_FLAG_RDONLY,
961                               &info->file);
962    if (ret != COUCHSTORE_SUCCESS) {
963        goto cleanup;
964    }
965
966    ret = read_view_group_header(info, &header);
967    if (ret != COUCHSTORE_SUCCESS) {
968        goto cleanup;
969    }
970    assert(info->num_btrees == header->num_views);
971
972    /* Setup purger context */
973    purge_ctx.count = 0;
974    purge_ctx.cbitmask = header->cleanup_bitmask;
975
976    ret = open_view_group_file(info->filepath,
977                               0,
978                               &index_file);
979    if (ret != COUCHSTORE_SUCCESS) {
980        goto cleanup;
981    }
982
983    index_file.pos = index_file.ops->goto_eof(&index_file.lastError,
984                                                         index_file.handle);
985
986    /* Cleanup id_bree */
987    ret = cleanup_id_btree(&index_file, header->id_btree_state, &id_root,
988                                                                &purge_ctx,
989                                                                error_info);
990    if (ret != COUCHSTORE_SUCCESS) {
991        goto cleanup;
992    }
993
994    if (header->id_btree_state != id_root) {
995        free(header->id_btree_state);
996    }
997
998    header->id_btree_state = id_root;
999    view_id_bitmask(id_root, &bm_cleanup);
1000    id_root = NULL;
1001
1002    /* Cleanup all view btrees */
1003    for (i = 0; i < info->num_btrees; ++i) {
1004        ret = cleanup_view_btree(&index_file,
1005                                 (node_pointer *) header->view_states[i],
1006                                 &info->btree_infos[i],
1007                                 &view_roots[i],
1008                                 &purge_ctx,
1009                                 error_info);
1010
1011        if (ret != COUCHSTORE_SUCCESS) {
1012            goto cleanup;
1013        }
1014
1015        if (header->view_states[i] != view_roots[i]) {
1016            free(header->view_states[i]);
1017        }
1018
1019        header->view_states[i] = view_roots[i];
1020        view_bitmask(view_roots[i], &bm_cleanup);
1021        view_roots[i] = NULL;
1022    }
1023
1024    /* Set resulting cleanup bitmask */
1025    /* TODO: This code can be removed, if we do not plan for cleanup STOP command */
1026    intersect_bitmaps(&bm_cleanup, &purge_ctx.cbitmask);
1027    header->cleanup_bitmask = bm_cleanup;
1028
1029    /* Update header with new btree infos */
1030    ret = write_view_group_header(&index_file, header_pos, header);
1031    if (ret != COUCHSTORE_SUCCESS) {
1032        goto cleanup;
1033    }
1034
1035    *purge_count = purge_ctx.count;
1036    ret = COUCHSTORE_SUCCESS;
1037
1038cleanup:
1039    free_index_header(header);
1040    close_view_group_file(info);
1041    tree_file_close(&index_file);
1042    free(id_root);
1043    if (view_roots != NULL) {
1044        for (i = 0; i < info->num_btrees; ++i) {
1045            free(view_roots[i]);
1046        }
1047        free(view_roots);
1048    }
1049
1050    return ret;
1051}
1052
1053static couchstore_error_t update_btree(const char *source_file,
1054                                       tree_file *dest_file,
1055                                       const node_pointer *root,
1056                                       size_t batch_size,
1057                                       compare_info *cmp,
1058                                       reduce_fn reduce_fun,
1059                                       reduce_fn rereduce_fun,
1060                                       purge_kv_fn purge_kv,
1061                                       purge_kp_fn purge_kp,
1062                                       view_reducer_ctx_t *red_ctx,
1063                                       view_purger_ctx_t *purge_ctx,
1064                                       uint64_t *inserted,
1065                                       uint64_t *removed,
1066                                       node_pointer **out_root)
1067{
1068    couchstore_error_t ret = COUCHSTORE_SUCCESS;
1069    couchfile_modify_request rq;
1070    node_pointer *newroot = (node_pointer *) root;
1071    arena *transient_arena = new_arena(0);
1072    FILE *f = NULL;
1073    couchfile_modify_action *actions = NULL;
1074    sized_buf *keybufs = NULL, *valbufs = NULL;
1075    size_t bufsize = 0;
1076    int last_record = 0;
1077    bitmap_t empty_bm;
1078    int max_actions = MAX_ACTIONS_SIZE /
1079                (sizeof(couchfile_modify_action) + 2 * sizeof(sized_buf));
1080
1081    memset(&empty_bm, 0, sizeof(empty_bm));
1082
1083    if (transient_arena == NULL) {
1084        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1085        goto cleanup;
1086    }
1087
1088
1089    actions = (couchfile_modify_action *) calloc(
1090                                            max_actions,
1091                                            sizeof(couchfile_modify_action));
1092    if (actions == NULL) {
1093        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1094        goto cleanup;
1095    }
1096
1097    keybufs = (sized_buf *) calloc(max_actions, sizeof(sized_buf));
1098    if (keybufs == NULL) {
1099        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1100        goto cleanup;
1101    }
1102
1103    valbufs = (sized_buf *) calloc(max_actions, sizeof(sized_buf));
1104    if (valbufs == NULL) {
1105        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1106        goto cleanup;
1107    }
1108
1109    rq.cmp = *cmp;
1110    rq.file = dest_file;
1111    rq.actions = actions;
1112    rq.num_actions = 0;
1113    rq.reduce = reduce_fun;
1114    rq.rereduce = rereduce_fun;
1115    rq.compacting = 0;
1116    rq.kv_chunk_threshold = VIEW_KV_CHUNK_THRESHOLD;
1117    rq.kp_chunk_threshold = VIEW_KP_CHUNK_THRESHOLD;
1118    rq.purge_kp = purge_kp;
1119    rq.purge_kv = purge_kv;
1120    rq.guided_purge_ctx = purge_ctx;
1121    rq.enable_purging = 1;
1122    rq.user_reduce_ctx = red_ctx;
1123
1124    /* If cleanup bitmask is empty, no need to try purging */
1125    if (is_equal_bitmap(&empty_bm, &purge_ctx->cbitmask)) {
1126        rq.enable_purging = 0;
1127    }
1128
1129    f = fopen(source_file, "rb");
1130    if (f == NULL) {
1131        ret = COUCHSTORE_ERROR_OPEN_FILE;
1132        goto cleanup;
1133    }
1134
1135    while (!last_record) {
1136        int read_ret;
1137        uint8_t op;
1138
1139        read_ret = read_record(f, transient_arena, &keybufs[rq.num_actions],
1140                                                   &valbufs[rq.num_actions],
1141                                                   &op);
1142        if (read_ret == 0) {
1143            last_record = 1;
1144            goto flush;
1145        } else if (read_ret < 0) {
1146            ret = (couchstore_error_t) read_ret;
1147            goto cleanup;
1148        }
1149
1150        /* Add action */
1151        actions[rq.num_actions].type = op;
1152        actions[rq.num_actions].key = &keybufs[rq.num_actions];
1153        actions[rq.num_actions].value.data = &valbufs[rq.num_actions];
1154
1155        if (inserted && op == ACTION_INSERT) {
1156            (*inserted)++;
1157        } else if (removed && op == ACTION_REMOVE) {
1158            (*removed)++;
1159        }
1160
1161        bufsize += keybufs[rq.num_actions].size +
1162                   valbufs[rq.num_actions].size +
1163                   sizeof(uint8_t);
1164        rq.num_actions++;
1165
1166flush:
1167        if (rq.num_actions && (last_record || bufsize > batch_size ||
1168                               rq.num_actions == max_actions)) {
1169            newroot = modify_btree(&rq, newroot, &ret);
1170            if (ret != COUCHSTORE_SUCCESS) {
1171                goto cleanup;
1172            }
1173
1174            rq.num_actions = 0;
1175            bufsize = 0;
1176            arena_free_all(transient_arena);
1177        }
1178    }
1179
1180    *out_root = newroot;
1181
1182cleanup:
1183    free(actions);
1184    free(keybufs);
1185    free(valbufs);
1186
1187    if (f != NULL) {
1188        fclose(f);
1189    }
1190
1191    if (transient_arena != NULL) {
1192        delete_arena(transient_arena);
1193    }
1194
1195    return ret;
1196}
1197
1198static couchstore_error_t update_id_btree(const char *source_file,
1199                                         tree_file *dest_file,
1200                                         const node_pointer *root,
1201                                         size_t batch_size,
1202                                         view_purger_ctx_t *purge_ctx,
1203                                         uint64_t *inserted,
1204                                         uint64_t *removed,
1205                                         node_pointer **out_root)
1206{
1207    couchstore_error_t ret;
1208    compare_info cmp;
1209
1210    cmp.compare = id_btree_cmp;
1211
1212    ret = update_btree(source_file,
1213                      dest_file,
1214                      root,
1215                      batch_size,
1216                      &cmp,
1217                      view_id_btree_reduce,
1218                      view_id_btree_rereduce,
1219                      view_id_btree_purge_kv,
1220                      view_id_btree_purge_kp,
1221                      NULL,
1222                      purge_ctx,
1223                      inserted,
1224                      removed,
1225                      out_root);
1226
1227    return ret;
1228}
1229
1230
1231static couchstore_error_t update_view_btree(const char *source_file,
1232                                            const view_btree_info_t *info,
1233                                            tree_file *dest_file,
1234                                            const node_pointer *root,
1235                                            size_t batch_size,
1236                                            view_purger_ctx_t *purge_ctx,
1237                                            uint64_t *inserted,
1238                                            uint64_t *removed,
1239                                            node_pointer **out_root,
1240                                            view_error_t *error_info)
1241{
1242    couchstore_error_t ret;
1243    compare_info cmp;
1244    view_reducer_ctx_t *red_ctx = NULL;
1245    char *error_msg = NULL;
1246
1247    cmp.compare = view_btree_cmp;
1248    red_ctx = make_view_reducer_ctx(info->reducers,
1249                                    info->num_reducers,
1250                                    &error_msg);
1251    if (red_ctx == NULL) {
1252        error_info->error_msg = (const char *) error_msg;
1253        error_info->view_name = (const char *) strdup(info->names[0]);
1254        return COUCHSTORE_ERROR_REDUCER_FAILURE;
1255    }
1256
1257    ret = update_btree(source_file,
1258                      dest_file,
1259                      root,
1260                      batch_size,
1261                      &cmp,
1262                      view_btree_reduce,
1263                      view_btree_rereduce,
1264                      view_btree_purge_kv,
1265                      view_btree_purge_kp,
1266                      red_ctx,
1267                      purge_ctx,
1268                      inserted,
1269                      removed,
1270                      out_root);
1271
1272    if (ret != COUCHSTORE_SUCCESS) {
1273        set_error_info(info, red_ctx->error, ret, error_info);
1274    }
1275
1276    free_view_reducer_ctx(red_ctx);
1277
1278    return ret;
1279}
1280
1281LIBCOUCHSTORE_API
1282couchstore_error_t couchstore_update_view_group(view_group_info_t *info,
1283                                               const char *id_records_file,
1284                                               const char *kv_records_files[],
1285                                               size_t batch_size,
1286                                               const sized_buf *header_buf,
1287                                               int is_sorted,
1288                                               const char *tmp_dir,
1289                                               view_group_update_stats_t *stats,
1290                                               sized_buf *header_outbuf,
1291                                               view_error_t *error_info)
1292{
1293    couchstore_error_t ret;
1294    tree_file index_file;
1295    index_header_t *header = NULL;
1296    node_pointer *id_root = NULL;
1297    node_pointer **view_roots = NULL;
1298    view_purger_ctx_t purge_ctx;
1299    bitmap_t bm_cleanup;
1300    int i;
1301
1302    ret = decode_index_header(header_buf->buf, header_buf->size, &header);
1303    if (ret < 0) {
1304        goto cleanup;
1305    }
1306
1307    memset(&bm_cleanup, 0, sizeof(bm_cleanup));
1308
1309    error_info->view_name = NULL;
1310    error_info->error_msg = NULL;
1311    index_file.handle = NULL;
1312    index_file.ops = NULL;
1313    index_file.path = NULL;
1314
1315    view_roots = (node_pointer **) calloc(info->num_btrees,
1316                                          sizeof(node_pointer *));
1317    if (view_roots == NULL) {
1318        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1319        goto cleanup;
1320    }
1321
1322    assert(info->num_btrees == header->num_views);
1323
1324    /* Setup purger context */
1325    purge_ctx.count = 0;
1326    purge_ctx.cbitmask = header->cleanup_bitmask;
1327
1328    ret = open_view_group_file(info->filepath,
1329                               0,
1330                               &index_file);
1331    if (ret != COUCHSTORE_SUCCESS) {
1332        goto cleanup;
1333    }
1334
1335    /* Move pos to end of file */
1336    index_file.pos = index_file.ops->goto_eof(&index_file.lastError,
1337                                              index_file.handle);
1338
1339    if (!is_sorted) {
1340        ret = (couchstore_error_t) sort_view_ids_ops_file(id_records_file, tmp_dir);
1341        if (ret != COUCHSTORE_SUCCESS) {
1342            char buf[1024];
1343            snprintf(buf, sizeof(buf),
1344                    "Error sorting records file: %s", id_records_file);
1345            error_info->error_msg = strdup(buf);
1346            error_info->view_name = (const char *) strdup("id_btree");
1347            goto cleanup;
1348        }
1349    }
1350
1351    ret = update_id_btree(id_records_file, &index_file,
1352                                           header->id_btree_state,
1353                                           batch_size,
1354                                           &purge_ctx,
1355                                           &stats->ids_inserted,
1356                                           &stats->ids_removed,
1357                                           &id_root);
1358    if (ret != COUCHSTORE_SUCCESS) {
1359        goto cleanup;
1360    }
1361
1362
1363    if (header->id_btree_state != id_root) {
1364        free(header->id_btree_state);
1365    }
1366
1367    header->id_btree_state = id_root;
1368    view_id_bitmask(id_root, &bm_cleanup);
1369    id_root = NULL;
1370
1371    for (i = 0; i < info->num_btrees; ++i) {
1372        if (!is_sorted) {
1373            ret = (couchstore_error_t) sort_view_kvs_ops_file(kv_records_files[i], tmp_dir);
1374            if (ret != COUCHSTORE_SUCCESS) {
1375                char buf[1024];
1376                snprintf(buf, sizeof(buf),
1377                        "Error sorting records file: %s", kv_records_files[i]);
1378                error_info->error_msg = strdup(buf);
1379                error_info->view_name = (const char *) strdup(info->btree_infos[i].names[0]);
1380                goto cleanup;
1381            }
1382        }
1383
1384        ret = update_view_btree(kv_records_files[i],
1385                                &info->btree_infos[i],
1386                                &index_file,
1387                                header->view_states[i],
1388                                batch_size,
1389                                &purge_ctx,
1390                                &stats->kvs_inserted,
1391                                &stats->kvs_removed,
1392                                &view_roots[i],
1393                                error_info);
1394
1395        if (ret != COUCHSTORE_SUCCESS) {
1396            goto cleanup;
1397        }
1398
1399        if (header->view_states[i] != view_roots[i]) {
1400            free(header->view_states[i]);
1401        }
1402
1403        header->view_states[i] = view_roots[i];
1404        view_bitmask(view_roots[i], &bm_cleanup);
1405        view_roots[i] = NULL;
1406    }
1407
1408    /* Set resulting cleanup bitmask */
1409    intersect_bitmaps(&bm_cleanup, &purge_ctx.cbitmask);
1410    header->cleanup_bitmask = bm_cleanup;
1411    stats->purged = purge_ctx.count;
1412
1413    ret = encode_index_header(header, &header_outbuf->buf, &header_outbuf->size);
1414    if (ret != COUCHSTORE_SUCCESS) {
1415        goto cleanup;
1416    }
1417
1418    ret = COUCHSTORE_SUCCESS;
1419
1420cleanup:
1421    free_index_header(header);
1422    close_view_group_file(info);
1423    tree_file_close(&index_file);
1424    free(id_root);
1425    if (view_roots != NULL) {
1426        for (i = 0; i < info->num_btrees; ++i) {
1427            free(view_roots[i]);
1428        }
1429        free(view_roots);
1430    }
1431
1432    return ret;
1433}
1434
1435/* Add the kv pair to modify result */
1436static couchstore_error_t compact_view_fetchcb(couchfile_lookup_request *rq,
1437                                        const sized_buf *k,
1438                                        const sized_buf *v)
1439{
1440    int ret;
1441    sized_buf *k_c, *v_c;
1442    view_compact_ctx_t *ctx = (view_compact_ctx_t *) rq->callback_ctx;
1443    compactor_stats_t *stats = ctx->stats;
1444
1445    if (k == NULL || v == NULL) {
1446        return COUCHSTORE_ERROR_READ;
1447    }
1448
1449    if (ctx->filter_fun) {
1450        ret = ctx->filter_fun(k, v, ctx->filterbm);
1451        if (ret < 0) {
1452            return (couchstore_error_t) ret;
1453        } else if (ret) {
1454            return COUCHSTORE_SUCCESS;
1455        }
1456    }
1457
1458    k_c = arena_copy_buf(ctx->transient_arena, k);
1459    v_c = arena_copy_buf(ctx->transient_arena, v);
1460    ret = mr_push_item(k_c, v_c, ctx->mr);
1461    if (ret != COUCHSTORE_SUCCESS) {
1462        return ret;
1463    }
1464
1465    if (stats) {
1466        stats->inserted++;
1467        if (stats->update_fun) {
1468            stats->update_fun(stats->freq, stats->inserted);
1469        }
1470    }
1471
1472    if (ctx->mr->count == 0) {
1473        arena_free_all(ctx->transient_arena);
1474    }
1475
1476    return (couchstore_error_t) ret;
1477}
1478
1479static couchstore_error_t compact_btree(tree_file *source,
1480                                 tree_file *target,
1481                                 const node_pointer *root,
1482                                 compare_info *cmp,
1483                                 reduce_fn reduce_fun,
1484                                 reduce_fn rereduce_fun,
1485                                 compact_filter_fn filter_fun,
1486                                 view_reducer_ctx_t *red_ctx,
1487                                 const bitmap_t *filterbm,
1488                                 compactor_stats_t *stats,
1489                                 node_pointer **out_root)
1490{
1491    couchstore_error_t ret = COUCHSTORE_SUCCESS;
1492    arena *transient_arena;
1493    arena *persistent_arena;
1494    couchfile_modify_result *modify_result;
1495    couchfile_lookup_request lookup_rq;
1496    view_compact_ctx_t compact_ctx;
1497    sized_buf nullkey = {NULL, 0};
1498    sized_buf *lowkeys = &nullkey;
1499
1500    if (!root) {
1501        return COUCHSTORE_SUCCESS;
1502    }
1503
1504    transient_arena = new_arena(0);
1505    persistent_arena = new_arena(0);
1506
1507    if (transient_arena == NULL || persistent_arena == NULL) {
1508        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1509        goto cleanup;
1510    }
1511
1512    /* Create new btree on new file */
1513    modify_result = new_btree_modres(persistent_arena,
1514                          transient_arena,
1515                          target,
1516                          cmp,
1517                          reduce_fun,
1518                          rereduce_fun,
1519                          red_ctx,
1520                          VIEW_KV_CHUNK_THRESHOLD + (VIEW_KV_CHUNK_THRESHOLD / 3),
1521                          VIEW_KP_CHUNK_THRESHOLD + (VIEW_KP_CHUNK_THRESHOLD / 3));
1522    if (modify_result == NULL) {
1523        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1524        goto cleanup;
1525    }
1526
1527    compact_ctx.filter_fun = NULL;
1528    compact_ctx.mr = modify_result;
1529    compact_ctx.transient_arena = transient_arena;
1530    compact_ctx.stats = stats;
1531
1532    if (filterbm) {
1533        compact_ctx.filterbm = filterbm;
1534        compact_ctx.filter_fun = filter_fun;
1535    }
1536
1537    lookup_rq.cmp.compare = cmp->compare;
1538    lookup_rq.file = source;
1539    lookup_rq.num_keys = 1;
1540    lookup_rq.keys = &lowkeys;
1541    lookup_rq.callback_ctx = &compact_ctx;
1542    lookup_rq.fetch_callback = compact_view_fetchcb;
1543    lookup_rq.node_callback = NULL;
1544    lookup_rq.fold = 1;
1545
1546    ret = btree_lookup(&lookup_rq, root->pointer);
1547    if (ret != COUCHSTORE_SUCCESS) {
1548        goto cleanup;
1549    }
1550
1551    *out_root = complete_new_btree(modify_result, &ret);
1552
1553cleanup:
1554    if (transient_arena != NULL) {
1555        delete_arena(transient_arena);
1556    }
1557
1558    if (persistent_arena != NULL) {
1559        delete_arena(persistent_arena);
1560    }
1561
1562    return ret;
1563}
1564
1565static couchstore_error_t compact_id_btree(tree_file *source,
1566                                    tree_file *target,
1567                                    const node_pointer *root,
1568                                    const bitmap_t *filterbm,
1569                                    compactor_stats_t *stats,
1570                                    node_pointer **out_root)
1571{
1572    couchstore_error_t ret;
1573    compare_info cmp;
1574
1575    cmp.compare = id_btree_cmp;
1576
1577    ret = compact_btree(source,
1578                        target,
1579                        root,
1580                        &cmp,
1581                        view_id_btree_reduce,
1582                        view_id_btree_rereduce,
1583                        view_id_btree_filter,
1584                        NULL,
1585                        filterbm,
1586                        stats,
1587                        out_root);
1588
1589    return ret;
1590}
1591
1592static couchstore_error_t compact_view_btree(tree_file *source,
1593                                      tree_file *target,
1594                                      const view_btree_info_t *info,
1595                                      const node_pointer *root,
1596                                      const bitmap_t *filterbm,
1597                                      compactor_stats_t *stats,
1598                                      node_pointer **out_root,
1599                                      view_error_t *error_info)
1600{
1601    couchstore_error_t ret;
1602    compare_info cmp;
1603    view_reducer_ctx_t *red_ctx = NULL;
1604    char *error_msg = NULL;
1605
1606    cmp.compare = view_btree_cmp;
1607    red_ctx = make_view_reducer_ctx(info->reducers,
1608                                    info->num_reducers,
1609                                    &error_msg);
1610    if (red_ctx == NULL) {
1611        error_info->error_msg = (const char *) error_msg;
1612        error_info->view_name = (const char *) strdup(info->names[0]);
1613        return COUCHSTORE_ERROR_REDUCER_FAILURE;
1614    }
1615
1616    ret = compact_btree(source,
1617                        target,
1618                        root,
1619                        &cmp,
1620                        view_btree_reduce,
1621                        view_btree_rereduce,
1622                        view_btree_filter,
1623                        red_ctx,
1624                        filterbm,
1625                        stats,
1626                        out_root);
1627
1628    if (ret != COUCHSTORE_SUCCESS) {
1629        set_error_info(info, red_ctx->error, ret, error_info);
1630    }
1631
1632    free_view_reducer_ctx(red_ctx);
1633
1634    return ret;
1635}
1636
1637LIBCOUCHSTORE_API
1638couchstore_error_t couchstore_compact_view_group(view_group_info_t *info,
1639                                                 const char *target_file,
1640                                                 const sized_buf *header_buf,
1641                                                 compactor_stats_t *stats,
1642                                                 sized_buf *header_outbuf,
1643                                                 view_error_t *error_info)
1644{
1645    couchstore_error_t ret;
1646    tree_file index_file;
1647    tree_file compact_file;
1648    index_header_t *header = NULL;
1649    node_pointer *id_root = NULL;
1650    node_pointer **view_roots = NULL;
1651    bitmap_t *filterbm = NULL;
1652    bitmap_t emptybm;
1653    int i;
1654
1655    memset(&emptybm, 0, sizeof(bitmap_t));
1656    error_info->view_name = NULL;
1657    error_info->error_msg = NULL;
1658    index_file.handle = NULL;
1659    index_file.ops = NULL;
1660    index_file.path = NULL;
1661    compact_file.handle = NULL;
1662    compact_file.ops = NULL;
1663    compact_file.path = NULL;
1664
1665    ret = decode_index_header(header_buf->buf, header_buf->size, &header);
1666    if (ret < 0) {
1667        goto cleanup;
1668    }
1669
1670    /* Set filter bitmask if required */
1671    if (!is_equal_bitmap(&emptybm, &header->cleanup_bitmask)) {
1672        filterbm = &header->cleanup_bitmask;
1673    }
1674
1675    view_roots = (node_pointer **) calloc(info->num_btrees,
1676                                          sizeof(node_pointer *));
1677    if (view_roots == NULL) {
1678        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1679        goto cleanup;
1680    }
1681
1682    assert(info->num_btrees == header->num_views);
1683
1684    ret = open_view_group_file(info->filepath,
1685                               COUCHSTORE_OPEN_FLAG_RDONLY,
1686                               &index_file);
1687    if (ret != COUCHSTORE_SUCCESS) {
1688        goto cleanup;
1689    }
1690
1691    /*
1692     * Open target file for compaction
1693     * Expects that caller created the target file
1694     */
1695    ret = open_view_group_file(target_file, 0, &compact_file);
1696    if (ret != COUCHSTORE_SUCCESS) {
1697        goto cleanup;
1698    }
1699
1700    compact_file.pos = compact_file.ops->goto_eof(&compact_file.lastError,
1701                                                  compact_file.handle);
1702    ret = compact_id_btree(&index_file, &compact_file,
1703                                        header->id_btree_state,
1704                                        filterbm,
1705                                        stats,
1706                                        &id_root);
1707    if (ret != COUCHSTORE_SUCCESS) {
1708        goto cleanup;
1709    }
1710
1711    free(header->id_btree_state);
1712    header->id_btree_state = id_root;
1713    id_root = NULL;
1714
1715    for (i = 0; i < info->num_btrees; ++i) {
1716        ret = compact_view_btree(&index_file,
1717                                 &compact_file,
1718                                 &info->btree_infos[i],
1719                                 header->view_states[i],
1720                                 filterbm,
1721                                 stats,
1722                                 &view_roots[i],
1723                                 error_info);
1724
1725        if (ret != COUCHSTORE_SUCCESS) {
1726            goto cleanup;
1727        }
1728
1729        free(header->view_states[i]);
1730        header->view_states[i] = view_roots[i];
1731        view_roots[i] = NULL;
1732    }
1733
1734    header->cleanup_bitmask = emptybm;
1735    ret = encode_index_header(header, &header_outbuf->buf, &header_outbuf->size);
1736    if (ret != COUCHSTORE_SUCCESS) {
1737        goto cleanup;
1738    }
1739
1740    ret = COUCHSTORE_SUCCESS;
1741
1742cleanup:
1743    free_index_header(header);
1744    close_view_group_file(info);
1745    tree_file_close(&index_file);
1746    tree_file_close(&compact_file);
1747    free(id_root);
1748    if (view_roots != NULL) {
1749        for (i = 0; i < info->num_btrees; ++i) {
1750            free(view_roots[i]);
1751        }
1752        free(view_roots);
1753    }
1754
1755    return ret;
1756}
1757