xref: /3.0.2-MP2/couchstore/src/views/view_group.c (revision 1b5905aa)
1/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2
3/**
4 * @copyright 2013 Couchbase, Inc.
5 *
6 * @author Filipe Manana  <filipe@couchbase.com>
7 *
8 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
9 * use this file except in compliance with the License. You may obtain a copy of
10 * the License at
11 *
12 *  http://www.apache.org/licenses/LICENSE-2.0
13 *
14 * Unless required by applicable law or agreed to in writing, software
15 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17 * License for the specific language governing permissions and limitations under
18 * the License.
19 **/
20
21#include "config.h"
22#include <assert.h>
23#include <stdlib.h>
24#include <string.h>
25#include "view_group.h"
26#include "reducers.h"
27#include "reductions.h"
28#include "values.h"
29#include "purgers.h"
30#include "util.h"
31#include "file_sorter.h"
32#include "../arena.h"
33#include "../couch_btree.h"
34#include "../internal.h"
35#include "../util.h"
36
37#define VIEW_KV_CHUNK_THRESHOLD (7 * 1024)
38#define VIEW_KP_CHUNK_THRESHOLD (6 * 1024)
39#define MAX_HEADER_SIZE         (64 * 1024)
40#define MAX_ACTIONS_SIZE        (2 * 1024 * 1024)
41
42static couchstore_error_t read_btree_info(view_group_info_t *info,
43                                          FILE *in_stream,
44                                          FILE *error_stream);
45
46static couchstore_error_t open_view_group_file(const char *path,
47                                               couchstore_open_flags open_flags,
48                                               tree_file *file);
49
50static couchstore_error_t build_btree(const char *source_file,
51                                      tree_file *dest_file,
52                                      compare_info *cmp,
53                                      reduce_fn reduce_fun,
54                                      reduce_fn rereduce_fun,
55                                      const char *tmpdir,
56                                      sort_record_fn sort_fun,
57                                      void *reduce_ctx,
58                                      node_pointer **out_root);
59
60static couchstore_error_t build_id_btree(const char *source_file,
61                                         tree_file *dest_file,
62                                         const char *tmpdir,
63                                         node_pointer **out_root);
64
65static couchstore_error_t build_view_btree(const char *source_file,
66                                           const view_btree_info_t *info,
67                                           tree_file *dest_file,
68                                           const char *tmpdir,
69                                           node_pointer **out_root,
70                                           view_error_t *error_info);
71
72static void close_view_group_file(view_group_info_t *info);
73
74static int read_record(FILE *f, arena *a, sized_buf *k, sized_buf *v,
75                                                        uint8_t *op);
76
77static couchstore_error_t update_btree(const char *source_file,
78                                       tree_file *dest_file,
79                                       const node_pointer *root,
80                                       size_t batch_size,
81                                       compare_info *cmp,
82                                       reduce_fn reduce_fun,
83                                       reduce_fn rereduce_fun,
84                                       purge_kv_fn purge_kv,
85                                       purge_kp_fn purge_kp,
86                                       view_reducer_ctx_t *red_ctx,
87                                       view_purger_ctx_t *purge_ctx,
88                                       uint64_t *inserted,
89                                       uint64_t *removed,
90                                       node_pointer **out_root);
91
92static couchstore_error_t update_id_btree(const char *source_file,
93                                         tree_file *dest_file,
94                                         const node_pointer *root,
95                                         size_t batch_size,
96                                         view_purger_ctx_t *purge_ctx,
97                                         uint64_t *inserted,
98                                         uint64_t *removed,
99                                         node_pointer **out_root);
100
101static couchstore_error_t update_view_btree(const char *source_file,
102                                            const view_btree_info_t *info,
103                                            tree_file *dest_file,
104                                            const node_pointer *root,
105                                            size_t batch_size,
106                                            view_purger_ctx_t *purge_ctx,
107                                            uint64_t *inserted,
108                                            uint64_t *removed,
109                                            node_pointer **out_root,
110                                            view_error_t *error_info);
111
112static couchstore_error_t compact_view_fetchcb(couchfile_lookup_request *rq,
113                                        const sized_buf *k,
114                                        const sized_buf *v);
115
116static couchstore_error_t compact_btree(tree_file *source,
117                                 tree_file *target,
118                                 const node_pointer *root,
119                                 compare_info *cmp,
120                                 reduce_fn reduce_fun,
121                                 reduce_fn rereduce_fun,
122                                 compact_filter_fn filter_fun,
123                                 view_reducer_ctx_t *red_ctx,
124                                 const bitmap_t *filterbm,
125                                 compactor_stats_t *stats,
126                                 node_pointer **out_root);
127
128static couchstore_error_t compact_id_btree(tree_file *source,
129                                    tree_file *target,
130                                    const node_pointer *root,
131                                    const bitmap_t *filterbm,
132                                    compactor_stats_t *stats,
133                                    node_pointer **out_root);
134
135static couchstore_error_t compact_view_btree(tree_file *source,
136                                      tree_file *target,
137                                      const view_btree_info_t *info,
138                                      const node_pointer *root,
139                                      const bitmap_t *filterbm,
140                                      compactor_stats_t *stats,
141                                      node_pointer **out_root,
142                                      view_error_t *error_info);
143
144LIBCOUCHSTORE_API
145view_group_info_t *couchstore_read_view_group_info(FILE *in_stream,
146                                                   FILE *error_stream)
147{
148    view_group_info_t *info;
149    char buf[4096];
150    char *dup;
151    couchstore_error_t ret;
152    uint64_t type;
153
154    info = (view_group_info_t *) calloc(1, sizeof(*info));
155    if (info == NULL) {
156        fprintf(error_stream, "Memory allocation failure\n");
157        goto out_error;
158    }
159
160    type = couchstore_read_int(in_stream, buf, sizeof(buf), &ret);
161    if (ret != COUCHSTORE_SUCCESS) {
162        fprintf(stderr, "Error reading view file type\n");
163        goto out_error;
164    }
165    switch (type) {
166    case VIEW_INDEX_TYPE_MAPREDUCE:
167        info->type = VIEW_INDEX_TYPE_MAPREDUCE;
168        break;
169    case VIEW_INDEX_TYPE_SPATIAL:
170        info->type = VIEW_INDEX_TYPE_SPATIAL;
171        break;
172    default:
173        fprintf(stderr, "Invalid view file type: %"PRIu64"\n", type);
174        goto out_error;
175    }
176
177    if (couchstore_read_line(in_stream, buf, sizeof(buf)) != buf) {
178        fprintf(stderr, "Error reading source index file path\n");
179        goto out_error;
180    }
181    dup = strdup(buf);
182    if (dup == NULL) {
183        fprintf(error_stream, "Memory allocation failure\n");
184        goto out_error;
185    }
186    info->filepath = (const char *) dup;
187
188    info->header_pos = couchstore_read_int(in_stream, buf,
189                                                      sizeof(buf),
190                                                      &ret);
191    if (ret != COUCHSTORE_SUCCESS) {
192        fprintf(error_stream, "Error reading header position\n");
193        goto out_error;
194    }
195
196    info->num_btrees = couchstore_read_int(in_stream, buf,
197                                                      sizeof(buf),
198                                                      &ret);
199    if (ret != COUCHSTORE_SUCCESS) {
200        fprintf(error_stream, "Error reading number of btrees\n");
201        goto out_error;
202    }
203
204    ret = read_btree_info(info, in_stream, error_stream);
205    if (ret != COUCHSTORE_SUCCESS) {
206        goto out_error;
207    }
208
209    return info;
210
211out_error:
212    couchstore_free_view_group_info(info);
213
214    return NULL;
215}
216
217
218/* Read in the information about the mapreduce view indexes */
219static couchstore_error_t read_btree_info(view_group_info_t *info,
220                                          FILE *in_stream,
221                                          FILE *error_stream)
222{
223    char buf[4096];
224    char *dup;
225    int i, j;
226    int reduce_len;
227    couchstore_error_t ret;
228
229    info->view_infos.btree = (view_btree_info_t *)
230        calloc(info->num_btrees, sizeof(view_btree_info_t));
231    if (info->view_infos.btree == NULL) {
232        fprintf(error_stream, "Memory allocation failure on btree infos\n");
233        info->num_btrees = 0;
234        return COUCHSTORE_ERROR_ALLOC_FAIL;
235    }
236
237    for (i = 0; i < info->num_btrees; ++i) {
238        view_btree_info_t *bti = &info->view_infos.btree[i];
239
240        bti->view_id = i;
241        bti->num_reducers = couchstore_read_int(in_stream,
242                                                buf,
243                                                sizeof(buf),
244                                                &ret);
245
246        if (ret != COUCHSTORE_SUCCESS) {
247            fprintf(error_stream,
248                    "Error reading number of reducers for btree %d\n", i);
249            return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
250        }
251
252        bti->names = (const char **) calloc(bti->num_reducers, sizeof(char *));
253        if (bti->names == NULL) {
254            fprintf(error_stream,
255                    "Memory allocation failure on btree %d reducer names\n",
256                    i);
257            bti->num_reducers = 0;
258            return COUCHSTORE_ERROR_ALLOC_FAIL;
259        }
260
261        bti->reducers = (const char **) calloc(bti->num_reducers, sizeof(char *));
262        if (bti->reducers == NULL) {
263            fprintf(error_stream,
264                    "Memory allocation failure on btree %d reducers\n", i);
265            bti->num_reducers = 0;
266            free(bti->names);
267            return COUCHSTORE_ERROR_ALLOC_FAIL;
268        }
269
270        for (j = 0; j < bti->num_reducers; ++j) {
271            if (couchstore_read_line(in_stream, buf, sizeof(buf)) != buf) {
272                fprintf(error_stream,
273                        "Error reading btree %d view %d name\n", i, j);
274                return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
275            }
276            dup = strdup(buf);
277            if (dup == NULL) {
278                fprintf(error_stream,
279                        "Memory allocation failure on btree %d "
280                        "view %d name\n", i, j);
281                return COUCHSTORE_ERROR_ALLOC_FAIL;
282            }
283            bti->names[j] = (const char *) dup;
284
285            reduce_len = couchstore_read_int(in_stream,
286                                             buf,
287                                             sizeof(buf),
288                                             &ret);
289            if (ret != COUCHSTORE_SUCCESS) {
290                fprintf(error_stream,
291                        "Error reading btree %d view %d "
292                        "reduce function size\n", i, j);
293                return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
294            }
295
296            dup = (char *) malloc(reduce_len + 1);
297            if (dup == NULL) {
298                fprintf(error_stream,
299                        "Memory allocation failure on btree %d "
300                        "view %d reducer\n", i, j);
301                return COUCHSTORE_ERROR_ALLOC_FAIL;
302            }
303
304            if (fread(dup, reduce_len, 1, in_stream) != 1) {
305                fprintf(error_stream,
306                        "Error reading btree %d view %d reducer\n", i, j);
307                free(dup);
308                return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
309            }
310            dup[reduce_len] = '\0';
311            bti->reducers[j] = (const char *) dup;
312        }
313    }
314    return COUCHSTORE_SUCCESS;
315}
316
317
318LIBCOUCHSTORE_API
319void couchstore_free_view_group_info(view_group_info_t *info)
320{
321    int i, j;
322
323    if (info == NULL)
324        return;
325
326    close_view_group_file(info);
327
328    for (i = 0; i < info->num_btrees; ++i) {
329        view_btree_info_t vi = info->view_infos.btree[i];
330
331        for (j = 0; j < vi.num_reducers; ++j) {
332            free((void *) vi.names[j]);
333            free((void *) vi.reducers[j]);
334        }
335        free(vi.names);
336        free(vi.reducers);
337    }
338    free(info->view_infos.btree);
339    free((void *) info->filepath);
340    free(info);
341}
342
343
344static void close_view_group_file(view_group_info_t *info)
345{
346    if (info->file.ops != NULL) {
347        info->file.ops->close(NULL, info->file.handle);
348        info->file.ops->destructor(NULL, info->file.handle);
349        info->file.ops = NULL;
350        info->file.handle = NULL;
351    }
352    free((void *) info->file.path);
353    info->file.path = NULL;
354}
355
356
357static couchstore_error_t open_view_group_file(const char *path,
358                                               couchstore_open_flags open_flags,
359                                               tree_file *file)
360{
361    couchstore_error_t ret;
362    const couch_file_ops *file_ops = couchstore_get_default_file_ops();
363    int flags = 0;
364
365    if ((open_flags & COUCHSTORE_OPEN_FLAG_RDONLY) &&
366        (open_flags & COUCHSTORE_OPEN_FLAG_CREATE)) {
367        return COUCHSTORE_ERROR_INVALID_ARGUMENTS;
368    }
369
370    if (open_flags & COUCHSTORE_OPEN_FLAG_RDONLY) {
371        flags |= O_RDONLY;
372    } else {
373        flags |= O_RDWR;
374    }
375
376    if (open_flags & COUCHSTORE_OPEN_FLAG_CREATE) {
377        flags |= O_CREAT;
378    }
379
380    ret = tree_file_open(file, path, flags, file_ops);
381
382    return ret;
383}
384
385
386LIBCOUCHSTORE_API
387couchstore_error_t couchstore_build_view_group(view_group_info_t *info,
388                                               const char *id_records_file,
389                                               const char *kv_records_files[],
390                                               const char *dst_file,
391                                               const char *tmpdir,
392                                               uint64_t *header_pos,
393                                               view_error_t *error_info)
394{
395    couchstore_error_t ret;
396    tree_file index_file;
397    index_header_t *header = NULL;
398    node_pointer *id_root = NULL;
399    node_pointer **view_roots = NULL;
400    int i;
401
402    error_info->view_name = NULL;
403    error_info->error_msg = NULL;
404    index_file.handle = NULL;
405    index_file.ops = NULL;
406    index_file.path = NULL;
407
408    view_roots = (node_pointer **) calloc(
409        info->num_btrees, sizeof(node_pointer *));
410    if (view_roots == NULL) {
411        return COUCHSTORE_ERROR_ALLOC_FAIL;
412    }
413
414    ret = open_view_group_file(info->filepath,
415                               COUCHSTORE_OPEN_FLAG_RDONLY,
416                               &info->file);
417    if (ret != COUCHSTORE_SUCCESS) {
418        goto out;
419    }
420
421    ret = read_view_group_header(info, &header);
422    if (ret != COUCHSTORE_SUCCESS) {
423        goto out;
424    }
425    assert(info->num_btrees == header->num_views);
426
427    ret = open_view_group_file(dst_file,
428                               COUCHSTORE_OPEN_FLAG_CREATE,
429                               &index_file);
430    if (ret != COUCHSTORE_SUCCESS) {
431        goto out;
432    }
433
434    ret = build_id_btree(id_records_file, &index_file, tmpdir, &id_root);
435    if (ret != COUCHSTORE_SUCCESS) {
436        goto out;
437    }
438
439    free(header->id_btree_state);
440    header->id_btree_state = id_root;
441    id_root = NULL;
442
443    for (i = 0; i < info->num_btrees; ++i) {
444        ret = build_view_btree(kv_records_files[i],
445                               &info->view_infos.btree[i],
446                               &index_file,
447                               tmpdir,
448                               &view_roots[i],
449                               error_info);
450        if (ret != COUCHSTORE_SUCCESS) {
451            goto out;
452        }
453
454        free(header->view_states[i]);
455        header->view_states[i] = view_roots[i];
456        view_roots[i] = NULL;
457    }
458
459    ret = write_view_group_header(&index_file, header_pos, header);
460    if (ret != COUCHSTORE_SUCCESS) {
461        goto out;
462    }
463
464    ret = COUCHSTORE_SUCCESS;
465
466out:
467    free_index_header(header);
468    close_view_group_file(info);
469    tree_file_close(&index_file);
470    free(id_root);
471    if (view_roots != NULL) {
472        for (i = 0; i < info->num_btrees; ++i) {
473            free(view_roots[i]);
474        }
475        free(view_roots);
476    }
477
478    return ret;
479}
480
481
482/*
483 * Similar to util.c:read_view_record(), but it uses arena allocator, which is
484 * required for the existing semantics/api of btree bottom-up build in
485 * src/btree_modify.cc.
486 */
487static int read_record(FILE *f, arena *a, sized_buf *k, sized_buf *v,
488                                                        uint8_t *op)
489{
490    uint16_t klen;
491    uint8_t oplen = 0;
492    uint32_t vlen, len;
493
494    if (fread(&len, sizeof(len), 1, f) != 1) {
495        if (feof(f)) {
496            return 0;
497        } else {
498            return COUCHSTORE_ERROR_READ;
499        }
500    }
501
502    /* For incremental update, read optype */
503    if (op != NULL) {
504        if (fread(op, sizeof(uint8_t), 1, f) != 1) {
505            return COUCHSTORE_ERROR_READ;
506        }
507
508        oplen = 1;
509    }
510
511    if (fread(&klen, sizeof(klen), 1, f) != 1) {
512        return COUCHSTORE_ERROR_READ;
513    }
514
515    klen = ntohs(klen);
516    vlen = len - sizeof(klen) - klen - oplen;
517
518    k->size = klen;
519    k->buf = (char *) arena_alloc(a, k->size);
520    if (k->buf == NULL) {
521        return COUCHSTORE_ERROR_ALLOC_FAIL;
522    }
523
524    v->size = vlen;
525    /* Handle zero len vals */
526    if (v->size) {
527        v->buf = (char *) arena_alloc(a, v->size);
528        if (v->buf == NULL) {
529            return COUCHSTORE_ERROR_ALLOC_FAIL;
530        }
531    } else {
532        v->buf = NULL;
533    }
534
535    if (fread(k->buf, k->size, 1, f) != 1) {
536        return FILE_MERGER_ERROR_FILE_READ;
537    }
538
539    if (v->size && fread(v->buf, v->size, 1, f) != 1) {
540        return FILE_MERGER_ERROR_FILE_READ;
541    }
542
543    return len;
544}
545
546
547static int id_btree_cmp(const sized_buf *key1, const sized_buf *key2)
548{
549    return view_id_cmp(key1, key2, NULL);
550}
551
552
553static int view_btree_cmp(const sized_buf *key1, const sized_buf *key2)
554{
555    return view_key_cmp(key1, key2, NULL);
556}
557
558
559/*
560 * For initial btree build, feed the btree builder as soon as
561 * sorted records are available.
562 */
563static file_merger_error_t build_btree_record_callback(void *buf, void *ctx)
564{
565    int ret;
566    sized_buf *k, *v;
567    view_file_merge_record_t *rec = (view_file_merge_record_t *) buf;
568    view_file_merge_ctx_t *merge_ctx = (view_file_merge_ctx_t *) ctx;
569    view_btree_builder_ctx_t *build_ctx =
570                    (view_btree_builder_ctx_t *) merge_ctx->user_ctx;
571    sized_buf src_k, src_v;
572
573    src_k.size = rec->ksize;
574    src_k.buf = VIEW_RECORD_KEY(rec);
575    src_v.size = rec->vsize;
576    src_v.buf = VIEW_RECORD_VAL(rec);
577
578    k = arena_copy_buf(build_ctx->transient_arena, &src_k);
579    v = arena_copy_buf(build_ctx->transient_arena, &src_v);
580    ret = mr_push_item(k, v, build_ctx->modify_result);
581
582    if (ret != COUCHSTORE_SUCCESS) {
583        return ret;
584    }
585
586    if (build_ctx->modify_result->count == 0) {
587        arena_free_all(build_ctx->transient_arena);
588    }
589
590    return ret;
591}
592
593
594static couchstore_error_t build_btree(const char *source_file,
595                                      tree_file *dest_file,
596                                      compare_info *cmp,
597                                      reduce_fn reduce_fun,
598                                      reduce_fn rereduce_fun,
599                                      const char *tmpdir,
600                                      sort_record_fn sort_fun,
601                                      void *reduce_ctx,
602                                      node_pointer **out_root)
603{
604    couchstore_error_t ret = COUCHSTORE_SUCCESS;
605    arena *transient_arena = new_arena(0);
606    arena *persistent_arena = new_arena(0);
607    couchfile_modify_result *mr;
608    view_btree_builder_ctx_t build_ctx;
609
610    if (transient_arena == NULL || persistent_arena == NULL) {
611        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
612        goto out;
613    }
614
615    mr = new_btree_modres(persistent_arena,
616                          transient_arena,
617                          dest_file,
618                          cmp,
619                          reduce_fun,
620                          rereduce_fun,
621                          reduce_ctx,
622                          VIEW_KV_CHUNK_THRESHOLD + (VIEW_KV_CHUNK_THRESHOLD / 3),
623                          VIEW_KP_CHUNK_THRESHOLD + (VIEW_KP_CHUNK_THRESHOLD / 3));
624    if (mr == NULL) {
625        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
626        goto out;
627    }
628
629    build_ctx.transient_arena = transient_arena;
630    build_ctx.modify_result = mr;
631
632    ret = (couchstore_error_t) sort_fun(source_file,
633                                        tmpdir,
634                                        build_btree_record_callback, &build_ctx);
635    if (ret != COUCHSTORE_SUCCESS) {
636        goto out;
637    }
638
639    *out_root = complete_new_btree(mr, &ret);
640    if (ret != COUCHSTORE_SUCCESS) {
641        goto out;
642    }
643
644    /* Don't care about success/failure. Erlang side will eventually delete it. */
645    remove(source_file);
646
647out:
648    if (transient_arena != NULL) {
649        delete_arena(transient_arena);
650    }
651    if (persistent_arena != NULL) {
652        delete_arena(persistent_arena);
653    }
654
655    return ret;
656}
657
658
659static couchstore_error_t build_id_btree(const char *source_file,
660                                         tree_file *dest_file,
661                                         const char *tmpdir,
662                                         node_pointer **out_root)
663{
664    couchstore_error_t ret;
665    compare_info cmp;
666
667    cmp.compare = id_btree_cmp;
668
669    ret = build_btree(source_file,
670                      dest_file,
671                      &cmp,
672                      view_id_btree_reduce,
673                      view_id_btree_rereduce,
674                      tmpdir,
675                      sort_view_ids_file,
676                      NULL,
677                      out_root);
678
679    return ret;
680}
681
682
683static couchstore_error_t build_view_btree(const char *source_file,
684                                           const view_btree_info_t *info,
685                                           tree_file *dest_file,
686                                           const char *tmpdir,
687                                           node_pointer **out_root,
688                                           view_error_t *error_info)
689{
690    couchstore_error_t ret;
691    compare_info cmp;
692    view_reducer_ctx_t *red_ctx = NULL;
693    char *error_msg = NULL;
694
695    cmp.compare = view_btree_cmp;
696    red_ctx = make_view_reducer_ctx(info->reducers,
697                                    info->num_reducers,
698                                    &error_msg);
699    if (red_ctx == NULL) {
700        error_info->error_msg = (const char *) error_msg;
701        error_info->view_name = (const char *) strdup(info->names[0]);
702        return COUCHSTORE_ERROR_REDUCER_FAILURE;
703    }
704
705    ret = build_btree(source_file,
706                      dest_file,
707                      &cmp,
708                      view_btree_reduce,
709                      view_btree_rereduce,
710                      tmpdir,
711                      sort_view_kvs_file,
712                      red_ctx,
713                      out_root);
714
715    if (ret != COUCHSTORE_SUCCESS) {
716        set_error_info(info, red_ctx->error, ret, error_info);
717    }
718
719    free_view_reducer_ctx(red_ctx);
720
721    return ret;
722}
723
724
725couchstore_error_t read_view_group_header(view_group_info_t *info,
726                                          index_header_t **header)
727{
728    couchstore_error_t ret;
729    cs_off_t pos = info->header_pos;
730    tree_file *file = &info->file;
731    char *header_buf = NULL;
732    int header_len;
733    char buf;
734
735    if (file->handle == NULL) {
736        return COUCHSTORE_ERROR_FILE_CLOSED;
737    }
738
739    if (info->file.ops->pread(NULL, file->handle, &buf, 1, pos) != 1) {
740        return COUCHSTORE_ERROR_READ;
741    }
742    if (buf == 0) {
743        return COUCHSTORE_ERROR_NO_HEADER;
744    } else if (buf != 1) {
745        return COUCHSTORE_ERROR_CORRUPT;
746    }
747
748    header_len = pread_header(file, pos, &header_buf, MAX_HEADER_SIZE);
749    if (header_len < 0) {
750        return (couchstore_error_t) header_len;
751    }
752
753    ret = decode_index_header(header_buf, (size_t) header_len, header);
754    free(header_buf);
755
756    return ret;
757}
758
759couchstore_error_t write_view_group_header(tree_file *file,
760                                           uint64_t *pos,
761                                           const index_header_t *header)
762{
763    couchstore_error_t ret;
764    sized_buf buf = { NULL, 0 };
765    cs_off_t p;
766
767    if (file->handle == NULL) {
768        return COUCHSTORE_ERROR_FILE_CLOSED;
769    }
770
771    ret = encode_index_header(header, &buf.buf, &buf.size);
772    if (ret != COUCHSTORE_SUCCESS) {
773        return ret;
774    }
775
776    ret = write_header(file, &buf, &p);
777    if (ret != COUCHSTORE_SUCCESS) {
778        goto out;
779    }
780
781    assert(p >= 0);
782    *pos = (uint64_t) p;
783
784out:
785    free(buf.buf);
786
787    return ret;
788}
789
790
791static couchstore_error_t view_id_bitmask(const node_pointer *root, bitmap_t *bm)
792{
793    view_id_btree_reduction_t *r = NULL;
794    couchstore_error_t errcode;
795    if (root == NULL) {
796        return COUCHSTORE_SUCCESS;
797    }
798
799    errcode = decode_view_id_btree_reduction(root->reduce_value.buf, &r);
800    if (errcode != COUCHSTORE_SUCCESS) {
801        goto cleanup;
802    }
803
804    union_bitmaps(bm, &r->partitions_bitmap);
805
806cleanup:
807    free_view_id_btree_reduction(r);
808    return errcode;
809}
810
811
812static couchstore_error_t view_bitmask(const node_pointer *root, bitmap_t *bm)
813{
814    view_btree_reduction_t *r = NULL;
815    couchstore_error_t errcode;
816    if (root == NULL) {
817        return COUCHSTORE_SUCCESS;
818    }
819
820    errcode = decode_view_btree_reduction(root->reduce_value.buf,
821                                          root->reduce_value.size, &r);
822    if (errcode != COUCHSTORE_SUCCESS) {
823        goto cleanup;
824    }
825
826    union_bitmaps(bm, &r->partitions_bitmap);
827
828cleanup:
829    free_view_btree_reduction(r);
830    return errcode;
831}
832
833
834static couchstore_error_t cleanup_btree(tree_file *file,
835                                        node_pointer *root,
836                                        compare_info *cmp,
837                                        reduce_fn reduce,
838                                        reduce_fn rereduce,
839                                        purge_kv_fn purge_kv,
840                                        purge_kp_fn purge_kp,
841                                        view_purger_ctx_t *purge_ctx,
842                                        view_reducer_ctx_t *red_ctx,
843                                        node_pointer **out_root)
844{
845    couchstore_error_t errcode;
846    couchfile_modify_request rq;
847
848    rq.cmp = *cmp;
849    rq.file = file;
850    rq.actions = NULL;
851    rq.num_actions = 0;
852    rq.reduce = reduce;
853    rq.rereduce = rereduce;
854    rq.compacting = 0;
855    rq.kv_chunk_threshold = VIEW_KV_CHUNK_THRESHOLD;
856    rq.kp_chunk_threshold = VIEW_KP_CHUNK_THRESHOLD;
857    rq.purge_kp = purge_kp;
858    rq.purge_kv = purge_kv;
859    rq.enable_purging = 1;
860    rq.guided_purge_ctx = purge_ctx;
861    rq.user_reduce_ctx = red_ctx;
862
863    *out_root = guided_purge_btree(&rq, root, &errcode);
864
865    return errcode;
866}
867
868static couchstore_error_t cleanup_id_btree(tree_file *file,
869                                           node_pointer *root,
870                                           node_pointer **out_root,
871                                           view_purger_ctx_t *purge_ctx,
872                                           view_error_t *error_info)
873{
874    couchstore_error_t ret;
875    compare_info cmp;
876
877    cmp.compare = id_btree_cmp;
878
879    ret = cleanup_btree(file,
880                        root,
881                        &cmp,
882                        view_id_btree_reduce,
883                        view_id_btree_rereduce,
884                        view_id_btree_purge_kv,
885                        view_id_btree_purge_kp,
886                        purge_ctx,
887                        NULL,
888                        out_root);
889
890    return ret;
891}
892
893
894static couchstore_error_t cleanup_view_btree(tree_file *file,
895                                             node_pointer *root,
896                                             const view_btree_info_t *info,
897                                             node_pointer **out_root,
898                                             view_purger_ctx_t *purge_ctx,
899                                             view_error_t *error_info)
900{
901    couchstore_error_t ret;
902    compare_info cmp;
903    view_reducer_ctx_t *red_ctx = NULL;
904    char *error_msg = NULL;
905
906    cmp.compare = view_btree_cmp;
907    red_ctx = make_view_reducer_ctx(info->reducers,
908                                    info->num_reducers,
909                                    &error_msg);
910    if (red_ctx == NULL) {
911        error_info->error_msg = (const char *) error_msg;
912        error_info->view_name = (const char *) strdup(info->names[0]);
913        return COUCHSTORE_ERROR_REDUCER_FAILURE;
914    }
915
916    ret = cleanup_btree(file,
917                        root,
918                        &cmp,
919                        view_btree_reduce,
920                        view_btree_rereduce,
921                        view_btree_purge_kv,
922                        view_btree_purge_kp,
923                        purge_ctx,
924                        red_ctx,
925                        out_root);
926
927    if (ret != COUCHSTORE_SUCCESS) {
928        char *error_msg = NULL;
929
930        if (red_ctx->error != NULL) {
931            error_msg = strdup(red_ctx->error);
932        } else {
933            error_msg = (char *) malloc(64);
934            if (error_msg != NULL) {
935                sprintf(error_msg, "%d", ret);
936            }
937        }
938        error_info->error_msg = (const char *) error_msg;
939        error_info->view_name = (const char *) strdup(info->names[0]);
940    }
941
942    free_view_reducer_ctx(red_ctx);
943
944    return ret;
945}
946
947
948LIBCOUCHSTORE_API
949couchstore_error_t couchstore_cleanup_view_group(view_group_info_t *info,
950                                                 uint64_t *header_pos,
951                                                 uint64_t *purge_count,
952                                                 view_error_t *error_info)
953{
954    couchstore_error_t ret;
955    tree_file index_file;
956    index_header_t *header = NULL;
957    node_pointer *id_root = NULL;
958    node_pointer **view_roots = NULL;
959    view_purger_ctx_t purge_ctx;
960    bitmap_t bm_cleanup;
961    int i;
962
963    memset(&bm_cleanup, 0, sizeof(bm_cleanup));
964    error_info->view_name = NULL;
965    error_info->error_msg = NULL;
966    index_file.handle = NULL;
967    index_file.ops = NULL;
968    index_file.path = NULL;
969
970    view_roots = (node_pointer **) calloc(
971        info->num_btrees, sizeof(node_pointer *));
972    if (view_roots == NULL) {
973        return COUCHSTORE_ERROR_ALLOC_FAIL;
974    }
975
976    /* Read info from current index viewgroup file */
977    ret = open_view_group_file(info->filepath,
978                               COUCHSTORE_OPEN_FLAG_RDONLY,
979                               &info->file);
980    if (ret != COUCHSTORE_SUCCESS) {
981        goto cleanup;
982    }
983
984    ret = read_view_group_header(info, &header);
985    if (ret != COUCHSTORE_SUCCESS) {
986        goto cleanup;
987    }
988    assert(info->num_btrees == header->num_views);
989
990    /* Setup purger context */
991    purge_ctx.count = 0;
992    purge_ctx.cbitmask = header->cleanup_bitmask;
993
994    ret = open_view_group_file(info->filepath,
995                               0,
996                               &index_file);
997    if (ret != COUCHSTORE_SUCCESS) {
998        goto cleanup;
999    }
1000
1001    index_file.pos = index_file.ops->goto_eof(&index_file.lastError,
1002                                                         index_file.handle);
1003
1004    /* Cleanup id_bree */
1005    ret = cleanup_id_btree(&index_file, header->id_btree_state, &id_root,
1006                                                                &purge_ctx,
1007                                                                error_info);
1008    if (ret != COUCHSTORE_SUCCESS) {
1009        goto cleanup;
1010    }
1011
1012    if (header->id_btree_state != id_root) {
1013        free(header->id_btree_state);
1014    }
1015
1016    header->id_btree_state = id_root;
1017    view_id_bitmask(id_root, &bm_cleanup);
1018    id_root = NULL;
1019
1020    /* Cleanup all view btrees */
1021    for (i = 0; i < info->num_btrees; ++i) {
1022        ret = cleanup_view_btree(&index_file,
1023                                 (node_pointer *) header->view_states[i],
1024                                 &info->view_infos.btree[i],
1025                                 &view_roots[i],
1026                                 &purge_ctx,
1027                                 error_info);
1028
1029        if (ret != COUCHSTORE_SUCCESS) {
1030            goto cleanup;
1031        }
1032
1033        if (header->view_states[i] != view_roots[i]) {
1034            free(header->view_states[i]);
1035        }
1036
1037        header->view_states[i] = view_roots[i];
1038        view_bitmask(view_roots[i], &bm_cleanup);
1039        view_roots[i] = NULL;
1040    }
1041
1042    /* Set resulting cleanup bitmask */
1043    /* TODO: This code can be removed, if we do not plan for cleanup STOP command */
1044    intersect_bitmaps(&bm_cleanup, &purge_ctx.cbitmask);
1045    header->cleanup_bitmask = bm_cleanup;
1046
1047    /* Update header with new btree infos */
1048    ret = write_view_group_header(&index_file, header_pos, header);
1049    if (ret != COUCHSTORE_SUCCESS) {
1050        goto cleanup;
1051    }
1052
1053    *purge_count = purge_ctx.count;
1054    ret = COUCHSTORE_SUCCESS;
1055
1056cleanup:
1057    free_index_header(header);
1058    close_view_group_file(info);
1059    tree_file_close(&index_file);
1060    free(id_root);
1061    if (view_roots != NULL) {
1062        for (i = 0; i < info->num_btrees; ++i) {
1063            free(view_roots[i]);
1064        }
1065        free(view_roots);
1066    }
1067
1068    return ret;
1069}
1070
1071static couchstore_error_t update_btree(const char *source_file,
1072                                       tree_file *dest_file,
1073                                       const node_pointer *root,
1074                                       size_t batch_size,
1075                                       compare_info *cmp,
1076                                       reduce_fn reduce_fun,
1077                                       reduce_fn rereduce_fun,
1078                                       purge_kv_fn purge_kv,
1079                                       purge_kp_fn purge_kp,
1080                                       view_reducer_ctx_t *red_ctx,
1081                                       view_purger_ctx_t *purge_ctx,
1082                                       uint64_t *inserted,
1083                                       uint64_t *removed,
1084                                       node_pointer **out_root)
1085{
1086    couchstore_error_t ret = COUCHSTORE_SUCCESS;
1087    couchfile_modify_request rq;
1088    node_pointer *newroot = (node_pointer *) root;
1089    arena *transient_arena = new_arena(0);
1090    FILE *f = NULL;
1091    couchfile_modify_action *actions = NULL;
1092    sized_buf *keybufs = NULL, *valbufs = NULL;
1093    size_t bufsize = 0;
1094    int last_record = 0;
1095    bitmap_t empty_bm;
1096    int max_actions = MAX_ACTIONS_SIZE /
1097                (sizeof(couchfile_modify_action) + 2 * sizeof(sized_buf));
1098
1099    memset(&empty_bm, 0, sizeof(empty_bm));
1100
1101    if (transient_arena == NULL) {
1102        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1103        goto cleanup;
1104    }
1105
1106
1107    actions = (couchfile_modify_action *) calloc(
1108                                            max_actions,
1109                                            sizeof(couchfile_modify_action));
1110    if (actions == NULL) {
1111        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1112        goto cleanup;
1113    }
1114
1115    keybufs = (sized_buf *) calloc(max_actions, sizeof(sized_buf));
1116    if (keybufs == NULL) {
1117        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1118        goto cleanup;
1119    }
1120
1121    valbufs = (sized_buf *) calloc(max_actions, sizeof(sized_buf));
1122    if (valbufs == NULL) {
1123        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1124        goto cleanup;
1125    }
1126
1127    rq.cmp = *cmp;
1128    rq.file = dest_file;
1129    rq.actions = actions;
1130    rq.num_actions = 0;
1131    rq.reduce = reduce_fun;
1132    rq.rereduce = rereduce_fun;
1133    rq.compacting = 0;
1134    rq.kv_chunk_threshold = VIEW_KV_CHUNK_THRESHOLD;
1135    rq.kp_chunk_threshold = VIEW_KP_CHUNK_THRESHOLD;
1136    rq.purge_kp = purge_kp;
1137    rq.purge_kv = purge_kv;
1138    rq.guided_purge_ctx = purge_ctx;
1139    rq.enable_purging = 1;
1140    rq.user_reduce_ctx = red_ctx;
1141
1142    /* If cleanup bitmask is empty, no need to try purging */
1143    if (is_equal_bitmap(&empty_bm, &purge_ctx->cbitmask)) {
1144        rq.enable_purging = 0;
1145    }
1146
1147    f = fopen(source_file, "rb");
1148    if (f == NULL) {
1149        ret = COUCHSTORE_ERROR_OPEN_FILE;
1150        goto cleanup;
1151    }
1152
1153    while (!last_record) {
1154        int read_ret;
1155        uint8_t op;
1156
1157        read_ret = read_record(f, transient_arena, &keybufs[rq.num_actions],
1158                                                   &valbufs[rq.num_actions],
1159                                                   &op);
1160        if (read_ret == 0) {
1161            last_record = 1;
1162            goto flush;
1163        } else if (read_ret < 0) {
1164            ret = (couchstore_error_t) read_ret;
1165            goto cleanup;
1166        }
1167
1168        /* Add action */
1169        actions[rq.num_actions].type = op;
1170        actions[rq.num_actions].key = &keybufs[rq.num_actions];
1171        actions[rq.num_actions].value.data = &valbufs[rq.num_actions];
1172
1173        if (inserted && op == ACTION_INSERT) {
1174            (*inserted)++;
1175        } else if (removed && op == ACTION_REMOVE) {
1176            (*removed)++;
1177        }
1178
1179        bufsize += keybufs[rq.num_actions].size +
1180                   valbufs[rq.num_actions].size +
1181                   sizeof(uint8_t);
1182        rq.num_actions++;
1183
1184flush:
1185        if (rq.num_actions && (last_record || bufsize > batch_size ||
1186                               rq.num_actions == max_actions)) {
1187            newroot = modify_btree(&rq, newroot, &ret);
1188            if (ret != COUCHSTORE_SUCCESS) {
1189                goto cleanup;
1190            }
1191
1192            rq.num_actions = 0;
1193            bufsize = 0;
1194            arena_free_all(transient_arena);
1195        }
1196    }
1197
1198    *out_root = newroot;
1199
1200cleanup:
1201    free(actions);
1202    free(keybufs);
1203    free(valbufs);
1204
1205    if (f != NULL) {
1206        fclose(f);
1207    }
1208
1209    if (transient_arena != NULL) {
1210        delete_arena(transient_arena);
1211    }
1212
1213    return ret;
1214}
1215
1216static couchstore_error_t update_id_btree(const char *source_file,
1217                                         tree_file *dest_file,
1218                                         const node_pointer *root,
1219                                         size_t batch_size,
1220                                         view_purger_ctx_t *purge_ctx,
1221                                         uint64_t *inserted,
1222                                         uint64_t *removed,
1223                                         node_pointer **out_root)
1224{
1225    couchstore_error_t ret;
1226    compare_info cmp;
1227
1228    cmp.compare = id_btree_cmp;
1229
1230    ret = update_btree(source_file,
1231                      dest_file,
1232                      root,
1233                      batch_size,
1234                      &cmp,
1235                      view_id_btree_reduce,
1236                      view_id_btree_rereduce,
1237                      view_id_btree_purge_kv,
1238                      view_id_btree_purge_kp,
1239                      NULL,
1240                      purge_ctx,
1241                      inserted,
1242                      removed,
1243                      out_root);
1244
1245    return ret;
1246}
1247
1248
1249static couchstore_error_t update_view_btree(const char *source_file,
1250                                            const view_btree_info_t *info,
1251                                            tree_file *dest_file,
1252                                            const node_pointer *root,
1253                                            size_t batch_size,
1254                                            view_purger_ctx_t *purge_ctx,
1255                                            uint64_t *inserted,
1256                                            uint64_t *removed,
1257                                            node_pointer **out_root,
1258                                            view_error_t *error_info)
1259{
1260    couchstore_error_t ret;
1261    compare_info cmp;
1262    view_reducer_ctx_t *red_ctx = NULL;
1263    char *error_msg = NULL;
1264
1265    cmp.compare = view_btree_cmp;
1266    red_ctx = make_view_reducer_ctx(info->reducers,
1267                                    info->num_reducers,
1268                                    &error_msg);
1269    if (red_ctx == NULL) {
1270        error_info->error_msg = (const char *) error_msg;
1271        error_info->view_name = (const char *) strdup(info->names[0]);
1272        return COUCHSTORE_ERROR_REDUCER_FAILURE;
1273    }
1274
1275    ret = update_btree(source_file,
1276                      dest_file,
1277                      root,
1278                      batch_size,
1279                      &cmp,
1280                      view_btree_reduce,
1281                      view_btree_rereduce,
1282                      view_btree_purge_kv,
1283                      view_btree_purge_kp,
1284                      red_ctx,
1285                      purge_ctx,
1286                      inserted,
1287                      removed,
1288                      out_root);
1289
1290    if (ret != COUCHSTORE_SUCCESS) {
1291        set_error_info(info, red_ctx->error, ret, error_info);
1292    }
1293
1294    free_view_reducer_ctx(red_ctx);
1295
1296    return ret;
1297}
1298
1299LIBCOUCHSTORE_API
1300couchstore_error_t couchstore_update_view_group(view_group_info_t *info,
1301                                               const char *id_records_file,
1302                                               const char *kv_records_files[],
1303                                               size_t batch_size,
1304                                               const sized_buf *header_buf,
1305                                               int is_sorted,
1306                                               const char *tmp_dir,
1307                                               view_group_update_stats_t *stats,
1308                                               sized_buf *header_outbuf,
1309                                               view_error_t *error_info)
1310{
1311    couchstore_error_t ret;
1312    tree_file index_file;
1313    index_header_t *header = NULL;
1314    node_pointer *id_root = NULL;
1315    node_pointer **view_roots = NULL;
1316    view_purger_ctx_t purge_ctx;
1317    bitmap_t bm_cleanup;
1318    int i;
1319
1320    ret = decode_index_header(header_buf->buf, header_buf->size, &header);
1321    if (ret < 0) {
1322        goto cleanup;
1323    }
1324
1325    memset(&bm_cleanup, 0, sizeof(bm_cleanup));
1326
1327    error_info->view_name = NULL;
1328    error_info->error_msg = NULL;
1329    index_file.handle = NULL;
1330    index_file.ops = NULL;
1331    index_file.path = NULL;
1332
1333    view_roots = (node_pointer **) calloc(info->num_btrees,
1334                                          sizeof(node_pointer *));
1335    if (view_roots == NULL) {
1336        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1337        goto cleanup;
1338    }
1339
1340    assert(info->num_btrees == header->num_views);
1341
1342    /* Setup purger context */
1343    purge_ctx.count = 0;
1344    purge_ctx.cbitmask = header->cleanup_bitmask;
1345
1346    ret = open_view_group_file(info->filepath,
1347                               0,
1348                               &index_file);
1349    if (ret != COUCHSTORE_SUCCESS) {
1350        goto cleanup;
1351    }
1352
1353    /* Move pos to end of file */
1354    index_file.pos = index_file.ops->goto_eof(&index_file.lastError,
1355                                              index_file.handle);
1356
1357    if (!is_sorted) {
1358        ret = (couchstore_error_t) sort_view_ids_ops_file(id_records_file, tmp_dir);
1359        if (ret != COUCHSTORE_SUCCESS) {
1360            char buf[1024];
1361            snprintf(buf, sizeof(buf),
1362                    "Error sorting records file: %s", id_records_file);
1363            error_info->error_msg = strdup(buf);
1364            error_info->view_name = (const char *) strdup("id_btree");
1365            goto cleanup;
1366        }
1367    }
1368
1369    ret = update_id_btree(id_records_file, &index_file,
1370                                           header->id_btree_state,
1371                                           batch_size,
1372                                           &purge_ctx,
1373                                           &stats->ids_inserted,
1374                                           &stats->ids_removed,
1375                                           &id_root);
1376    if (ret != COUCHSTORE_SUCCESS) {
1377        goto cleanup;
1378    }
1379
1380
1381    if (header->id_btree_state != id_root) {
1382        free(header->id_btree_state);
1383    }
1384
1385    header->id_btree_state = id_root;
1386    view_id_bitmask(id_root, &bm_cleanup);
1387    id_root = NULL;
1388
1389    for (i = 0; i < info->num_btrees; ++i) {
1390        if (!is_sorted) {
1391            ret = (couchstore_error_t) sort_view_kvs_ops_file(kv_records_files[i], tmp_dir);
1392            if (ret != COUCHSTORE_SUCCESS) {
1393                char buf[1024];
1394                snprintf(buf, sizeof(buf),
1395                        "Error sorting records file: %s", kv_records_files[i]);
1396                error_info->error_msg = strdup(buf);
1397                error_info->view_name = (const char *) strdup(info->view_infos.btree[i].names[0]);
1398                goto cleanup;
1399            }
1400        }
1401
1402        ret = update_view_btree(kv_records_files[i],
1403                                &info->view_infos.btree[i],
1404                                &index_file,
1405                                header->view_states[i],
1406                                batch_size,
1407                                &purge_ctx,
1408                                &stats->kvs_inserted,
1409                                &stats->kvs_removed,
1410                                &view_roots[i],
1411                                error_info);
1412
1413        if (ret != COUCHSTORE_SUCCESS) {
1414            goto cleanup;
1415        }
1416
1417        if (header->view_states[i] != view_roots[i]) {
1418            free(header->view_states[i]);
1419        }
1420
1421        header->view_states[i] = view_roots[i];
1422        view_bitmask(view_roots[i], &bm_cleanup);
1423        view_roots[i] = NULL;
1424    }
1425
1426    /* Set resulting cleanup bitmask */
1427    intersect_bitmaps(&bm_cleanup, &purge_ctx.cbitmask);
1428    header->cleanup_bitmask = bm_cleanup;
1429    stats->purged = purge_ctx.count;
1430
1431    ret = encode_index_header(header, &header_outbuf->buf, &header_outbuf->size);
1432    if (ret != COUCHSTORE_SUCCESS) {
1433        goto cleanup;
1434    }
1435
1436    ret = COUCHSTORE_SUCCESS;
1437
1438cleanup:
1439    free_index_header(header);
1440    close_view_group_file(info);
1441    tree_file_close(&index_file);
1442    free(id_root);
1443    if (view_roots != NULL) {
1444        for (i = 0; i < info->num_btrees; ++i) {
1445            free(view_roots[i]);
1446        }
1447        free(view_roots);
1448    }
1449
1450    return ret;
1451}
1452
1453/* Add the kv pair to modify result */
1454static couchstore_error_t compact_view_fetchcb(couchfile_lookup_request *rq,
1455                                        const sized_buf *k,
1456                                        const sized_buf *v)
1457{
1458    int ret;
1459    sized_buf *k_c, *v_c;
1460    view_compact_ctx_t *ctx = (view_compact_ctx_t *) rq->callback_ctx;
1461    compactor_stats_t *stats = ctx->stats;
1462
1463    if (k == NULL || v == NULL) {
1464        return COUCHSTORE_ERROR_READ;
1465    }
1466
1467    if (ctx->filter_fun) {
1468        ret = ctx->filter_fun(k, v, ctx->filterbm);
1469        if (ret < 0) {
1470            return (couchstore_error_t) ret;
1471        } else if (ret) {
1472            return COUCHSTORE_SUCCESS;
1473        }
1474    }
1475
1476    k_c = arena_copy_buf(ctx->transient_arena, k);
1477    v_c = arena_copy_buf(ctx->transient_arena, v);
1478    ret = mr_push_item(k_c, v_c, ctx->mr);
1479    if (ret != COUCHSTORE_SUCCESS) {
1480        return ret;
1481    }
1482
1483    if (stats) {
1484        stats->inserted++;
1485        if (stats->update_fun) {
1486            stats->update_fun(stats->freq, stats->inserted);
1487        }
1488    }
1489
1490    if (ctx->mr->count == 0) {
1491        arena_free_all(ctx->transient_arena);
1492    }
1493
1494    return (couchstore_error_t) ret;
1495}
1496
1497static couchstore_error_t compact_btree(tree_file *source,
1498                                 tree_file *target,
1499                                 const node_pointer *root,
1500                                 compare_info *cmp,
1501                                 reduce_fn reduce_fun,
1502                                 reduce_fn rereduce_fun,
1503                                 compact_filter_fn filter_fun,
1504                                 view_reducer_ctx_t *red_ctx,
1505                                 const bitmap_t *filterbm,
1506                                 compactor_stats_t *stats,
1507                                 node_pointer **out_root)
1508{
1509    couchstore_error_t ret = COUCHSTORE_SUCCESS;
1510    arena *transient_arena;
1511    arena *persistent_arena;
1512    couchfile_modify_result *modify_result;
1513    couchfile_lookup_request lookup_rq;
1514    view_compact_ctx_t compact_ctx;
1515    sized_buf nullkey = {NULL, 0};
1516    sized_buf *lowkeys = &nullkey;
1517
1518    if (!root) {
1519        return COUCHSTORE_SUCCESS;
1520    }
1521
1522    transient_arena = new_arena(0);
1523    persistent_arena = new_arena(0);
1524
1525    if (transient_arena == NULL || persistent_arena == NULL) {
1526        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1527        goto cleanup;
1528    }
1529
1530    /* Create new btree on new file */
1531    modify_result = new_btree_modres(persistent_arena,
1532                          transient_arena,
1533                          target,
1534                          cmp,
1535                          reduce_fun,
1536                          rereduce_fun,
1537                          red_ctx,
1538                          VIEW_KV_CHUNK_THRESHOLD + (VIEW_KV_CHUNK_THRESHOLD / 3),
1539                          VIEW_KP_CHUNK_THRESHOLD + (VIEW_KP_CHUNK_THRESHOLD / 3));
1540    if (modify_result == NULL) {
1541        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1542        goto cleanup;
1543    }
1544
1545    compact_ctx.filter_fun = NULL;
1546    compact_ctx.mr = modify_result;
1547    compact_ctx.transient_arena = transient_arena;
1548    compact_ctx.stats = stats;
1549
1550    if (filterbm) {
1551        compact_ctx.filterbm = filterbm;
1552        compact_ctx.filter_fun = filter_fun;
1553    }
1554
1555    lookup_rq.cmp.compare = cmp->compare;
1556    lookup_rq.file = source;
1557    lookup_rq.num_keys = 1;
1558    lookup_rq.keys = &lowkeys;
1559    lookup_rq.callback_ctx = &compact_ctx;
1560    lookup_rq.fetch_callback = compact_view_fetchcb;
1561    lookup_rq.node_callback = NULL;
1562    lookup_rq.fold = 1;
1563
1564    ret = btree_lookup(&lookup_rq, root->pointer);
1565    if (ret != COUCHSTORE_SUCCESS) {
1566        goto cleanup;
1567    }
1568
1569    *out_root = complete_new_btree(modify_result, &ret);
1570
1571cleanup:
1572    if (transient_arena != NULL) {
1573        delete_arena(transient_arena);
1574    }
1575
1576    if (persistent_arena != NULL) {
1577        delete_arena(persistent_arena);
1578    }
1579
1580    return ret;
1581}
1582
1583static couchstore_error_t compact_id_btree(tree_file *source,
1584                                    tree_file *target,
1585                                    const node_pointer *root,
1586                                    const bitmap_t *filterbm,
1587                                    compactor_stats_t *stats,
1588                                    node_pointer **out_root)
1589{
1590    couchstore_error_t ret;
1591    compare_info cmp;
1592
1593    cmp.compare = id_btree_cmp;
1594
1595    ret = compact_btree(source,
1596                        target,
1597                        root,
1598                        &cmp,
1599                        view_id_btree_reduce,
1600                        view_id_btree_rereduce,
1601                        view_id_btree_filter,
1602                        NULL,
1603                        filterbm,
1604                        stats,
1605                        out_root);
1606
1607    return ret;
1608}
1609
1610static couchstore_error_t compact_view_btree(tree_file *source,
1611                                      tree_file *target,
1612                                      const view_btree_info_t *info,
1613                                      const node_pointer *root,
1614                                      const bitmap_t *filterbm,
1615                                      compactor_stats_t *stats,
1616                                      node_pointer **out_root,
1617                                      view_error_t *error_info)
1618{
1619    couchstore_error_t ret;
1620    compare_info cmp;
1621    view_reducer_ctx_t *red_ctx = NULL;
1622    char *error_msg = NULL;
1623
1624    cmp.compare = view_btree_cmp;
1625    red_ctx = make_view_reducer_ctx(info->reducers,
1626                                    info->num_reducers,
1627                                    &error_msg);
1628    if (red_ctx == NULL) {
1629        error_info->error_msg = (const char *) error_msg;
1630        error_info->view_name = (const char *) strdup(info->names[0]);
1631        return COUCHSTORE_ERROR_REDUCER_FAILURE;
1632    }
1633
1634    ret = compact_btree(source,
1635                        target,
1636                        root,
1637                        &cmp,
1638                        view_btree_reduce,
1639                        view_btree_rereduce,
1640                        view_btree_filter,
1641                        red_ctx,
1642                        filterbm,
1643                        stats,
1644                        out_root);
1645
1646    if (ret != COUCHSTORE_SUCCESS) {
1647        set_error_info(info, red_ctx->error, ret, error_info);
1648    }
1649
1650    free_view_reducer_ctx(red_ctx);
1651
1652    return ret;
1653}
1654
1655LIBCOUCHSTORE_API
1656couchstore_error_t couchstore_compact_view_group(view_group_info_t *info,
1657                                                 const char *target_file,
1658                                                 const sized_buf *header_buf,
1659                                                 compactor_stats_t *stats,
1660                                                 sized_buf *header_outbuf,
1661                                                 view_error_t *error_info)
1662{
1663    couchstore_error_t ret;
1664    tree_file index_file;
1665    tree_file compact_file;
1666    index_header_t *header = NULL;
1667    node_pointer *id_root = NULL;
1668    node_pointer **view_roots = NULL;
1669    bitmap_t *filterbm = NULL;
1670    bitmap_t emptybm;
1671    int i;
1672
1673    memset(&emptybm, 0, sizeof(bitmap_t));
1674    error_info->view_name = NULL;
1675    error_info->error_msg = NULL;
1676    index_file.handle = NULL;
1677    index_file.ops = NULL;
1678    index_file.path = NULL;
1679    compact_file.handle = NULL;
1680    compact_file.ops = NULL;
1681    compact_file.path = NULL;
1682
1683    ret = decode_index_header(header_buf->buf, header_buf->size, &header);
1684    if (ret < 0) {
1685        goto cleanup;
1686    }
1687
1688    /* Set filter bitmask if required */
1689    if (!is_equal_bitmap(&emptybm, &header->cleanup_bitmask)) {
1690        filterbm = &header->cleanup_bitmask;
1691    }
1692
1693    view_roots = (node_pointer **) calloc(info->num_btrees,
1694                                          sizeof(node_pointer *));
1695    if (view_roots == NULL) {
1696        ret = COUCHSTORE_ERROR_ALLOC_FAIL;
1697        goto cleanup;
1698    }
1699
1700    assert(info->num_btrees == header->num_views);
1701
1702    ret = open_view_group_file(info->filepath,
1703                               COUCHSTORE_OPEN_FLAG_RDONLY,
1704                               &index_file);
1705    if (ret != COUCHSTORE_SUCCESS) {
1706        goto cleanup;
1707    }
1708
1709    /*
1710     * Open target file for compaction
1711     * Expects that caller created the target file
1712     */
1713    ret = open_view_group_file(target_file, 0, &compact_file);
1714    if (ret != COUCHSTORE_SUCCESS) {
1715        goto cleanup;
1716    }
1717
1718    compact_file.pos = compact_file.ops->goto_eof(&compact_file.lastError,
1719                                                  compact_file.handle);
1720    ret = compact_id_btree(&index_file, &compact_file,
1721                                        header->id_btree_state,
1722                                        filterbm,
1723                                        stats,
1724                                        &id_root);
1725    if (ret != COUCHSTORE_SUCCESS) {
1726        goto cleanup;
1727    }
1728
1729    free(header->id_btree_state);
1730    header->id_btree_state = id_root;
1731    id_root = NULL;
1732
1733    for (i = 0; i < info->num_btrees; ++i) {
1734        ret = compact_view_btree(&index_file,
1735                                 &compact_file,
1736                                 &info->view_infos.btree[i],
1737                                 header->view_states[i],
1738                                 filterbm,
1739                                 stats,
1740                                 &view_roots[i],
1741                                 error_info);
1742
1743        if (ret != COUCHSTORE_SUCCESS) {
1744            goto cleanup;
1745        }
1746
1747        free(header->view_states[i]);
1748        header->view_states[i] = view_roots[i];
1749        view_roots[i] = NULL;
1750    }
1751
1752    header->cleanup_bitmask = emptybm;
1753    ret = encode_index_header(header, &header_outbuf->buf, &header_outbuf->size);
1754    if (ret != COUCHSTORE_SUCCESS) {
1755        goto cleanup;
1756    }
1757
1758    ret = COUCHSTORE_SUCCESS;
1759
1760cleanup:
1761    free_index_header(header);
1762    close_view_group_file(info);
1763    tree_file_close(&index_file);
1764    tree_file_close(&compact_file);
1765    free(id_root);
1766    if (view_roots != NULL) {
1767        for (i = 0; i < info->num_btrees; ++i) {
1768            free(view_roots[i]);
1769        }
1770        free(view_roots);
1771    }
1772
1773    return ret;
1774}
1775