1#!/usr/bin/python
2
3try: import simplejson as json
4except ImportError: import json
5import couchdb
6import httplib
7import urllib
8import time
9import common
10import unittest
11
12
13HOST = "localhost:5984"
14SET_NAME = "test_suite_set_view"
15NUM_PARTS = 8
16NUM_DOCS = 800000
17DDOC = {
18    "_id": "_design/test",
19    "language": "javascript",
20    "views": {
21        "mapview": {
22            "map": "function(doc) { emit(doc.integer, doc.string); }"
23        }
24    }
25}
26
27
28class TestUpdateCleanup(unittest.TestCase):
29
30    def setUp(self):
31        self._params = {
32            "host": HOST,
33            "ddoc": DDOC,
34            "nparts": NUM_PARTS,
35            "ndocs": NUM_DOCS,
36            "setname": SET_NAME,
37            "server": couchdb.Server(url = "http://" + HOST)
38            }
39        # print "Creating %d databases, %d documents per database" % (NUM_PARTS, NUM_DOCS / NUM_PARTS)
40        common.create_dbs(self._params)
41        common.populate(self._params)
42        # print "Databases created"
43        # print "Configuring set view with:"
44        # print "\tmaximum of 8 partitions"
45        # print "\tactive partitions = [0, 1, 2, 3]"
46        # print "\tpassive partitions = []"
47        common.define_set_view(self._params, [0, 1, 2, 3], [])
48
49
50    def tearDown(self):
51        # print "Deleting test data"
52        common.create_dbs(self._params, True)
53
54
55    # Test that the updater also does a cleanup.
56    def test_update_cleanup(self):
57        # print "Triggering initial view update"
58        t0 = time.time()
59        (resp, view_result) = common.query(self._params, "mapview", {"limit": "1"})
60        # print "Update took %.2f seconds" % (time.time() - t0)
61
62        # print "Verifying group info"
63
64        info = common.get_set_view_info(self._params)
65        stats = info["stats"]
66
67        self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
68        self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
69        self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
70        self.assertEqual(stats["full_updates"], 1, "1 full update done so far")
71        self.assertEqual(stats["stopped_updates"], 0, "no updater interruptions so far")
72        self.assertEqual(stats["cleanups"], 0, "0 cleanups done")
73        self.assertEqual(stats["cleanup_interruptions"], 0, "no cleanup interruptions so far")
74
75        # print "Adding new partitions 5, 6, 7 and 8 and marking partitions 1 and 4 for cleanup"
76        common.set_partition_states(self._params, active = [4, 5, 6, 7], cleanup = [0, 3])
77
78        # print "Querying view (should trigger update + cleanup)"
79        t0 = time.time()
80        (resp, view_result) = common.query(self._params, "mapview")
81        t1 = time.time()
82
83        # print "Verifying query results"
84
85        expected_total = common.set_doc_count(self._params, [1, 2, 4, 5, 6, 7])
86        self.assertEqual(view_result["total_rows"], expected_total, "total rows is %d" % expected_total)
87        self.assertEqual(len(view_result["rows"]), expected_total, "got %d tows" % expected_total)
88        common.test_keys_sorted(view_result)
89
90        all_keys = {}
91        for r in view_result["rows"]:
92            all_keys[r["key"]] = True
93
94        for key in xrange(1, self._params["ndocs"], self._params["nparts"]):
95            self.assertFalse(key in all_keys, "Key %d (partition 1) not in query result after update+cleanup" % key)
96        for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
97            self.assertTrue(key in all_keys, "Key %d (partition 2) in query result after update+cleanup" % key)
98        for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
99            self.assertTrue(key in all_keys, "Key %d (partition 3) in query result after update+cleanup" % key)
100        for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
101            self.assertFalse(key in all_keys, "Key %d (partition 4) not in query result after update+cleanup" % key)
102        for key in xrange(5, self._params["ndocs"], self._params["nparts"]):
103            self.assertTrue(key in all_keys, "Key %d (partition 5) in query result after update+cleanup" % key)
104        for key in xrange(6, self._params["ndocs"], self._params["nparts"]):
105            self.assertTrue(key in all_keys, "Key %d (partition 6) in query result after update+cleanup" % key)
106        for key in xrange(7, self._params["ndocs"], self._params["nparts"]):
107            self.assertTrue(key in all_keys, "Key %d (partition 7) in query result after update+cleanup" % key)
108        for key in xrange(8, self._params["ndocs"], self._params["nparts"]):
109            self.assertTrue(key in all_keys, "Key %d (partition 8) in query result after update+cleanup" % key)
110
111        # print "Verifying group info"
112
113        info = common.get_set_view_info(self._params)
114        stats = info["stats"]
115
116        self.assertEqual(info["active_partitions"], [1, 2, 4, 5, 6, 7], "right active partitions list")
117        self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
118        self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
119        self.assertEqual(stats["full_updates"], 2, "2 full updates done so far")
120        self.assertEqual(stats["stopped_updates"], 0, "no updater interruptions so far")
121        self.assertEqual(stats["cleanups"], 1, "1 full cleanup done")
122        self.assertEqual(stats["cleanup_interruptions"], 1, "1 cleanup interruption done")
123
124        for i in info["active_partitions"]:
125            expected_seq = common.partition_update_seq(self._params, i)
126            self.assertTrue(str(i) in info["update_seqs"], "partition %d in info.update_seqs" % (i + 1))
127            self.assertEqual(info["update_seqs"][str(i)], expected_seq, "right seq in info.update_seqs[%d]" % i)
128
129        # print "Update+cleanup took %.2f seconds, cleanup took %.2f seconds" % \
130        #    ((t1 - t0), stats["last_cleanup_duration"])
131
132
133
134